diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..324641f92 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Preserve exact unified-diff context markers in the archived measured experiment. +experiments/us-acs-code-memo-not-adopted-20260910.patch -whitespace diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0c721babe..71c21c387 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -168,6 +168,8 @@ jobs: # python3 (not bare `python`): guaranteed on the Ubuntu image, and this # step runs before any sync so it must hold with the stdlib alone. run: python3 tools/ci_test_groups.py --verify + - name: Check file-shard and workflow coverage without importing data tests + run: python3 -I -B -S packages/microcosm-build/tests/test_ci_test_groups.py - name: Verify the graph acceptance burndown is monotone # The charter's second process rule, executable: the number of red # properties in the microcosm-graph acceptance suite never rises @@ -186,7 +188,15 @@ jobs: fail-fast: false matrix: python-version: ["3.13", "3.14"] - group: [trade, spine-uk, rest] + lane: + - {"group": "trade", "shard": "1/1"} + - {"group": "spine-uk", "shard": "1/1"} + - {"group": "rest", "shard": "1/6"} + - {"group": "rest", "shard": "2/6"} + - {"group": "rest", "shard": "3/6"} + - {"group": "rest", "shard": "4/6"} + - {"group": "rest", "shard": "5/6"} + - {"group": "rest", "shard": "6/6"} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -194,10 +204,13 @@ jobs: python-version: ${{ matrix.python-version }} - name: Sync workspace without engines run: uv sync --all-packages --locked - - name: Engine-free tests (${{ matrix.group }}) + - name: Engine-free tests (${{ matrix.lane.group }} ${{ matrix.lane.shard }}) run: | - mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.group }}") - /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider + set -euo pipefail + selected="$(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.lane.group }}" --shard "${{ matrix.lane.shard }}")" + test -n "$selected" + mapfile -t files <<< "$selected" + /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25 engine-shared: # Deliberately does NOT depend on `changes`: it reads no classifier output, @@ -207,7 +220,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.13", "3.14"] + python-version: ["3.13", "3.14.4"] steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -217,8 +230,75 @@ jobs: run: uv sync --all-packages --locked --extra us --extra uk - name: Shared/spec engine tests run: | - mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list shared-spec) - /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider + set -euo pipefail + selected="$(uv run --no-sync python tools/ci_test_groups.py --list shared-spec)" + test -n "$selected" + mapfile -t files <<< "$selected" + /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25 + + spec-seed-diagnostics: + # Produce bounded diagnosis immediately, independently of the full suite. + # This remains required by ci-ok and never updates assertion goldens. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.14.4" + - name: Sync workspace with US and UK engines + run: uv sync --all-packages --locked --extra us --extra uk + - name: Check diagnostic bootstrap and failure reporting + timeout-minutes: 2 + env: + PYTHONDONTWRITEBYTECODE: "1" + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + run: >- + uv run --no-sync pytest --noconftest -p no:cacheprovider -q + packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py + packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py + packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py + packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py + packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py + - name: Derive candidate spec and seed identities (diagnostic only) + id: spec_seed_diagnostics + timeout-minutes: 15 + env: + DIAG_WORKFLOW_SHA: ${{ github.workflow_sha }} + DIAG_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + DIAG_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + DIAG_MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + PYTHONDONTWRITEBYTECODE: "1" + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + VECLIB_MAXIMUM_THREADS: "1" + NUMEXPR_NUM_THREADS: "1" + BLIS_NUM_THREADS: "1" + POPULACE_FIT_N_JOBS: "1" + POPULACE_FIT_PREDICT_WORKERS: "1" + run: | + DIAG_CHECKOUT_SHA="$(git rev-parse --verify HEAD)" + DIAG_UV_VERSION="$(uv --version)" + export DIAG_CHECKOUT_SHA DIAG_UV_VERSION + uv run --no-sync python tools/spec_seed_identity_diagnostics.py \ + --output-dir "$RUNNER_TEMP/spec-seed-diagnostics" + - name: Retain bounded candidate identity evidence + if: ${{ always() && steps.spec_seed_diagnostics.outcome != 'skipped' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: spec-seed-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}-py3.14.4 + path: | + ${{ runner.temp }}/spec-seed-diagnostics/candidate-digests.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-protocol.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-map.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-bindings.json + ${{ runner.temp }}/spec-seed-diagnostics/environment-and-source.json + ${{ runner.temp }}/spec-seed-diagnostics/diagnostic-status.json + if-no-files-found: error + retention-days: 3 + compression-level: 0 + include-hidden-files: false engine-us: needs: changes @@ -228,7 +308,14 @@ jobs: fail-fast: false matrix: python-version: ["3.13", "3.14"] - group: [us-p, us-qs, us-not, us-am] + lane: + - {"group": "us-p", "shard": "1/1"} + - {"group": "us-qs", "shard": "1/1"} + - {"group": "us-not", "shard": "1/1"} + - {"group": "us-am", "shard": "1/4"} + - {"group": "us-am", "shard": "2/4"} + - {"group": "us-am", "shard": "3/4"} + - {"group": "us-am", "shard": "4/4"} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -236,12 +323,18 @@ jobs: python-version: ${{ matrix.python-version }} - name: Sync workspace with US and UK engines run: uv sync --all-packages --locked --extra us --extra uk - - name: US engine tests (${{ matrix.group }}) + - name: US engine tests (${{ matrix.lane.group }} ${{ matrix.lane.shard }}) run: | - while IFS= read -r proc; do - mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.group }}:${proc}") - /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider - done < <(uv run --no-sync python tools/ci_test_groups.py --procs "${{ matrix.group }}") + set -euo pipefail + selected_procs="$(uv run --no-sync python tools/ci_test_groups.py --procs "${{ matrix.lane.group }}")" + test -n "$selected_procs" + mapfile -t procs <<< "$selected_procs" + for proc in "${procs[@]}"; do + selected="$(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.lane.group }}:${proc}" --shard "${{ matrix.lane.shard }}")" + test -n "$selected" + mapfile -t files <<< "$selected" + /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25 + done engine-uk: needs: changes @@ -261,10 +354,16 @@ jobs: run: uv sync --all-packages --locked --extra us --extra uk - name: UK engine tests run: | - while IFS= read -r proc; do - mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list "uk:${proc}") - /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider - done < <(uv run --no-sync python tools/ci_test_groups.py --procs uk) + set -euo pipefail + selected_procs="$(uv run --no-sync python tools/ci_test_groups.py --procs uk)" + test -n "$selected_procs" + mapfile -t procs <<< "$selected_procs" + for proc in "${procs[@]}"; do + selected="$(uv run --no-sync python tools/ci_test_groups.py --list "uk:${proc}")" + test -n "$selected" + mapfile -t files <<< "$selected" + /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25 + done wheels: # The charter-mandated packaging gate: editable installs hide packaging @@ -277,6 +376,15 @@ jobs: fail-fast: false matrix: python-version: ["3.13", "3.14"] + lane: + - {"group": "wheels", "shard": "1/8"} + - {"group": "wheels", "shard": "2/8"} + - {"group": "wheels", "shard": "3/8"} + - {"group": "wheels", "shard": "4/8"} + - {"group": "wheels", "shard": "5/8"} + - {"group": "wheels", "shard": "6/8"} + - {"group": "wheels", "shard": "7/8"} + - {"group": "wheels", "shard": "8/8"} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -336,10 +444,56 @@ jobs: PY - name: Spec identity section digests (cross-environment diffing aid) run: env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/spec_envelope_digests.py be uk - - name: Run the suite against the installed wheels + - name: Add engine-free source preparation extras + # Real source fixtures require microunit for tax-unit construction and + # PyTables for pandas HDF input, alongside h5py. Request the extras from + # built wheels so their metadata, not editable installs, supplies these + # dependencies. The preceding step still checks the base wheel boundary; + # neither source extra installs a country rules engine. + run: | + frame_wheel="$(ls dist/microcosm_frame-*.whl)" + build_wheel="$(ls dist/microcosm_build-*.whl)" + uv pip install --python /tmp/wheels-venv/bin/python \ + --constraint /tmp/wheel-constraints.txt \ + "microcosm-frame[us] @ file://$(realpath "$frame_wheel")" \ + "microcosm-build[source-io] @ file://$(realpath "$build_wheel")" + env -u PYTHONPATH /tmp/wheels-venv/bin/python -I - <<'PY' + from importlib import metadata, util + from pathlib import Path + from tempfile import TemporaryDirectory + + import h5py + import pandas as pd + import tables + + assert util.find_spec("microunit") is not None, ( + "microcosm-frame[us] must install microunit into the wheel venv" + ) + # Exercise the same HDF reader used by source preparation. Imports + # alone do not establish that the compiled PyTables backend works. + with TemporaryDirectory() as directory: + path = Path(directory) / "invented-source.h5" + expected = pd.DataFrame({"person_id": [1, 2], "amount": [0.0, 12.5]}) + expected.to_hdf(path, key="person") + pd.testing.assert_frame_equal(pd.read_hdf(path, key="person"), expected) + assert util.find_spec("policyengine_us") is None + try: + metadata.distribution("policyengine-us") + except metadata.PackageNotFoundError: + pass + else: + raise AssertionError("policyengine-us must be absent from the base wheel gate") + print("microunit", metadata.version("microunit")) + print("h5py", h5py.__version__, "tables", tables.__version__) + PY + - name: Run the suite against the installed wheels (${{ matrix.lane.shard }}) run: | + set -euo pipefail + selected="$(env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/ci_test_groups.py --list wheels --shard "${{ matrix.lane.shard }}")" + test -n "$selected" + mapfile -t files <<< "$selected" /usr/bin/time -v env -u PYTHONPATH /tmp/wheels-venv/bin/python -I -m pytest \ - packages/*/tests -p no:cacheprovider + "${files[@]}" -p no:cacheprovider --durations=25 ci-ok: # The single aggregate context for branch protection. Per-leg matrix names @@ -350,7 +504,7 @@ jobs: # dependency-skip lets a failed `changes` take the engine lanes down without # any required check going red: a lane skip is accepted only when # classification itself succeeded. - needs: [changes, lint, fast, engine-shared, engine-us, engine-uk, wheels] + needs: [changes, lint, fast, engine-shared, spec-seed-diagnostics, engine-us, engine-uk, wheels] if: always() runs-on: ubuntu-latest steps: @@ -365,6 +519,7 @@ jobs: require_success lint "${{ needs.lint.result }}" require_success fast "${{ needs.fast.result }}" require_success engine-shared "${{ needs['engine-shared'].result }}" + require_success spec-seed-diagnostics "${{ needs['spec-seed-diagnostics'].result }}" require_success wheels "${{ needs.wheels.result }}" for lane in "engine-us:${{ needs['engine-us'].result }}" "engine-uk:${{ needs['engine-uk'].result }}"; do name="${lane%%:*}"; result="${lane#*:}" diff --git a/.gitignore b/.gitignore index 1e77cefd2..a83de8598 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ tools/release.env # Finder metadata — never payload .DS_Store + +# Lane-local before/after measurement staging (see docs/us-native-verification-once.md) +.measure/ diff --git a/CLAUDE.md b/CLAUDE.md index 41490df9e..e37ab5f0f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,7 +26,8 @@ PR CI (`.github/workflows/test.yml`) has four lanes — `lint`, `fast`, the diff into `shared`/`us`/`uk`. `lint` verifies `tools/ci_test_groups.py --verify`, syncs with `--locked`, and runs ruff. `fast` runs the full tracked test-file inventory without engine extras in -three groups (`trade`, `spine-uk`, `rest`); engine-gated tests skip there +three groups (`trade`, `spine-uk`, `rest`), with `rest` split across six +parallel matrix jobs per Python version; engine-gated tests skip there through whichever guard they carry — the `requires_*` markers, or the `importorskip` calls that remain the norm on the US side. `engine-shared` always syncs `--extra us --extra uk` and runs the shared/spec group. `engine-us` and @@ -39,6 +40,31 @@ shard's real wheel, install into a clean uv-export-constrained venv, assert the wheel/import boundary and spec digests, and run the suite against installed wheels. +`us-am` likewise has four matrix subdivisions. Within each, build and other-shard +tests retain separate pytest processes to isolate their import state. Every wheel +job retains the complete build/install/smoke boundary and runs one of eight +disjoint file subdivisions, including engine-only files whose markers skip when +the engine is absent. Other groups and both Python versions are unchanged. +`tools/ci_test_groups.py --list GROUP[:PROCESS] --shard INDEX/COUNT` selects +sorted whole files round-robin; indices are 1-based and counts are capped at 64. +An invalid or empty selection fails before pytest. `--verify` proves the +configured shard unions and process partitions, while the stdlib-only +`python3 -I -B -S packages/microcosm-build/tests/test_ci_test_groups.py` checks +the actual matrix and selection contract. See [CI file sharding](docs/ci-file-sharding.md). +File counts are not measured runtime weights; a single expensive module can +still dominate a job, and module-scoped fixtures must remain intact. + +After checking the base wheel boundary, the wheel lane installs the built +`microcosm-frame[us]` and `microcosm-build[source-io]` extras for engine-free +source tests. An invented pandas HDF round trip checks the compiled reader; +the country rules engine must still be absent. + +The shared engine lane also runs a bounded source/seed identity diagnostic on +Python 3.14.4 after its normal tests, including when those tests fail. Its six +JSON artifacts retain candidate digests and their complete canonical seed +records for review. They never replace the test assertions or certify coverage; +review the captured source and dependency identities before updating any pins. + New commits to a PR cancel older unfinished CI runs for that same PR. Each main-push run has a unique concurrency group, so all main-push runs remain independent and can finish validating their merged changes. @@ -53,10 +79,9 @@ are already editing that test for another reason. **Adding a test file.** It must sit directly in `packages//tests/` — flat, no subdirectories; `fixtures/` and `golden/` hold data only — and be named -`test_*.py`. The lanes run explicit file lists built from a flat pathspec, while -local `uv run pytest` and the wheels lane discover recursively, so a test parked -next to its fixtures would run locally and stay green in CI without ever -executing against an engine. `--verify` fails on such a file rather than letting +`test_*.py`. All lanes, including wheels, run explicit flat file lists, while +local `uv run pytest` discovers recursively. A test parked next to its fixtures +could run locally without being selected in CI. `--verify` fails on such a file rather than letting it hide. Build tests that exercise a country engine must be named `test_us_*` or `test_uk_*` so they land in that country's lane; an engine-dependent file named anything else falls into the always-on `shared-spec` group and runs on every PR. @@ -128,6 +153,13 @@ A sealed deny-list in `microcosm.build.us_runtime.h5_io` overrides this opt-in for known-excluded publications while preserving their scoring-only diagnostic path. +The native survey development handoff is documented in +[the native handoff guide](docs/us-native-survey-development-handoff.md). +`prepare_native_survey_development_input` in the maintained US release tool +accepts a live issued enrichment owner and returns a verified Frame checkpoint +plus missing-input/gate inventory. This development checkpoint is not accepted +as native authority, a legacy pool manifest, or a certified dataset. + ## Root journals are history, not state The root `PROGRESS*.md`, `FINAL_REPORT.md`, `*_COVERAGE_PROGRESS.md`, and diff --git a/FINAL_REPORT.md b/FINAL_REPORT.md index 7648239b2..6f588fbcc 100644 --- a/FINAL_REPORT.md +++ b/FINAL_REPORT.md @@ -1,130 +1,269 @@ -# Final report: stacked-pool to release CD-vintage provenance - -## Outcome - -The producer/consumer defect is fixed. The production stacked route now -produces the same authenticated congressional-district contract that release -preflight requires: the canonical crosswalk SHA, target vintage -`119th_congress`, and positive household -`congressional_district_geoid` support. Geography remains outside source -assembly, and the release guard still fails closed on every mismatch -(`packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py:346-406`; -`tools/build_us_fiscal_refresh_release.py:2565-2613,8608-8612`). - -## Five-part repair - -1. The stacked CLI requires path/SHA pairs for the national PUMA ladder and - canonical 117th-to-119th CD crosswalk. Both declared hashes must equal the - repository pins before the source bytes are authenticated - (`tools/build_us_multispine_pool.py:563-594,897-1004`). The real geography - operator runs immediately after operator-free source assembly and before - gap fill or cloning - (`packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py:346-406`; - `tools/build_us_multispine_pool.py:5290-5334`). - -2. The configured namespace, checkpoint identity, persisted stage receipts, - and terminal manifest bind both byte authorities, source and target - vintages, assignment declaration, overlap algorithm, operator order, seed - site/stream/value, and ordered native-household output receipt - (`tools/build_us_multispine_pool.py:814-928,1300-1469,1711-1930,3530-3635,3989-4027,4133-4215`). - -3. Nullable H5 publication accepts validated root attributes, writes them in - the same temporary HDFStore as the fixed entity tables, verifies their - exact round trip, and atomically replaces the destination only after all - checks pass. The stacked publisher supplies the crosswalk-SHA and target- - vintage attrs - (`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:1463-1699`; - `tools/build_us_multispine_pool.py:4423-4519`). Schema-9 loading also binds - those physical attrs and the live household geography/clone lineage back to - the manifest - (`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:671-923,1032-1193,1321-1364`). - -4. Release preflight now reads root attrs and the household frame from one - HDFStore snapshot through the shared fixed/table-aware reader. Its existing - equality checks for crosswalk SHA, current target vintage, and positive - household district support are unchanged - (`tools/build_us_fiscal_refresh_release.py:2565-2677`). - -5. A tiny fixture invokes the real stacked entry point, post-assembly - geography operator, fixed-H5 publisher, and real release assertion. It - proves both root attrs and positive household district support; consumer - tests reject changed attrs, changed native geography, missing lineage, and - divergent clone geography - (`packages/microcosm-build/tests/test_us_multispine_pool_tool.py:2279-2335`; - `packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py:1357-1440`; - `packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py:1061-1184`). - -## Assignment design and anti-rot - -A one-value deterministic PUMA-to-CD lookup is not defensible because 2020 -PUMAs and congressional districts do not nest. The existing national ladder -preserves observed ACS PUMA, assigns missing ASEC PUMA proportional to 2020 -PUMA population, then draws CD and county within PUMA proportional to block- -population overlap. Stable row/state/PUMA order and the ledgered -`legacy_puma_ladder` / `geography_legacy` / seed `0` contract make the draw -reproducible for fixed inputs -(`packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py:1-20,293-383,638-698`; -`packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py:870-887`; -`packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml:421-432`). - -The spec and schemas now declare the crosswalk source/vintage authority and -geography assignment, while field-usage and inventory owners cover the new -surface exactly -(`packages/microcosm-build/src/microcosm/build/us/spec/geography.yaml:3-37`; -`packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml:69-86`; -`packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml:45-53`; -`packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py:387-393,671-692,803-822`; -`packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py:348-380,1649-1712`). - -The generated US spec SHA is -`5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e`. -Coverage is exact at 42,120/42,120 fields, 49 claims, and 41/41 inventory -checks. Principal legitimate movements are pointer inventory -`6d7353c6...` to `bc4a948a...`, full checkpoint `b6a47fac...` to -`a128a85f...`, and the new geography identity `f49425ca...`; executable -versions move checkpoint 11 to 12, terminal manifest 8 to 9, and nullable H5 -materializer 2 to 3 -(`tools/spec_engine_coverage.py:42-45`; -`packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py:348-380`; -`docs/evidence/spec-engine/us-f0-coverage.json:10,775-788,1890,2027-2057,2602`; -`tools/build_us_multispine_pool.py:332-360`; -`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:75-105`). - -## Verification - -- `microcosm-calibrate`: 203 passed. -- `microcosm-data`: 318 passed / 2 skipped. -- `microcosm-fit`: 93 passed. -- `microcosm-frame`: 295 passed / 36 skipped. -- `microcosm-build` partition A: 4,155 passed / 36 skipped. -- `microcosm-build` partition B: 2,177 passed / 3 skipped. -- Accepted workspace total: **7,241 passed / 77 skipped / 0 failed**. -- The high-risk real integration and H5 consumer cases are included in those - build partitions - (`packages/microcosm-build/tests/test_us_multispine_pool_tool.py:2279-2335`; - `packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py:1357-1440`). -- Bundle freshness reports the spec SHA above; coverage freshness reports - 42,120/42,120 fields and 41/41 inventory checks - (`tools/generate_us_bundle_from_constants.py:355-437`; - `tools/spec_engine_coverage.py:378-398`). -- Repository-wide Ruff, smoke-script shell syntax, and `git diff --check` pass. - The source-blind import-graph anti-rot pin now covers the exact 69-module - graph reached by the three new shared validators - (`packages/microcosm-build/tests/test_us_spine_blindness.py:3270-3315`). - -No production pool or release build ran, no push occurred, and -`logbook-pending-chain.txt` was not touched. - -## Candidate handoff - -Stage 1 must add the canonical PUMA-ladder and CD-crosswalk path/SHA pairs -listed exactly at the end of `_LANE-NOTES.md`, run off-chain, and rebuild under -the new checkpoint identity. The old smoke checkpoints and pool publication -are not reusable: configured identity now includes both new input pins and the -geography contract, checkpoint materializer is 12, terminal manifest schema is -9, and nullable H5 materializer is 3 -(`tools/build_us_multispine_pool.py:332-360,1300-1555`; -`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:75-105`). -The six pre-existing immutable source artifacts can be reused; the Stage-1 -checkpoint namespace and pool H5 must be regenerated with the two additional -authenticated authorities (`tools/build_us_multispine_pool.py:931-1004,1292-1319`). +# Bounded completion-host corrections — 2026-09-13 + +Source corrections are complete within the requested 20-minute window beginning +22:09 UTC. **All tests, collection, project imports and runtime acceptance remain +UNRUN.** This section supersedes the earlier implementation report preserved below. +No runtime, native-capacity, scientific, release, default-adoption or PR893 change +is claimed. No commit or staging occurred: the later explicit prohibition governs +the standing commit instruction. PROGRESS.md was maintained without committing. + +## Provenance and ownership + +Worktree: `/Users/maxghenis/PolicyEngine/_worktrees/microcosm-us-completion-host-integration-20260913`. +Unchanged HEAD: `6aa745c8762c3e4a140b8168470de5827e12c5cb`. +The dirty shared worktree was intentional. This pass edited exactly: + +- `packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_completion_host.py` +- `packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py` +- `docs/us-survey-completion-host.md` +- `PROGRESS.md` +- `FINAL_REPORT.md` + +Read CLAUDE.md and the full independent review and full approved R2 plan at the +exact supplied recovered paths under +`/Users/maxghenis/PolicyEngine/_recovered/scratch-backup/893/codex-us-followup-host-integration-20260913/`. +The review's plan-location objection resulted from its search scope; the supplied +absolute plan exists and its complete file hash matches the approved hash. + +| Input | SHA-256, unchanged at final source check | +| --- | --- | +| `HOST-INTEGRATION-PLAN-R2.md` | `b402d6be9d374774e27da7bf42fb83a216305d4cddd2ad8347c8149536269a29` | +| `HOST-SOURCE-INDEPENDENT-REVIEW.md` | `f98a9927e882b3f788eaebf55863d150023efd467a9bc2fc57770225ec5c5a4e` | +| Shared `graph_atomic_survey_financial.py` | `7545d1315b978735b8da4e02ad10ad78ce3d72854e2cc31a607bbd073210fb84` | +| Frozen `graph_survey_completion.py` | `858b4f42d9d61596901797976bfb2a9bf9bca9f9820b930fb3484e3f641ef0e1` | +| Frozen `graph_property_tax_leaves.py` | `883afb4a95652dd664d636b3a2eaf8481e18f4c43e3d7a24ae0cfb51b6b68c93` | + +Earlier review/failure packets were not changed. Shared source, other tests, +inventory, CI and changelog files were not edited. A new dirty entry for +`test_us_spine_blindness.py` appeared during the pass; it is another owner's work +and was neither inspected nor changed here. No reset, clean, rebase, commit, +staging, project imports, collection/tests, project scripts, dependency resolution, +network/browser/agents, native/private/installed payloads, or execution-evidence +containers were used. Source reads, metadata/hashes, standard-library AST parsing, +direct Ruff and whitespace checks were the inspection/validation operations. + +## Corrections and source evidence + +1. **Fragment identity/counts (S1):** exact ordered Node IDs now bind receiving1, + optional roles0/2, child6 and tax3 before compilation. Receiving and role + stages also check before their intermediate compilation. The child document + count derives from retained nodes. Direct boundary controls use real retained + declarations: moving an actual tax node into child7 while reducing tax to2 + preserves the total but refuses; wrong receiving identity and reversed roles + refuse too. No issuer, executor or compiler is replaced. +2. **Knownness provenance (S2):** the portable flag derives from the child + verification bytes produced by `_child_expected` reconstruction. The existing + materialized child verifier and exact payload comparison remain mandatory + before issuance. The document hashes those same bytes. The positive assertion + binds the flag and hash to the retained payload and issued artifact hashes. + Arbitrary JSON flags acquire no authority. +3. **Roles disabled (T1):** separate test selectors author actual cold/required + 49-node status-enabled and 45-node status-disabled host runs. Each explicitly + selects the partial ACS+ASEC fixture (2/3 selection, seed41) with the full-source + teenage donor outside the selected ASEC household. They assert actual checked + views, 39/35-node retained bases, no role fragment or writes, exact incumbent + role preservation/absence, final completeness and declaration/manifest/payload/ + artifact parity, staged base hits, extension misses on cold and all required + hits. The 51-node adverse suite is not multiplied. +4. **Child adversaries/copying (T2/T6):** exact full refusal strings are + `CHILD_PROPERTY_RETAINED_VALUES_CHANGED`, + `CHILD_PROPERTY_RECONSTRUCTED_ARTIFACT_CHANGED` and + `CHILD_PROPERTY_MATERIALIZED_VERIFICATION`. Source ordering is + `verify_materialized` evidence seal -> `_current`/`_pure` -> reconstruction -> + `_artifacts` -> `_verification_value`; the parent mutation fails in `_pure`, + artifact bytes at reconstruction comparison, and producer/key mismatch at + `_verification_value`. Revocation and original-base checks remain. The comment + now states verification can reconstruct/refit. Executor `_apply_result` FILTER + calls `Frame.select`; `Frame.__init__` copies every table. Role `_patch_columns` + calls `_copied_tables(deep=True)`. The test additionally asserts the actual + base stamp is unchanged immediately after mutating the reconstructed parent. + No counterfeit issuer or replacement parent is used to avoid this cost. +5. **Ordering and conservation (T3):** assertions cover all actual prefix9 nodes, + financial/property attachment and enabled status BIND before completion + receiving. Both actual observed FILTER transitions must preserve entity + dimensions, complete tables, schema, strata and weights and append precisely + one ledger record. Source uses separate `operation="filter"` and + `policy="conserve"` fields, not a `filter_conserve` operation token. Total, + stratum and partition mass must match. Actual owners/versions are checked from + property/status writers to completion FILTER, optional role BIND, child ATTACH, + tax FILTER and final tax-leaf writer. No pseudo-node or generic API change. +6. **Authentic identity controls (T4/T5):** the existing issued tax-rebased51 run + refuses as a second completion parent. The 49-node warm handle undergoes a + same-values/different-Population-identity substitution; restoration cannot undo + revocation and its base/cold peer remain checked. Extracted private union + observation and final-I/O checks remain in the production `_extend` path. + Tests reuse actual observed51 evidence to refuse a base miss (even though the + manifest content key stays equal), a missing observation and changed retained + source/node/implementation/artifact identities. The final-I/O checker re-reads + the real source/store and implementations, without executing kernels or adding + a fit. No fake runner/owner is used. Other original final-I/O population-mutation + controls remain distinct from these exact final-identity refusals. + +The fixture's source-construction wrappers now restore before issuance so later +fixture composition cannot recursively wrap an earlier fixture's closure, replace +its new literals, or reuse the earlier restoration output directory. Source pins +remain under the existing real fixture's monkeypatch lifetime. This is authored +source logic, not demonstrated fixture execution. + +S3's empty-ordering generic suggestion is not a live host defect: the host supplies +two typed property edges and the completion FILTER follows the full cloned spine. +Frozen receiving/tax APIs and compiler/source-blindness guards remain unchanged. +The independently confirmed old status+tax42 complement-owner defect is unrelated; +its isolated successor remains root-owned. This pass does not change shared +`graph_atomic_survey_financial.py`, broaden accepted owners, or certify that path. + +## Authored selectors and finite scope — all UNRUN + +AST only: **16 test functions / 32 source-expanded cases**, up from10/20. Counts +come from literal parameter-list lengths; no pytest collection was performed. +Prefix every selector below with +`packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py::`. + +| Selector | Cases | +| --- | ---: | +| `test_refused_options_precede_source_io` | 6 | +| `test_roles_disabled_status_enabled_49_cold_required_and_output_identity` | 1 | +| `test_roles_disabled_status_disabled_45_cold_required` | 1 | +| `test_disabled_signature_and_serialized_option_parity` | 1 | +| `test_exact_51_roster_order_and_actual_required_hits` | 1 | +| `test_actual_donor_outside_selection_and_paired_child_draws` | 1 | +| `test_full_weights_geography_status_roles_and_complete_gate` | 1 | +| `test_detached_parent_refuses_without_owner_substitution` | 1 | +| `test_actual_tax_rebased_parent_refuses` | 1 | +| `test_wrong_fragment_roster_refuses_before_compile` | 3 | +| `test_actual_union_observation_and_base_cache_refusals` | 2 | +| `test_actual_union_final_io_identity_refusals_without_execution` | 4 | +| `test_explicit_scientific_identity_changes_declarations` | 3 | +| `test_incomplete_adult_refuses_actual_puf_qualification_before_fitting` | 1 | +| `test_real_child_verifier_refuses_changed_custody` | 3 | +| `test_final_source_io_mutation_permanently_revokes_host` | 2 | + +The 49 and45 selectors each construct their own pair and are selectable without +the shared51 fixture. The complete file authors seven top-level host constructions: +three cold/required pairs plus one incomplete-adult run. Each host stages a base +and extension; seven is not a graph-execution count or a fit budget. Repeated +checked views and child materialized verification can requalify/reconstruct/refit. +The four final-identity parameters each perform one positive and one adverse real +I/O check. Root must budget these reads and reconstructions explicitly. Final +population-poisoning controls remain last on their shared handles. + +## Source checks and unresolved scope + +- Standard-library `ast.parse` passed for both owned Python files; selector counts + were obtained from their AST without importing project code. +- Direct `/opt/homebrew/bin/ruff check` passed for both owned Python files. +- Direct `/opt/homebrew/bin/ruff format --check` passed: both files formatted. +- `git diff --check` returned0 without findings. Individual untracked source/test/ + doc checks used `git diff --no-index --check /dev/null PATH`; each reported no + whitespace diagnostics and returned the no-index difference status1. +- No runtime failure/success evidence was created. Some source-location searches + missed guessed filenames and were resolved with `rg`; no execution containers + were opened. Direct Ruff found no lint failures in this correction pass. + +| Remaining scope | Concrete reason and next action | +| --- | --- | +| T4 financial-only parent (`property_income=None`) | No authentic19/23 financial-only run is retained in this fixture. Author/budget a separate real issuance; do not substitute a forged state. The tax-rebased-parent branch is now covered by source-authored test. | +| T5 initial base-key/artifact and final manifest/declaration adversaries | Final source/key/implementation/artifact identity and observed/base-hit controls are authored, but not every distinct earlier/later refusal is provoked. Exact additional boundaries need authentic retained evidence and separate finite scope; do not fake executor/compiler output. | +| T5 role reconciliation and union compiler roster | A conflicting original-source role needs coherent pre-issuance source ownership, not post-issuance erasure. Exact fragments now prevent compensating counts; a compiler-corruption control would require prohibited substitution or a separately reviewed authentic construction. | +| T6 fixture/checked-view behavior | All new and existing composition assumptions remain UNRUN. Guarded execution must validate source attachment restoration, member pins, selected-child/adult completeness, and actual role/status custody. Source confirms `puf55_survey_recipients.qualify_puf55_survey_recipients` calls `require_complete_property_taxes` before recipient/source admission. | +| T7 unsupported GQ / zero weights / modeled-zero science | Positive sources remove GQ and the upstream helper deliberately requests positive weights. A valid GQ/zero-weight source fixture needs separate source-owner review/changes. The selected positive child draw does not establish modeled-zero scientific coverage. Keep unknowns unknown; no zero-fill or scope waiver. | +| Root integration and resources | Independent combined-source review; source/import-use inventories, test-spine/CI classification, identity/golden reconciliation, status42 repair, generic/country/status gates, fresh guarded stage/resource budget and all runtime acceptance remain outstanding. | + +## Exact authored file identities + +| File | Preimage SHA-256 | Final SHA-256 | +| --- | --- | --- | +| `graph_survey_completion_host.py` | `6ce066b4e1b33ed1572a5e82a3414070a5f6b7360ee451028634d0862109beac` | `25852246f2690ec754dbc62ae88173f0ba1fb8115d9aa1ea86375f81daf3df40` | +| `test_us_graph_atomic_completion_host.py` | `54678f4758f9b8a96d9dc885e2f22a5fa997b1458033dcafc754075b6f8c9857` | `14735a01e119f853935d04ea574f38f452b230d16717a0ef3ea898548753abae` | +| `docs/us-survey-completion-host.md` | `f59702f85d69298bddd8ccfe2c3e3399267c66daf70b28e348c9312fb187553f` | `d654164704eb728b75c01fa8e02f54fcae6a235164f540626827a266905fc372` | +| `PROGRESS.md` | `8680141c6857cf7a8fde48c625dc47e1fba5a1f868f07f1be09bf7bf96aeeb2f` | `b42bfb5e4de0e13a1edf45229ae857e75cc4032756d21744ecad356873274071` | +| `FINAL_REPORT.md` | `95a74f7d0a49c896e1e4934e998eb7d63166a1103978d3a6fff192491a34100c` | Reported externally in the final response to avoid a self-referential hash. | + +## Historical implementation report (preserved) + +The following report predates the independent correction pass. Its10/20 counts, +45/49 coverage gaps and earlier verification descriptions are historical; the +correction section above is the current source-only handoff. + +# US completion host source report — 2026-09-13 + +Source implementation is authored for root inspection. **All tests and runtime +checks remain UNRUN. This is not combined-host acceptance.** + +Branch: `us-completion-host-integration-20260913`. +Resolved base / unchanged HEAD: `6aa745c8762c3e4a140b8168470de5827e12c5cb`. +Frozen R2 plan verified SHA-256: +`b402d6be9d374774e27da7bf42fb83a216305d4cddd2ad8347c8149536269a29`. +Read the original independent APPROVE_SOURCE_PLAN review and repository instructions. + +Implemented explicit `household_roles=False` and `child_property=None`, strict +unsupported-mode refusals and exact explicit ChildPropertyOptions. Disabled +serialization remains conditional. The actual issued 35/39 property parent is +retained with its original preparation, store, keys, registry and legacy/property +outputs. Registries are copied before extension. A privately reconstructed full-row +receiving version, optional reconciled roles, unchanged child6 and separate tax3 +compose the union; all issued prefix nodes must hit cache. Tax declarations use +the independently completed Frame and bind the actual child verification edge. + +The existing issuer/state/checker retain the full union, all node populations and +states, registry identity and completion custody. Actual completed/support +Populations and typed store evidence reach the real child materialized verifier, +including required replay. Source/store work precedes final pure seals. Failed +completion checks revoke permanently. Old property verification receives its +retained original output. Documentation labels old routing as pre-child source +gaps, preserves modeled-zero semantics and the all-person tax/PUF gate. + +| Changed path | SHA-256 | +| --- | --- | +| `packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py` | `7545d1315b978735b8da4e02ad10ad78ce3d72854e2cc31a607bbd073210fb84` | +| `packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_completion_host.py` | `6ce066b4e1b33ed1572a5e82a3414070a5f6b7360ee451028634d0862109beac` | +| `packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py` | `54678f4758f9b8a96d9dc885e2f22a5fa997b1458033dcafc754075b6f8c9857` | +| `docs/us-survey-completion-host.md` | `f59702f85d69298bddd8ccfe2c3e3399267c66daf70b28e348c9312fb187553f` | +| `changelog.d/us-survey-completion-host.added.md` | `0f16800fb200eca15adcce43d8b44f8a73e6d89c1af79b8c85bfdee00006eef6` | +| `PROGRESS.md` | `8680141c6857cf7a8fde48c625dc47e1fba5a1f868f07f1be09bf7bf96aeeb2f` | + +Output file: `FINAL_REPORT.md` (this report). PROGRESS.md was maintained in place. +No commit or push was made: the task-specific prohibition overrides the standing +commit order. No reset, rebase, clean, staging or other-owner edit occurred. + +Static verification: +- AST parsing passed for all three owned Python files. +- `_FinancialRunState` has 41 annotated fields and its constructor has 41 + positional arguments; the new state-tail alignment was inspected statically. +- Direct `/opt/homebrew/bin/ruff check` passed for the three owned Python files. +- Direct Ruff format checks passed; only owned files were formatted. +- `git diff --check` passed for the edited tracked host/journal files. +- Exactly **10 test functions / 20 source-expanded cases** authored. No collection + or tests ran. Positive fixture: one shared cold51/required51 pair. Separate + incomplete-adult case: one host run and actual PUF qualifier refusal. Three + adverse child-custody cases reuse actual run evidence with separately constructed + private boundaries; they do not add graph fits. Final-I/O controls poison the + two positive handles last. No mocked owner or executor substitutes were added. + +Remaining work / missing acceptance seams: +- Independently review the full combined diff and the other agent's receiving/tax + seam APIs. Their files were read but never edited by this task. +- Execute the new fixture only under root's fresh bounded resource approval. + Its pre-issuance ACS housing restriction, ASEC donor sampling, coherent role and + status literals, final member pins and monetary attachment composition are + authored assumptions, not demonstrated runtime results. No outside-owned + fixture edit has been established as necessary by source inspection. +- 45/49 counts are source-derived; this test file's full positive execution target + is 51. Dedicated 45/49 execution controls, unsupported-GQ and zero-weight host + fixture coverage, and broader identity/adversarial cases remain to be added or + reconciled with existing component controls by root. +- Root owns graph/source inventories, AST import-use records, test-spine/CI + classification, identity/golden reconciliation and the exact final closure and + resource-call budget. None were updated or run here. Existing upstream component + passes do not certify this composition; generic/country/status/runtime gates + remain independently required. +- No national-scale/native/PUF-fit, scientific age-transfer, fiscal, release, + publication or default-adoption claim is made. + +Failure evidence: initial direct Ruff runs found only import-order findings in +the new helper and new test file; these were corrected and subsequent checks +passed. Several source-search commands returned missing-path/no-match statuses +while locating maintained modules; no runtime failure evidence was produced or +removed. No project imports, collection, dependency resolution, network/browser/agents, +source staging, preflight, guard, native/private payload access or installed-resource +inspection was performed. Work stayed inside the requested 20-minute source window. diff --git a/PROGRESS-908-target-snapshot-fixes.md b/PROGRESS-908-target-snapshot-fixes.md new file mode 100644 index 000000000..708880f6d --- /dev/null +++ b/PROGRESS-908-target-snapshot-fixes.md @@ -0,0 +1,145 @@ +# microcosm#908 — closing the four reviewed findings on the snapshot slice + +Lane journal for branch `calibration-target-snapshots-908-fixes-20260912`. +Journals are history, not state (see CLAUDE.md): check git/GitHub for current truth. + +Historical note, 12 September 2026: this journal records the earlier repair +lane. The lane closed at `9a325c8b0`; root independently repaired the remaining +strict-null codec case in `b8380c475`, with 52 tests passing. Current scope and +remaining integration work are in [the maintained guide](docs/calibration-target-snapshots.md). +The earlier work-in-progress and not-pushed statements below are historical. + +## State + +Merged the completed slice (`bae1887ff`) onto `origin/main` `116d46ee9`, resolved +the four overlapping spec-engine identity pins by recomputing them on the merge +ref, and am now implementing the four reviewed findings. Not pushed, no PR. + +## Review basis + +`908-final-review-review-full-report.md` (head `bae1887ff`, merge base +`35fc76dd1`) with `probe.py` / `reproductions.json`. Four open findings: + +1. **C1** — the aggregate-only claim is enforced by a denylist over arbitrary + nested JSON, so record-level vectors ride through `context` and a + mapping-valued `candidate_id`. +2. **A1** — nested metadata is copied only by `dict(...)`, so a sink mutating a + delivered payload changes the caller's mapping and the next snapshot. +3. **A2** — `_write_chunk` creates the final `history/.json` name before + writing it, so a concurrent reader sees a partial chunk, and a failed write + leaves an invalid immutable chunk behind. +4. **A3** — the public codec admits impossible metadata: `epoch > epochs`, a + non-timestamp `created_at`, a non-string `candidate_id`, and a + `best_retained` whose availability, epoch and loss contradict each other. + +## Done + +- Merged `bae1887ff` into this branch and recomputed every conflicting + identity pin on the merge ref (table below). +- 14 red regressions in `packages/microcosm-calibrate/tests/test_target_snapshots.py` + (9 of them failing on the reviewed head), then the implementation: + - **C1** — a closed, typed, bounded aggregate metadata contract + (`normalize_metadata`, `normalize_best_retained`, `METADATA_LOCATIONS`, + `MAX_METADATA_*`) applied uniformly to `context`, `search`, `selection` + and `best_retained`, with string-only identifiers checked rather than + coerced. A list is not a scalar, so a record vector has no shape to ride + in at any depth. The record-level key-name rule is kept on top, because a + *scalar* `household_id` is still record-level identity. + - **A1** — every metadata container in a delivered payload is freshly built + from immutable scalars, pinned by a structural test asserting the payload + shares no mutable object with the caller, the observer, the bound view or + the next snapshot. + - **A2** — history chunks are written and `fsync`-ed to a hidden temporary + and then published atomically with `os.link`, which refuses rather than + overwrites an existing immutable chunk. Failed writes clean up their + temporary. Atomic `latest.json`, run ownership, duplicate refusal and + bounded retention are unchanged. + - **A3** — strict public-codec validation of `created_at` (timezone-aware + ISO-8601), identifiers, `epoch <= epochs`, `sequence >= 1`, + `non_finite_rows <= n_targets`, the closed `best_retained` triple's + availability/epoch/loss consistency, and a `selected` snapshot whose epoch + contradicts its own selection receipt — paired with normalization at the + emitting edge so honest nonfinite values still serialize as explicit nulls + and counts and the observer still cannot abort a run. +- `experiments/908_review_findings_recheck.py` replays the review's own four + counterexamples against this branch; receipt in + `experiments/908-review-findings-recheck.json`. All four report closed. + +## Deliberately narrow choices + +- `context` has **no in-tree producer**. Rather than invent a wider shape for a + caller that does not exist, it takes the same flat scalar contract as the + seams `solve.py` really emits. A caller that needs structure should add a + typed seam and extend the contract, in the PR that adds the caller. +- The metadata bounds (32 entries, 64-character keys, 256-character strings) + are the narrowest values that comfortably hold everything `solve.py` emits. + +## Verification (isolated interpreter, no installs, threads=1) + +`/Users/maxghenis/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910/.venv/bin/python -I -B -S` +with this worktree's shard `src` roots ahead of that venv's site-packages and +an import-path assertion, `OMP/MKL/OPENBLAS/NUMEXPR/VECLIB_NUM_THREADS=1`. + +| battery | result | +| --- | --- | +| `packages/microcosm-calibrate/tests` | 307 passed (12 s) | +| `packages/microcosm-fit/tests` | 124 passed (13 s) — main's QRF pins intact | +| `test_spec_engine_loader.py` + `test_us_multispine_pool_tool.py` | 197 passed (309 s) | +| `packages/microcosm-build/tests -k "inventory or coverage or seed"` | 448 passed, 2 skipped (343 s) | +| `packages/microcosm-graph/tests -k parity` | 25 passed (88 s) | +| `tools/spec_engine_coverage.py --check` | 42156/42156 fields, 41/41 inventory | +| `tools/ci_test_groups.py --verify` | `verification=ok`; the snapshot test file lands in fast `rest` / engine `us-am`, never `[defaulted]` | +| `ruff check .` | clean; `ruff format --check` clean on both touched files | + +`test_target_snapshots.py` holds 52 tests, 16 of them new here and 2 existing +ones updated to the closed contract's refusal points. Replaying the whole file +against the reviewed head's `target_snapshots.py` (the only source that +differs) gives **12 failed, 40 passed** — 10 new red regressions plus the 2 +updated tests. Every finding has at least one red regression: + +- **C1** — `test_context_refuses_the_record_vectors_the_review_smuggled_through`, + `test_metadata_refuses_arbitrary_nested_payloads`, + `test_identifier_fields_must_be_strings`, + `test_supported_scalar_metadata_survives_and_is_bounded`. +- **A1** — `test_metadata_refuses_arbitrary_nested_payloads` is the review's + own aliasing counterexample; the detachment invariant itself is pinned by + `test_a_delivered_snapshot_shares_no_mutable_object_with_its_caller` and + `test_sink_mutation_cannot_reach_caller_metadata_or_the_next_snapshot`, + which pass on both heads because the flat case was never the defect. +- **A2** — `test_a_history_chunk_is_published_only_after_its_bytes_are_complete`, + `test_a_failed_chunk_write_leaves_no_partial_or_leftover_file`. +- **A3** — `test_codec_refuses_the_impossible_payloads_the_review_reproduced`, + `test_non_finite_diagnostics_stay_null_statuses_rather_than_aborting`, + `test_the_codec_is_stricter_than_the_emitting_edge_and_says_so`, + `test_created_at_is_a_timezone_aware_timestamp`. + +The remaining 6 new tests are preservation tests (the structured metadata +`solve.py` really emits, the epochs the solver really selects, duplicate chunk +refusal, and observer-off/observer-on weight parity on the L0, budget-search, +refit and proximal paths); they pass on both heads by design. + +## Next + +- Independent re-review by main before integration/PR updates. + +## Identity pins recomputed on the merge ref + +`microcosm.calibrate.solve` is attested (`_DIRECT_KERNEL_MODULES`); main's #912 +edited the attested `microcosm.fit.qrf`, so both sides' pins were stale and the +merge conflicted on all four. Each was recomputed on the merged tree, after +first proving with the same interpreter that restoring **main's** `solve.py` +into the merged tree reproduces **main's** committed pins exactly — so the +drift is attributable to the slice's `solve.py` edit, not to an environment +leak. + +| pin | main `116d46ee9` | slice `bae1887ff` | merged | +| --- | --- | --- | --- | +| `EXPECTED_HASHES["seed_protocol"]` | `553d5e0b…` | `f4dc507c…` | `d052fd87…` | +| `EXPECTED_HASHES["seed_map"]` | `20058e54…` | `32dea304…` | `d5a9694a…` | +| US resolved-spec `spec_sha256` | `1eeca53a…` | `35a3623b…` | `ff2c9703…` | +| minimal-spec loader golden | `b4659890…` | `a67bb78c…` | `8a240898…` | + +`docs/evidence/spec-engine/us-f0-coverage.json` was regenerated with +`tools/spec_engine_coverage.py` (42156/42156 fields, 41/41 inventory checks). +The graph `calibrate.adam@1` parity pin merged cleanly and re-verified green +(25 parity tests). `fit.qrf` and `simulate` pins are untouched. diff --git a/PROGRESS-908-target-snapshots.md b/PROGRESS-908-target-snapshots.md new file mode 100644 index 000000000..7a7f9dedc --- /dev/null +++ b/PROGRESS-908-target-snapshots.md @@ -0,0 +1,171 @@ +# microcosm#908 — per-target calibration estimate snapshots (bounded slice 1) + +Lane journal for branch `calibration-target-snapshots-908-20260912`. +Journals are history, not state (see CLAUDE.md): check git/GitHub for current truth. + +Historical note, 12 September 2026: the initial implementation below was +subsequently merged onto main `116d46ee9` and repaired through `b8380c475`. +Its original review and source-pin claims describe earlier revisions. +[The maintained guide](docs/calibration-target-snapshots.md) records current +component scope; country integration and issue 908 remain incomplete. + +## State + +Bounded first slice implemented, tested and committed on this branch. Not +pushed, no PR opened. #908 is NOT complete — see "Remaining #908 acceptance +scope" below. + +## Scope of this slice + +1. A shared, leaf-level typed snapshot codec in `microcosm-calibrate`: + schema name + version, aggregate-only per-target rows, ordered target + identity digest, finite-value and ordering validation, signed relative + error with the repo's existing zero-target convention. +2. Real emission from the actual Adam solver loop in + `microcosm.calibrate.solve`, at a configurable bounded cadence with an + explicit every-epoch option, opt-in (off by default). +3. Honest iterate labelling: current / best_retained / selected, with the + selected snapshot taken from the weights the solver actually returns. +4. Atomic local `latest.json` plus immutable, bounded history chunks. +5. A synthetic-only benchmark at US/UK-sized dimensions (invented matrices). + +## Out of scope for this slice (remaining #908 acceptance scope) + +- Version-2 staging upload / remote publication (PR896's country host). +- Calibration Diagnostics UI (different repository, by the issue's own text). +- Real UK/US native calibration measurements — this host is forbidden native + microdata, engine runs and installs, so no real-run cadence default is + announced here. + +## Done + +- `packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py`: + shared leaf codec. Schema name + version, aggregate-only validation + (unknown top-level keys and record-level key names refused at any depth), + compiled-order row check, finite-value check, signed relative error with + the solver's existing zero-target convention, ordered `(name, value)` + sha256 identity digest, `TargetSnapshotCadence` (bounded + `EVERY_EPOCH`), + and `TargetSnapshotWriter` (atomic `latest.json`, write-once history + chunks, bounded retention with recorded drops). +- Solver integration in `solve.py`: emission from the Adam loop and the + proximal loop off the estimate tensor the epoch's loss was already + computed from (no second forward pass, so the hard-concrete gates' RNG + stream is untouched); budget-search probes carry their own search + identity; `l0_selection` / `post_l0_refit` carry their phase; one monotone + sequence counter spans every phase; the closing `selected` snapshot comes + off the float64 estimates the final diagnostics use. +- Public opt-in parameter `target_snapshots=` on `calibrate`, + `refit_l0_selection` and `calibrate_l0_refit`; exports in the shard + `__init__`. Off by default. +- 24 new tests in `packages/microcosm-calibrate/tests/test_target_snapshots.py` + (flat path; `tools/ci_test_groups.py --verify` = ok, lands in fast `rest` + and engine `us-am`, never `[defaulted]`). +- Synthetic benchmark + receipts under `experiments/`, explicitly labelled + synthetic. +- changelog.d fragment `908-calibration-target-snapshots.added.md`. + +## Remaining #908 acceptance scope (NOT done here) + +- Version-2 staging persistence/upload of the snapshot and its history. + PR #896 is open, draft, CONFLICTING and absent from `main`; its + `calibration_progress` filters on `kind == "calibration_epoch"` and its + per-event schema is `additionalProperties: false`, so it needs a new + schema name, not a widened one. Nothing here claims remote publication. +- US host wiring (`tools/build_us_fiscal_refresh_release.py` still builds v1 + `StagingTelemetry`) and UK host cadence flags. +- Exact-k ladder phases in `microcosm.build.us_runtime.exact_k_ladder` and + the UK size-search phases in `uk_runtime.dataset_size` are not wired. +- A test asserting intermediate snapshots never substitute for the canonical + `calibration_diagnostics.json`. +- Real UK/US target/epoch counts, wall-clock, and upload time, and therefore + the production default cadence. Forbidden on this host; the default stays + off. +- Calibration Diagnostics UI (different repository, by the issue's own text). + +## Adversarial review round (same lane, before hand-off) + +Three independent review lenses (determinism, label honesty, codec/store) +raised 21 findings; each was handed to an adversarial verifier told to refute +it, and 10 survived. All 10 are fixed on this branch, with a regression test +each: + +- Enabling the observer aborted runs that succeed without it, in two ways — + duplicate compiled row labels (`row_name` is the lossy `f"{name}@{period}"`, + so `("income", 2024)` and `("income", "2024")` collide) and a non-finite + float32 in-loop estimate (the capped loss absorbs it and the run returns + weights). The digest now carries the row index instead of refusing + duplicates, and a non-finite estimate or target serializes as null with a + `non_finite_rows` count, following `diagnostics._finite`. +- The `selected` snapshot stamped the closing epoch on a retained-best + iterate, and claimed `best_retained.available: false` on a run that + retained and returned a best. Both now read the selection receipt. +- The store silently collided across runs sharing a directory (sequences + restart at 1 per observer) and could prune another run's chunks, or the + chunk it had just written. Construction now refuses a populated directory + unless explicitly adopted, a store refuses a second run's snapshots, and + pruning is scoped to the chunks that writer wrote. +- The aggregate-only scan covered a hand-listed subset of keys; it now walks + the whole payload. Numeric fields are type-checked, so a stringified number + no longer validates. + +## Source identity re-pins (disclosed) + +`microcosm.calibrate.solve` is an attested module: it is in +`_DIRECT_KERNEL_MODULES` (`spec_engine/seeds.py`) and it defines the graph +shard's `calibrate.adam@1` kernel. Editing it legitimately moves several +pinned identities. Each was re-pinned to a value computed from this branch, +after first confirming with the *same* interpreter that pristine `origin/main` +reproduces the committed pins exactly — so the drift is attributable to the +source edit, not to an environment leak: + +- `packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json` + (`implementation_hash`, `node_key`). `fit.qrf` and `simulate` untouched; the + regeneration asserted `graph.json`, `inputs.csv` and `direct.csv` come back + byte-identical, so the kernel's numbers did not change. +- `EXPECTED_HASHES["seed_protocol"]` and `["seed_map"]` in + `spec_engine/inventory_coverage.py`, plus the regenerated + `docs/evidence/spec-engine/us-f0-coverage.json`. +- The resolved-spec golden vectors in + `test_spec_engine_loader.py` and `test_us_multispine_pool_tool.py`. + +No other pin was touched. + +**These pins are correct for this branch's base (0c3f4f651) only.** +`origin/main` has since advanced to 116d46ee9 (#912, +`amend-keyed-seed-and-uniform-draws`), which itself edited the attested +`microcosm.fit.qrf` and re-pinned the same loader golden vector and pool-tool +spec digest. Every identity pin above must therefore be recomputed on the +merge ref before this branch merges, per CLAUDE.md's "CI tests the merge ref, +so merge main and re-pin". Do not treat the values here as final. + +## How this lane ran the tests (no installs, no uv sync) + +An isolated interpreter with this worktree's shard sources ahead of a shared +venv's site-packages, plus an import-path assertion so `microcosm.*` can never +resolve outside this worktree: + +```python +# run.py — invoke as: /bin/python -I -B -S run.py +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VENV_SITE = Path("/site-packages") +sys.path[:0] = [ + *sorted(str(p) for p in ROOT.glob("packages/*/src")), + str(VENV_SITE), + str(ROOT), +] +import microcosm.calibrate as _cal +assert str(ROOT) in str(Path(_cal.__file__).resolve()), _cal.__file__ +import pytest +raise SystemExit(pytest.main(sys.argv[1:])) +``` + +Note pytest `addopts` already carries `-q`, so gate on the exit code rather +than adding another `-q` (which hides the summary line). + +## Next + +- Independent review of this slice, then the staging/host wiring as a + separate slice once #896 lands. diff --git a/PROGRESS-current-asec-income-routing.md b/PROGRESS-current-asec-income-routing.md new file mode 100644 index 000000000..9f76d23d9 --- /dev/null +++ b/PROGRESS-current-asec-income-routing.md @@ -0,0 +1,175 @@ +# Current ASEC income reporting/routing source qualifier — lane progress + +Branch `current-asec-income-routing-20260912`, worktree +`_worktrees/microcosm-current-asec-income-routing-20260912`, base +`d5cbe60b2c6402648565f5139dbf7d94be209def` (reviewed US integration). + +Journal, not state. Check git/GitHub for current truth. + +## Scope + +Deliver a thin **source qualifier** plus a **pure reporting/routing projection** +for the five original-channel ASEC income families the current graph does not +yet supply: pension/annuity, IRA distributions, net property income, farm, and +other income. Consumed later by the already-owned `graph_us_survey_enrichment` +host, which root's active amount owner owns. This lane creates **no** host, no +issuer, no engine execution, no native cell writes, no release claim. + +## State + +- [x] Read the actual-current-graph trace + (`_recovered/scratch-backup/893/codex-takeover-20260912/original-tax-graph-gap-source-trace.md`) + and confirmed the current host does not call `derive_cps_carried_current_leaves` + or the preclone gap-fill. +- [x] Verified the official 2025 dictionary bytes locally: + SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +- [x] Read the accepted UC pattern + (`current_asec_unemployment_source.py`, reviewed hash + `624c76ae058e7b2a466a017eba348327c94751088d97011d3474cd8314187519`) read-only + in the successor worktree; not mutated. +- [x] Read the in-base siblings `current_social_security_source.py`, + `current_asec_demographics.py`, `current_survey_predictors.py`. +- [x] Extracted every target field's printed literal (length/position/range, + verbatim Universe and Values) from the verified dictionary, and + cross-checked each against `asec_current_money_domains_v1.json`: exact match + on position, length, universe and values for all nine money fields. +- [x] Module + `packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py` + (commit 12ac3a75b, corrected in aa7bde7d4 and 7e2c7e77b). +- [x] Tests `packages/microcosm-build/tests/test_us_current_asec_income_routing.py` + — 64 passing, no engine, no PUF fixture. +- [x] Source contract note `docs/us-current-asec-income-routing-source.md` + plus `changelog.d/us-current-asec-income-routing-source.added.md`. +- [x] `ruff check .` and `ruff format --check` clean; + `tools/ci_test_groups.py --verify` reports `verification=ok` and the new + file lands in `[fast] rest` and `[engine] us-am`, not `[defaulted]`. +- [x] Mutation-checked the regressions: default-zero completion of an ambiguous + recipient zero, reading NIU from the raw number instead of the parent + status axis, collapsing receipt-with-net-zero into known nonreceipt, and + resolving an unknown slot account into a known non-IRA zero each turn the + suite red; reverting each returns it green. + +## Second review round: corrections after adversarial review + +An independent five-lens adversarial review of the finished lane (source +fidelity, projection semantics, owner pattern, tests, repository fit) found a +further set of defects, every one of which is fixed on this branch: + +1. **Fail-closed roster (CI-breaking).** `test_us_spine_blindness.py` globs + every `us_runtime` module and fails closed on an unregistered one. The + qualifier is now classified there and listed among the reviewed source-spine + provenance owners. Three of that file's tests remain red for four modules + already unclassified at the reviewed base `d5cbe60b2` — + `graph_fiscal_dense_calibration.py`, `graph_fiscal_measurement.py`, + `graph_puf55_route_attachment.py`, `graph_survey_puf55.py` — which this lane + neither owns nor touched. **Root should route those to their owners.** +2. **Recipient zeros were completed on an unprinted rationale.** `RNT_VAL` and + `FRSE_VAL` print the same `0 = none or niu` label the gross entries print, so + a signed net measure's recipient zero is no longer a known amount. Only + `ANN_VAL`, whose printed zero is `valid_zero_dollars`, resolves; that test is + read from the pinned domains artifact rather than decided in code. +3. **Off-route distribution evidence yielded an affirmative zero.** Off-route + dollars or an answered off-route recipiency now contradict the row, and an + applicable slot declaring an account whose amount is a "none or niu" zero + leaves both the total and the regular-IRA share unresolved. +4. **Other-income routing ignored the printed receipt universe**, so an + out-of-universe row read as reported alimony. +5. **Allocation origins asserted publisher-confirmed non-allocation** from flags + whose printed universes are conditional and unevaluated. +6. **The printed zero receipt label was collapsed** to one shared `niu_or_none` + across nine entries that print three different labels. +7. **`asec_literals` escaped with no digest**, outside the final identity check. +8. **`I_FRMYN` prints an empty `Values:` block**; its codes were described as + following `I_ANNVAL`. **`FARM_AMOUNT_SCOPE` dropped its composite clause.** + The claim that every other family prints a 15+ floor was false — the farm + family prints none. +9. **The doc called `TAXABLE_PENSION_FRACTION` archived**; it is live in + `cps_carried_current.py` under the prepared-ASEC stage, so a second split + attached here would double-count. +10. Printed entry tables are `NamedTuple`s now, the account code domain is + compared against `retirement_distributions._VALID_ACCOUNT_CODES` rather than + a re-typed literal, and `project_income_routing` validates its routing token + arrays. + +## Corrections made after independent review of the first draft + +1. The money owner normalizes `ANN_VAL`'s printed `-1` to a stored zero and + records `DECLARED_NIU` (`asec_current_money.py:974-977`). The first draft + re-derived NIU from the stored number, which would have read that cell as a + zero dollar annuity and also failed the literal-identity join. The amount + reading now comes from the parent's status axis. +2. The nine printed money entries were retyped by hand; they are already + attested per vintage in `asec_current_money_domains_v1.json`. They are now + read from that packaged artifact under `money.RESOURCE_PINS[0]`. +3. The published-allocation roster wrongly listed `DST_VAL1`, `DST_VAL2`, + `DST_YN`, `DST_SC1`, `DST_SC2` and `FRMOTR` as unflagged; all six are + flagged. `OI_YN` is unflagged and was missing from the list. +4. Three printed universes were transcribed with ASCII `>=` where the + dictionary prints U+2265; they now carry the printed character. + +## Next + +Root review and integration into `graph_us_survey_enrichment`. The open +decisions are listed under "Remaining work" in the contract note: canonical +attachment and clone policy for these PUF-overlapping leaves, the ACS clone0 +conditional model and its reconciliation against the ACS aggregate anchors, the +unobserved pension and distribution tax composition, the net property +decomposition, and any other-income residual rule. + +## Verified source literals (2025 dictionary, pages 43-49 + allocation pages) + +All quoted verbatim from the pinned PDF. + +| Field | Len | Pos | Range | Universe as printed | +| --- | --- | --- | --- | --- | +| PNSN_VAL | 7 | 571 | (0:9999999) | `PEN_YN = 1` | +| PEN_YN | 1 | 570 | (0:2) | `All Persons aged 15+` | +| ANN_VAL | 6 | 438 | (-1:999999) | `ANN_YN = 1` | +| ANN_YN | 1 | 444 | (0:2) | `All Persons aged 15+` | +| DST_VAL1 | 6 | 495 | (000000:999999) | `DST_SC1 = 1` | +| DST_VAL1_YNG | 6 | 501 | (000000:999999) | `DST_SC1_YNG = 1` | +| DST_VAL2 | 6 | 507 | (000000:999999) | `DST_SC2 = 1` | +| DST_VAL2_YNG | 6 | 513 | (000000:999999) | `DST_SC2_YNG = 1` | +| DST_SC1 | 1 | 491 | (0:7) | `DST_VAL1 > 0 and a_age >= 58` | +| DST_SC1_YNG | 1 | 492 | (0:7) | `DST_YN_YNG = 1 and a_age < 58` | +| DST_SC2 | 1 | 493 | (0:7) | `DST_VAL2 > 0 and a_age >= 58` | +| DST_SC2_YNG | 1 | 494 | (0:7) | `DST_VAL_YNG > 0 and a_age < 58` | +| DST_YN | 1 | 519 | (0:2) | `Persons aged 58 and over (a_age >= 58)` | +| DST_YN_YNG | 1 | 520 | (0:2) | `Persons under age 58 (a_age < 58)` | +| RNT_VAL | 6 | 621 | (-9999:999999) | `RNT_YN = 1` | +| RNT_YN | 1 | 627 | (0:2) | `All Persons aged 15+` | +| FRSE_VAL | 7 | 390 | (-9999999:9999999) | `ERN_YN=1 or FRMOTR=1` | +| FRSE_YN | 1 | 397 | (0:2) | `ERN_YN=1 or FRMOTR=1` | +| ERN_YN | 1 | 381 | (0:2) | `WORKYN=1 OR WTEMP=1` | +| FRMOTR | 1 | 389 | (0:2) | `ERN_OTR = 1` | +| OI_VAL | 6 | 549 | (0:999999) | `OI_YN = 1` | +| OI_OFF | 2 | 547 | (0:20) | `OI_YN = 1` | +| OI_YN | 1 | 555 | (0:2) | `All Persons aged 15+` | + +Source-level questions that stay open (evidence, not modeling judgments): + +- `PNSN_VAL` is printed as "total combined amount of pension income received + from **all** pension sources". It is not an observed private/taxable amount. + The legacy 0.590 split is a modeled assumption and is not applied here. +- `RNT_YN`'s printed question covers rent, royalties, roomers/boarders **and** + estates or trusts; `RNT_VAL`'s printed question asks only about "income from + rent after expenses". The receipt and amount questions do not have the same + printed coverage, so the total is not independently labelled rental. +- `DST_SC*` code 4 is `Regular IRA`. Account identity does not observe a taxable + fraction, so no taxable component is derived. +- `DST_VAL1`'s printed universe is `DST_SC1 = 1` (401k account) although its + label is the source-1 distribution amount; preserved verbatim as an + unresolved printed-universe ambiguity rather than silently corrected. +- `DST_SC2_YNG`'s printed universe names `DST_VAL_YNG`, a field with no + dictionary entry; preserved verbatim. +- `OI_YN`'s printed `0` label is `none or niu`, unlike `PEN_YN`/`ANN_YN`/ + `RNT_YN`/`DST_YN` whose `0` is `niu`. Zero receipt literals are therefore not + interchangeable across the five families. +- Published allocation flags exist for `ANN_VAL` (`I_ANNVAL`), `ANN_YN` + (`I_ANNYN`), `PEN_YN` (`I_PENYN`), `RNT_VAL` (`I_RNTVAL`), `RNT_YN` + (`I_RNTYN`), `OI_VAL` (`I_OIVAL`), `ERN_YN` (`I_ERNYN`), and the DST + composites (`I_DSTSC`, `I_DSTSCCOMP`, `I_DSTVAL1COMP`, `I_DSTVAL2COMP`, + `I_DSTYNCOMP`). The dictionary publishes **no** direct flag for `PNSN_VAL`, + `FRSE_VAL`, `FRSE_YN`, `OI_OFF`, `DST_VAL*`, `DST_YN` or `FRMOTR`; their + allocation provenance is therefore recorded as unresolved, not as "raw". +- `I_DSTVAL1COMP`'s printed `Universe:` line is empty in the dictionary. diff --git a/PROGRESS-native-byte-transports.md b/PROGRESS-native-byte-transports.md new file mode 100644 index 000000000..076195c5a --- /dev/null +++ b/PROGRESS-native-byte-transports.md @@ -0,0 +1,111 @@ +# Lane journal: the US native build's byte-transport ceilings + +Branch `native-byte-transports`, from `native-row-ceilings` (PR #949's head, +`c5ac78d2d`). Worktree `~/PolicyEngine/_worktrees/microcosm-native-byte-transports`. +Opened 2026-09-18. Session-handoff journal: accurate when written, historical +afterward (see `CLAUDE.md`, "Root journals are history, not state"). + +Design authority once written: `docs/us-native-byte-transports.md`. Lane report: +`experiments/native-byte-transports/out.md` (root `out.md` is another lane's +tracked report and is not touched). + +## State (fifth session, 2026-09-18, from 17:20) + +The fourth session (16:20-16:17+, ended on its limit) committed the remaining +whole-roster transports, the budget view's streamed comparison, the agreement +sweep and the 1/15 harness copy; its salvage ref held only the untracked raw +census and pytest logs, both now in the tree. This session verified the two +test files that failed before the fix commit (73 and 54 passed at +`475c9ba9d`), found no pin moving, and then closed what the raw census and a +coverage grep showed open: the child lane's `graph_joint_empirical.MAX_RECIPIENTS` +(1,048,576, a row count over every stacked person, the first refusal on that +lane above about 1/3 of source) takes the row-count rule's 15,000,000; the +three `raise ... from None` sites still in `survey_population_preparation` +(`REQUEST_REFUSED`, two `PREPARATION_VERIFICATION_REFUSED`) chain their cause +like the seven already converted; and every moved bound now has a test that +drives its refusal at a patched-down value and its acceptance at a +full-source-sized count without allocating one (numeric bounds, budget +consumer, age artifact, predictor projection, `MAX_ORIGINALS`, +`MAX_RECIPIENT_ROWS`, `MAX_RECIPIENTS`), plus the byte family's rule table in +`test_us_native_row_ceilings.py`. The 1/15 harness copy keeps the retention +lane's file with three named edits: the venv is an argument (this tree's lock +moved), and the two receipt fields that lane found stale (`sample.fraction`, +`scope`) say 1/15. A 59-agent verification workflow (43 refuters, 13 batch +checkers, 3 completeness critics) is running read-only over the raw census. + +Still open at this entry: the verified census table, the design note, the +1/15 run (detached worktree `microcosm-byte-transports-fifteenth` and its +venv are built), the dependent-file battery at the finished head, the report, +the draft PR. + +## State (third session, 2026-09-18) + +The moved base (`origin/native-row-ceilings` at `0319b1c73`, merged level with +main) is merged in at `77bafbb3a`; the venv is resynced to the merged lock +(policyengine-us 2.2.1). The second session's uncommitted edits were committed +unchanged (`2ed7b0715`) and then tested: two of its new tests failed, both +fixed in the commit after this journal entry -- the digest and segmented +encoders no longer require the segment to be at most the maximum (a digest has +no accumulation), and the budget's `checked_view` no longer re-encodes the +whole document through the 64 MiB encoder (`graph._json_matches` streams the +comparison). Pins re-derived after the merge: 124 inventory contracts and all +four `_ACCEPTED` entries unchanged, ten stage manifests build. A census +workflow (nine module families, adversarial verification, completeness critic) +is running read-only over this head; no source file moves until it returns. + +Earlier state, first session, at `c5ac78d2d`: both prior reports read (#949 row ceilings, +#950 retention seal §8/§11), the transport design note, the prior census +(146 bounds, 91 byte-shaped rows), and every module that holds a named byte +bound on the 19-node and 45-node paths. `uv sync --all-packages --locked +--extra us` built `.venv` (Python 3.14.4; `uv.lock` identical to the +`microcosm-native-v5` venv the prior harnesses ran on). Baseline gates at the +untouched head: `tools/ci_test_groups.py --verify` ok; spec-engine coverage +42156/42156 and 41/41. + +## Done + +- Read at this head: `acs_person_coverage_authentication.py` (whole), + `acs_native_coverage_binding.py` (whole), the roster transport in + `survey_population_preparation.py` (`_roster_segments`, `_spill_roster`, + `_roster_payload`, the three `raise ... from None` sites), `_bounded_json` + and `_allocation_payload` in `graph_survey_population.py`, + `survey_origin_budget._document`, `graph_survey_budget`, + `graph_survey_calibration`, `graph_survey_age_artifact`, + `puf_diagnostic_consumer.current_survey_recipient_matrix`, the housing + prepared receipt, the child-property projections, and the fit adapter's + draw document. +- Established reachability: the 45-node pilot ran with household roles and + person status disabled (`_roles_disabled_pair`; pilot report §"The 45-node + graph, derived from the code"), so the roles projection and status artifact + are reachable only from the 49/51-node variants. The origin budget, numeric + bounds and age artifact run in `survey_age_calibration`, not in either + graph, but the origin budget is one of the three the brief names, and the + numeric bounds consume its payload. + +## Done (third session) + +- Merged the moved base; resynced; committed and tested the second session's + edits (ACS 116 passed with the row ceilings; child property 61; graph survey + population 70 after the fix; origin budget 54 after the fix; spine blindness + 503; preparation 75; calibration and age 67; fit adapter, child source and + predictors 95). +- Found and closed a consumer gap the second session missed: the budget's + `checked_view` re-encoded the whole document through `_bounded_json` + (`survey_origin_budget.py`, `FINAL_BUDGET_VIEW_DOCUMENT`), which would refuse + at the same 5.7% of source the producer used to. +- Wrote the census brief (`experiments/native-byte-transports/census-brief.md`) + and the seeded agreement sweep (`agreement_sweep.py`). + +## Next + +0. Act on the census: any bound it finds binding that this branch has not + lifted; then pins, the design note, the report, the 1/15 run from a + detached worktree at the finished head, the draft PR. +1. (first session's plan) Byte census: measurement script over the recovered 1/1000 artifact and the + captured public ACS archive; agent fan-out per module family with an + adversarial pass over every binding verdict; `census.json`. +2. Commit 1: the ACS serialno and body budgets. +3. The preparation consumer, the origin budget, the numeric bounds, the age + artifact, the diagnostic matrix; catch-alls carry their cause; pins. +4. Tests as CI runs them; the 1/15 run; the design note; the report; draft PR + against `native-row-ceilings`. diff --git a/PROGRESS-native-row-ceilings.md b/PROGRESS-native-row-ceilings.md new file mode 100644 index 000000000..fe0985017 --- /dev/null +++ b/PROGRESS-native-row-ceilings.md @@ -0,0 +1,95 @@ +# Lane journal: lift the native build's row-count ceilings (`native-row-ceilings`) + +Branch `native-row-ceilings`, from `origin/native-scale-transport` at `a64f7b733`. +PR base is `native-scale-transport`, draft, and it stays draft. + +This file is a session-handoff journal. Per the repo's CLAUDE.md, its +"State"/"Next" sections are accurate when written and historical afterward — +check git and GitHub for current truth. + +## The rule (the one argument) + +**A bound moves only if a full-source native build meets it, and a bound that +moves becomes four times the measured full-source count of exactly what it +counts, rounded up to the next whole million.** + +Byte transports are *not* this rule's. The transport lane's answer to a byte +ceiling is a segmented stream under `MAX_ROSTER_BYTES`, not a larger single cap, +and applying this rule to one would be the wrong argument. + +## The counts are measured, not extrapolated + +The recovered 1/1000 pilot artifact +(`_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json`, +sha256 `34b362d8…`) carries the whole ACS and ASEC **catalogues**, which do not +scale with the selection fraction. Its ACS occupied + institutional-GQ + +noninstitutional-GQ households plus its ASEC households equal its own +`selection.supplied_households` exactly (1,348,408 + 84,422 + 98,784 + 55,762 = +1,587,376), so "the whole catalogue" and "what a full-source selection supplies" +are the same set: + +| | households | persons | +|---|---|---| +| ACS | 1,531,614 | 3,422,888 | +| ASEC | 55,762 | 142,125 | +| stacked | 1,587,376 | 3,565,013 | +| combined clone | 3,174,752 | 7,130,026 | + +The transport lane's ~3,471,000 stacked / ~6,943,000 cloned were the 1/1000 +per-household ratio extrapolated; these supersede them. +`experiments/native-row-ceilings/roster_census.py` re-derives them and asserts +the reconciliation. + +## Done + +- **Census**: 146 bounds across five module families, reachable from the 19-node + financial graph and the 45-node pilot graph, each with its enforcement site, + refusal code and exception type, what it protects, its counts at 1/10 and at + full source, and whether it binds. 41 of those verdicts then went through an + adversarial pass; no headline binding call was overturned. + **14 bind at full source, 6 of them at 1/10.** +- `survey_origin_budget.MAX_GROUPS` **established**: `allocation_instructions` + requires one instruction per selected household, so a full-source budget has + 1,587,376 groups against a 1,000,000 bound. +- **Seven ceilings lifted** under the rule, each with a boundary test. +- Pins re-derived through their generators: one moved + (`acs_native_coverage_binding._ACCEPTED["acs_pums.py"]`). 124 inventory + contracts checked, 10 stage manifests built. +- **An inherited break re-pinned**: base-branch `b6081efcb` added + `path.read_bytes()` to `_spill_roster` without regenerating + `survey_population_preparation.py`'s `resource_accesses_sha256`, so + `implementation_manifest()` raised for every stage containing it. Proven + pre-existing by recomputing the contract from `origin/native-scale-transport`, + `a64f7b733` and `b6081efcb`'s own blobs, and by running the affected tests + against a base worktree: 7 fail there with that exact error and all pass here. +- `docs/us-native-row-ceilings.md`, and the rule as an executable table. + +## The three findings the report leads with + +1. **`acs_person_coverage_authentication.MAX_BODY_BYTES` admits 12,911 selected + ACS persons — 0.38% of source.** Measured over 200,000 real records of the + pilot's captured public archive. The tightest ceiling on the path, 265× under + at full source, and below 1/100. +2. **The preparation-receipt ceiling the transport lane lifted is still enforced + one module downstream** at `PREPARATION_MAX_BYTES` = 64 MiB, which admits + 96,839 households — to within rounding the exact 96,860 that lane reported as + lifted. +3. **`survey_origin_budget.MAX_PAYLOAD_BYTES` admits 87,838 households, 5.53%**, + and cannot be raised at all in that module: `graph._bounded_json` refuses any + limit above 64 MiB before encoding a byte. + +All three are byte transports and take the transport lane's argument, not this +one. All three are pinned in tests. + +## State + +Complete. **[PR #949](https://github.com/PolicyEngine/microcosm/pull/949)**, +draft against `native-scale-transport`, MERGEABLE. CI does not run on it by +design (`test.yml` triggers on `pull_request: branches: [main]`); the local +gates in the report's section 8 are the only ones it has, and all are green: +399 passed over the seven touched test files, `verification=ok`, spec-engine +42156/42156 and 41/41, ruff clean, all three pin generators idempotent. + +## Next +- Open for Max: whether the inherited re-pin stays here or moves to #945; and + whether the byte transports above are one follow-up lane or several. diff --git a/PROGRESS-native-scale-transport.md b/PROGRESS-native-scale-transport.md new file mode 100644 index 000000000..fba72b90d --- /dev/null +++ b/PROGRESS-native-scale-transport.md @@ -0,0 +1,134 @@ +# Lane journal — native scale: transport ceiling and per-node retention + +Branch `native-scale-transport`, worktree +`~/PolicyEngine/_worktrees/microcosm-native-scale`, stacked on +`native-verify-once` (PR #935). Draft PR only; never ready, never merged. + +This file is a session-handoff journal in the sense CLAUDE.md means: accurate +when written, history afterwards. Check git/GitHub for current truth. + +## State + +**Started 2026-09-17.** Orientation complete; baseline run not yet launched. + +## Charter, in one line + +Lift the two hard limits the 2026-09-16 profiler found — the 64 MiB +whole-roster receipt transport (refuses above 6.1% of the US source) and +per-node full-population retention (~300 GiB at full source) — then prove a +build above the old ceiling at 1/10. + +## Sites, line-cited in this tree (not the profiler's tree) + +| what | where, at `5ff889814` | +|---|---| +| preparation payload limit | `survey_population_preparation.py:50` `MAX_PAYLOAD_BYTES`, `:104` `_encode`, `:108` `PAYLOAD_LIMIT` | +| the whole-roster document | `survey_population_preparation.py:1698-1699` (`"selection": _plan_document(plan)`, `"origins": origins`) | +| second origin budget | `survey_population_preparation.py:816-818` `_bounded_append` → `ORIGIN_LIMIT` | +| allocation payload limit | `graph_survey_population.py:62` `ALLOCATION_MAX_BYTES`, `:344-369` `_allocation_payload` | +| per-cell frame identity | `survey_population_preparation.py:601` `_frame_cell_encode`, the `_frame_identity` loop | +| the observer that retains 19 populations | `graph_atomic_survey_financial.py` `observe(...)`, called from `executor.py` `_observer_snapshot` | + +## Done + +- Read the profiler's §5, §9, §10, §11 and the verify-once report's measurement + section. Baseline to beat: **2,010.07 CPU-s / 2,016.08 s wall / 13.31 GB peak + RSS** for the 19-node financial graph at 1/1000 (verify-once file C). +- Confirmed no live microcosm build on the machine; 128 GiB / 18 cores. +- Located the 19-node harness: + `~/PolicyEngine/_worktrees/microcosm-verify-once/.measure/harness19_verify_once.py`. + +## Next + +1. Baseline run of the 19-node harness at 1/1000 in a throwaway worktree at the + base commit, with an RSS-over-time sampler and runner-vs-node-loop split. +2. Design note `docs/us-native-scale-transport.md`. +3. Implement, with refusal tests, identity-equality proofs, replay proof. +4. 1/10 run above the old ceiling. +5. CI-shaped battery, draft PR. + +## Session log, 2026-09-17 + +### Baseline (requirement 1) — done, and it settles the profiler's open question + +One uncapped-by-node run of the 19-node financial graph at 1/1000, on the +branch point `5ff889814`, in the throwaway worktree +`~/PolicyEngine/_worktrees/microcosm-native-scale-baseline`, pid 70342 / pgid +70342 (session leader), RLIMIT_CPU hard 9,060 s, harness soft 9,000 CPU-s / +12,000 wall-s / 48 GiB, launched only after `vm_stat` showed 69.9 GB available. +Status `COMPLETED_NINETEEN_NODE`. + +| | | +|---|---| +| wall | 2,048.93 s | +| CPU | **2,028.69 s** | +| peak RSS | 13,157,728,256 B = 13.16 GB / 12.25 GiB | +| loadavg | start [6.68, 6.97, 6.40], end [10.75, 9.68, 8.66] | +| runner call | 2,045.91 s wall / 2,026.78 s CPU; 3.02 s of process wall sits outside it | +| financial node loop, 19 nodes | 277.36 s | +| prefix node loop, 9 nodes | 55.93 s | +| **runner outside both node loops** | **1,712.62 s = 83.7% of the runner call** | + +The four ASEC predictor nodes are 267.99 s of the financial loop's 277.36 s; +the other fifteen are 9.37 s. The retention slope is **not visible** in the RSS +trace at this fraction: 19 snapshots at the measured 9.6–9.8 B/cell over +1.70e6 cells is 0.31 GiB, inside the trace's own 1–2 GB sawtooth (peak 13.16 GB +at t=1303 s, 5.92 GB at exit). That corroborates the report's §9 rather than +adding to it. + +### Done since + +- Vectorised per-column frame seal in `survey_population_preparation`, proved + byte-for-byte against the predecessor traversal oracle that + `test_us_survey_frame_identity_encoding.py` already carried (160 tests; the + file's own runtime fell 7.16 s → 0.67 s). +- Segmented, content-addressed transport for all four whole-roster receipts. + `docs/us-native-scale-transport.md` has the design, the five measured ceilings + in binding order, and the two refusals that narrow. +- 1,013 line citations across seven mechanisms read and adversarially verified. + +### Next + +1. After-run at 1/1000 on this head — **queued behind a memory gate** (another + lane's `smoke.py`, pid 68076, holds 52.7 GB; the waiter polls every 60 s). +2. 1/10 run above the old ceiling. +3. Replay proof, pins, CI-shaped battery, draft PR. + +### 2026-09-17, later + +**Landed:** the segmented transport for all four whole-roster receipts, the +vectorised per-column frame seal, the hardening of the spill path, the ceiling +receipt, the seal rate receipt, the row-count ceiling census, and draft PR +[#945](https://github.com/PolicyEngine/microcosm/pull/945) against +`native-verify-once`. + +**Measured:** + +| | | +|---|---| +| ceiling, full source | single `_encode` **REFUSES** `PAYLOAD_LIMIT`; the segmented transport carries 1,099,892,722 B in 17 segments, digest equal to the stream digest | +| ceiling, 1/10 | single `_encode` **REFUSES**; transport carries 109,804,304 B in 2 segments | +| seal, blended on the frame's own dtype census | 0.5195 → 0.0677 µs/cell, **7.68×**, bytes asserted equal per column first | +| pins | **none move**: every inventory contract matches the tree, all ten stage manifests build | +| graph suite | 778 passed, 1 skipped | +| touched `us_runtime` tests | 359 passed | +| frame identity file | 187 passed (was 160), 15.62 s | + +**Zero `packages/microcosm-graph` hunks**, so nothing in this PR is main-only. + +**Not implemented, by decision and with the mechanism recorded:** the memory half +of the retention. `store.py` refuses a store-backed population snapshot three +ways (masked-storage zeroing at `:456`, `MultiIndex` at `:619`, `CategoricalDtype` +at `:486-487`), object identity is pinned at +`graph_atomic_survey_financial.py:1897`, and `same_replayed_population` needs the +whole object while the replay cannot exist until `run_graph` has returned. So +nineteen detached populations must live in RAM or on disk between observation and +comparison unless the object comparison becomes a content seal — a +verification-contract decision, put to Max as a question. +`docs/us-native-scale-transport.md` §4 specifies the executor mode item by item. + +### Next + +1. The 1/10 graph run (worktree `microcosm-native-scale-tenth`, launcher waits + for 70 GB), after the 1/1000 after-run's required replay finishes. +2. Fold the after-run and 1/10 numbers into the report and the PR body. diff --git a/PROGRESS-us-canonical-household-roles.md b/PROGRESS-us-canonical-household-roles.md new file mode 100644 index 000000000..4491117af --- /dev/null +++ b/PROGRESS-us-canonical-household-roles.md @@ -0,0 +1,57 @@ +# US canonical household-role fragment — lane journal + +Lane: `us-canonical-household-roles-20260913`, based on integration +`a96fc850942d3fd96d24251c07d9ebc38b79f8de`. Independent parallel US +source-only build. No push, no merge, no runtime staging, no test execution. + +## Scope + +Bounded canonical household-role fragment for the current Microcosm US graph: + +- A thin **qualifier** over the genuine retained ACS/ASEC preparation/issuers. +- A **common graph fragment** mapping exact original ACS/ASEC source + identities onto both initial clone families, preserving int64 ids, + household relationships, assigned geography, weights, provenance and every + other Frame cell. +- `is_household_head` bound **only** where the actual documented source + definition matches. Ambiguous source roles stay unbound and explicitly + unknown. +- Independent-minor roles: **source/consumer scope audit only** unless an + existing explicit canonical contract plus genuine source authority + establish the mapping. + +Out of scope (owned elsewhere or deferred): canonical sex, raw +disability/student observations, housing participation, native restoration, +parent-link preservation, wiring the shared country host. + +## Original lane checklist (historical) + +- [x] Worktree/branch/base verified; AGENTS.md + CLAUDE.md read. +- [ ] Trace maintained source owners + canonical `is_household_head`. +- [ ] Mapping/unknownness specification. +- [ ] Qualifier module. +- [ ] Graph fragment module. +- [ ] Pure/graph/fixture tests (written, not run). +- [ ] CI source classification. +- [ ] Docs + changelog fragment. +- [ ] BUILD-RESULT.md. + +## Original next step (historical) + +Trace owners with a parallel read sweep over the ACS/ASEC source runtime. + +## Source-only continuation, 2026-09-13 + +The external builder stopped after commit `593aa157a46d4736238b4535ed3f9d9eb6dd7a52`, containing a qualifier module without graph/tests/docs. Its commits are preserved. The continuation rechecked the clean assigned branch and fetched main: 239 commits ahead, zero behind before further edits. + +Review found the declared live fence was never called by the qualifier. The continuation checks it around every owner callback, binds individual column aliases, and independently reconstructs the final projection after owner I/O. Root also explicitly confirmed that unsupported known incumbents must refuse at canonical binding, including artificial group-quarters heads; reconciliation retains the original evidence for adjudication. + +The source continuation adds a common two-node graph fragment, pure/graph/actual-owner tests, the mapping and independent-minor audit in `docs/us-current-survey-household-roles.md`, and a changelog fragment. No country-host wiring, housing routing, independent-minor calculation, source staging, project import, pytest, native or installed-model read, push or release is performed. The packet's source pins and bounded runtime proposal record the final scope; written tests remain unrun until separately authorized. + +Source AST parsing/compilation, Ruff formatting/lint, diff whitespace and CI inventory verification pass. The inventory contains 122 planned cases (58 pure, 50 graph, 14 actual owners), not executed results. All three new files receive the existing fast/rest and engine/us-am/build classification; no CI classifier change is required. + +## Independent-review corrections, 2026-09-13 + +Fable reviewed exact `8a3226c37a2b0a3e9aee0090f901bfdca4efd549`. The original commit, review and source inventory remain preserved. The concrete graph-test receipt key is corrected from `acs_allocation_provenance` to the actual `relationship_allocation_provenance`. Five additional pure cases cover non-nullable Boolean preservation/refusal, invalid incumbent dtypes and the published current ASEC maximum roster line. Five additional real-owner cases exercise detached ASEC native/line/household mismatches and ACS household/retained-line coverage failures without modifying the issuer, original source or owner buffer. + +The source review's proposed legitimate ASEC line above 20 is not supported by the exact 2025 dictionary: A_LINENO is printed 01–16. No production bound is widened. The docs explicitly distinguish the operative 64 MiB serialized-artifact cap, the qualifier row ceiling and the ASEC owner's 600,000-row cap; no national capacity is asserted. The revised planned roster is 132 cases: 63 pure, 50 graph and 19 actual-owner cases (A113/B19). All remain unrun. The two production modules are unchanged. diff --git a/PROGRESS.md b/PROGRESS.md index 8be780caf..26471a821 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -365,6 +365,148 @@ one tolerance so none can report a bundle differently from the others. - Whole-workspace run and PR CI to finish; hand to human review. Do not merge; do not publish. +# Grouped solver x calibration target snapshots integration - 2026-09-12 + +Historical note, 12 September 2026: this grouped-lane journal was subsequently +integrated with PR #914's corrected iteration identities and the actual fiscal +host observer. Its 443-test evidence and statements that host wiring is absent +describe that earlier lane. Current source and acceptance are recorded in +[the fiscal-host evidence](experiments/fiscal-target-snapshot-host-20260912.md). + +Lane: `microcosm-grouped-target-snapshot-integration-20260912`, branch +`grouped-target-snapshot-integration-20260912`. Base: fresh `origin/main` +`116d46ee9dc2aafdc68259b7c06e4c3462522e8b` (unmoved; it is exactly the shared +base both reviewed heads were cut from). Integrates the exact reviewed heads +`536f1ceefcdafda3cc619c14b4da18e012a7be57` (G, US grouped/fixed-zero Adam) and +`b43369dc49e175803f62020cc1e72fa53926aed8` (S, calibration target snapshots, +PR #914) against the read-only checklist +`grouped-snapshot-integration-review.md`. + +Everything below this section is prior-lane history and was accurate when +written; see "Root journals are history, not state" in `CLAUDE.md`. + +## State + +Complete and local. Four commits: the merge, the grouped instrumentation, the +identity recomputation, and the cross-product controls. Nothing pushed, no PR, +no release action. Root reviews and integrates. + +Scope is the shared solver seam only. `graph_fiscal_dense_calibration.py` still +calls `calibrate` without an observer, so this branch emits no snapshot for the +real US fiscal path; that host wiring is a separate step, and no observer +registry, `Node.params` callback or replay-reruns-the-optimizer claim was +invented here. See +[the lane experiment](experiments/grouped-target-snapshot-integration-20260912.md) +for scope, evidence and residual risks. + +## Done + +- Fetched `origin/main`; confirmed it is still `116d46ee9`, so no main drift + had to be preserved. Recorded as the integration base. +- Created this worktree on a new branch from that base; fast-forwarded to G, + then merged S. Verified all four pinned source hashes + (`solve.py` and `calibrate/__init__.py` on both heads) match + `grouped-snapshot-integration-review-pins.json` byte for byte before merging. +- `calibrate/__init__.py` auto-merged as a true union: `GroupedUpperBounds` + export retained alongside every snapshot export. +- `solve.py`: the single textual conflict was the `calibrate()` signature; + resolved as a union of S's `target_snapshots` and G's + `grouped_upper_bounds` / `grouped_preserve_zeros` / + `_post_projection_observer`. +- `test_us_multispine_pool_tool.py`: resolved semantically. G's unrelated US + integration delta (schema_version 2 PUMA-ladder fixture with joint + PUMA/tract/CD overlap arrays and per-layer `source` labels) merged cleanly + and is retained; the only textual conflict was the `spec_sha256` pin. +- The five source-derived identity conflicts + (`inventory_coverage.py` EXPECTED_HASHES, `us-f0-coverage.json`, + `test_spec_engine_loader.py` golden, the multispine `spec_sha256`, and the + calibrate parity `pins.json`) carry a placeholder in this merge commit. None + of them is an ours/theirs decision: both sides' values describe their own + tree, and neither describes the merged one. They are recomputed against the + final merged sources in a later commit on this branch. + +## Done (continued) + +- Instrumented `_optimize_grouped`: the grouped early return in `_optimize` + bypassed every hook S added, so grouped runs emitted only a closing snapshot. + The in-loop emission sits after the progress callback and before `backward()` + and reads the exact float32 estimate tensor the epoch's loss was computed + from. Returned weights, trajectory and RNG state are bit-identical with the + observer on and off, and no evaluation is added. +- Separated retain-best detection from the receipt for grouped runs, so a later + change that populated a grouped receipt cannot make the reused final emitter + claim a retained best. The receipt stays empty. +- Recomputed six source-derived identity artifacts against this checkout (the + five conflicted ones plus the country-bundle digests, which were not + conflicted because only G had touched them but which move for the same + reason). No value equals either branch's. The simulate and fit.qrf pins were + deliberately left alone. +- Added the checklist's cross product to the three existing grouped test files + rather than a new one, so every case reuses fixtures already there. + +## Next + +Root's independent review and integration. Open follow-ups, none owned here: +the host observer seam for the US fiscal dense calibration path, a country-scale +cadence choice backed by a real measurement, and the dashboard consumer. + +--- + +# #893 reconciliation to main's amended graph interface (amendments 19 and 20) + +Lane: `microcosm-us-launch-verified-lanes-20260910` (the live integration +worktree for PR #893, branch `microcosm-us-launch-integration-20260909`). +Started 2026-09-12 at `069d5ed9a`. Everything below the `---` rule at the end +of this section is prior-lane history; see "Root journals are history, not +state" in `CLAUDE.md`. + +## State + +Merges done, four graph pieces re-applied, graph package green (518 passed), +fit green (191), spec check / groups verify / ruff clean. Build-side consumer +suites running at the time of this entry; final results in `out.md` §5 and +`experiments/893-reconciliation-amendments-19-20-20260912.md`. Nothing +pushed, no PR, no new branch, `uv.lock` untouched. + +## Done + +- Read `CLAUDE.md`, the charter's "Interface freeze" at `23ba24770`, PR + #893's body, the Amendment 19 lane's "Scope" list; diffed the graph package + against `23ba24770` per file; AST-scanned every non-graph consumer. +- `uv sync --all-packages --locked --extra us --extra uk` exit 0. Baseline + graph suite: 5 failed / 362 passed (the five the brief names). +- `3010b7788` merge `origin/main`: graph package, fit sources, lock and + charter resolved to main's bytes; branch-only `attachments.py`, + `availability.py`, `schema.py` and the two branch-only graph tests removed + in the merge, to return only where consumed. +- `051357909` merge `23ba24770` (amendment 20, merged from the branch head + because #912 was still on CI; the dispatcher re-runs `git merge origin/main` + after it lands — expected no-op for graph/fit/lock/charter). +- `dc621c14c` seed digests re-pinned (the branch's `acs_transfer` and + `housing_inputs` plus amendment 20's `fit.qrf` move them); coverage report + regenerated; `--check` exit 0. +- `cff8fbf32` calibrate/simulate H1 pins re-recorded (the branch's solver and + `Frame.__reduce__` changes move them); parity files 27 passed. +- Pieces re-applied on main's files, each with graph-level tests the branch + never had: `752ab840f` raw-byte codec (13 US consumers; 8 codec tests), + `db1b7821a` Frame-metadata store (3 named consumers; 274 passed across the + store/population/executor/manifest files), `d1019762b` execution states + (the post-clone geography gate's typed artifact; replaces the amendment-19 + gate refusal and its test; whole graph package 518 passed), `a16eeacf8` + population observer (4 consumers; 94 passed on the executor + B/F files). +- Dropped for lack of a consumer: lazy retention (`attachments.py`, layer + 08), `schema.py`, `keys._stream_file`, the `_write_node` refactor. +- Layer map computed from `git log --name-only` attribution plus an AST + import scan with hard / name-hard / soft link classes; written to `out.md`. + +## Next + +- Record the build-side consumer results in `out.md` §5, commit the + `experiments/` copy of the report, leave `out.md` uncommitted (it is + another lane's tracked report; see the memory note). +- For Max: piece C supersedes amendment 19's gate refusal (`out.md` §8.1); + the dropped lazy retention / `_stream_file` / `_write_node` pieces (§8.2–3). + --- # Amendment 19 — typed opaque artifacts on the graph interface @@ -1329,3 +1471,354 @@ The still-earlier PolicyEngine-US 1.819.0 lock-bump lane merged into `origin/main` at `7b90bb18` on 2026-08-24; its final state remains at commit `05d254aa` and its detailed receipts remain in the historical section of `_LANE-NOTES.md`. + +## US launch integration staging — 2026-09-09 + +### State +Source-only staging in progress. Execution and source/data admission remain root-owned. + +### Done +Verified requested clean branch, base HEAD, main ancestry and preservation pins. + +### Next +Apply thirteen explicit source layers, commit each, then separately review isolated ordinary execution. Existing journal history above is retained. + +Layer 1: SAFE-ADDITIVE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 2: GRAPH-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 3: ACCEPTED-SHARED-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 4: PUF-SUPPORT-MERGE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 5: SOLVE-MERGE-PROPOSAL.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 6: J-GRAPH-COMPATIBILITY.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 7: F-CATALOGUE-OPTIMIZATION.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 8: GRAPH-ATTACHMENT-METADATA.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 9: F-JOINT-GEOGRAPHY-GATE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 10: SOURCE-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 11: PLACEMENT-ADDITIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 12: ORDINARY-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 13: INTEGRATION-REGRESSIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +### State +All thirteen source layers staged and committed; behavioral qualification pending. + +### Done +Per-layer pins and actual commit messages checked; store/current-main preservation retained. + +### Next +Root reviews exact ordinary guard/source/resource admissions before execution. Full65 findings and survey/SCF lanes remain separately owned. No remote action is authorized. + +### Integration source-resource closure — 2026-09-09 + +Root preflight found seven JSON source definitions declared by the ordinary guard but omitted by the staged patch delivery. Added the exact previously reviewed resource bytes from the accepted full65 source projection; no new resource admission or genuine payload. Preserved source staging and earlier evidence. + +## Layer14 full65 replay correction — 2026-09-09 + +State: exact accepted two-file correction staged; successor57 integration execution pending. +Done: verified clean84243 preimages, exact r2 postimages, fixed store/current-main sources and frozen36 evidence. +Next: root admits the separate exact-allowlist57-case guard and final source identities before execution; no new resources. + +## Source review publication — 2026-09-09 + +The user explicitly authorized pushing the current source work and creating PRs for Anthony to review. This supersedes earlier source-only local restrictions for source publication; it does not authorize a data release, merge or deployment. + +Integration controls passed 36 cases; the exact replay correction subsequently passed all 57 cases at dfa7f872cd3eba3c42adf5758cde8b3ca38f3d17. Source/control, model-declaration and resource hashes matched externally after both runs. A final formatting/import cleanup and explicit test-observer loop binding are included for CI; no release result is claimed. See docs/us-launch-review.md for current scope, evidence and related PRs. + + +# US completion host integration — 2026-09-13 + +State: source implementation and bounded tests authored for root review; all runtime checks and tests UNRUN. +The task-specific no-commit instruction overrides the standing commit order. +Base: 6aa745c8762c3e4a140b8168470de5827e12c5cb. + +Done: inspected status, branch, HEAD, AGENTS.md, CLAUDE.md, frozen R2 plan and +independent source-plan review. Concurrent completion/tax seam files are excluded +from this work's ownership. + +Done: added private completion custody, retained issued property parent, registry +copies, actual union artifact/state checks, role/child reconstruction and separate +tax gate wiring. Authored 10 test functions / 20 source-expanded cases and docs. +AST, direct Ruff lint/format, and tracked whitespace checks pass. + +Next: root review and reconciliation, inventories/test-spine updates, bounded +resource/guard proposal, and actual fixture/cold/required/adversarial validation. +45/49 counts remain source-derived; combined 51 and fixture assumptions are UNRUN. +Final report: FINAL_REPORT.md. No acceptance or publication claim. + +## Bounded completion-host source corrections — 2026-09-13 + +State: bounded source corrections complete; all runtime checks remain UNRUN. +This correction pass used the 20-minute source-only window beginning 22:09 UTC. +The later explicit no-commit/no-staging instruction governs this correction pass. +Ownership is limited to graph_survey_completion_host.py, +test_us_graph_atomic_completion_host.py, docs/us-survey-completion-host.md, +and this exact PROGRESS.md / FINAL_REPORT.md update. + +Done: read the full independent source review and full approved R2 plan from +the supplied recovered absolute paths. The R2 SHA-256 matches +b402d6be9d374774e27da7bf42fb83a216305d4cddd2ad8347c8149536269a29. +Confirmed intentional dirty worktree and HEAD +6aa745c8762c3e4a140b8168470de5827e12c5cb. Earlier journal content is preserved. + +Done: exact receiving1/roles0-or-2/child6/tax3 ordered rosters are bound before +compilation; child document count and knownness derive from retained nodes and +reconstructed/materialized-verified evidence. Authored separate 49/45 cold+required +acceptance, equal-values/replaced-Population refusal, real observed base-hit/union +and final-I/O identity controls, actual tax-rebased parent refusal, exact child +adversary reasons, full-spine writer ordering, both FILTER conservation ledgers, +dimensions and owner/version transitions. Source proves table copying; immediate +base preservation assertions remain alongside permanent revocation. Fixture-only +composition wrappers now restore before issuance to avoid cross-fixture chaining. + +Done: AST parsing, direct Ruff lint/format and git diff --check pass. AST reports +16 test functions / 32 source-expanded cases. All tests, collection and project +imports remain UNRUN. Only the five explicitly owned files were edited; no +commits, staging, agents, dependency/network/browser access or payload execution. + +Next: root reviews exact source/hashes and budgets separately selected execution. +T4 financial-only-parent coverage, remaining T5 initial/final/reconciliation +adversaries, T6 actual fixture validation and T7 GQ/zero-weight scientific fixtures +remain open with concrete reasons in FINAL_REPORT.md. The independently confirmed +old status/tax42 owner repair, inventories, CI/source closure, scientific/native +capacity and release/default/PR893 decisions remain separately owned. + +--- + +# Lane: verify native sources once per run (`native-verify-once`) + +Branch `native-verify-once`, worktree `~/PolicyEngine/_worktrees/microcosm-verify-once`, +base `f7bb88525a78786f91bc3ebe2083ef4b1c85de18` (PR #893 head). Started +2026-09-15. + +**State (2026-09-16, third session):** all five mechanisms landed with proofs +and the epoch wired into both atomic capsules; draft PR #935 open; the +CI-shaped test battery is green (no test group failed; `ruff format --check .` +is red on 77 pre-existing files that this branch does not touch, and CI does +not run it); the before/after probe and the nineteen-node harness have run; the +lane report `experiments/native-verify-once/out.md` is complete, with the +measurement, the pins, the verbatim summaries and the open questions. The open +work is the main-only split (its commits sit on +`graph-verify-once-main-stale-20260916-0057`, unpushed) and Max's answers to the +report's open questions. +*(Historical as of 2026-09-17: the split is now draft PR #938, and an +adversarial verification of this branch has since been resolved by ten further +commits on a rebased branch — see "Done (2026-09-17, fourth session)" below.)* + +**Goal.** The 9/15 pilot v5 measurement +(`~/PolicyEngine/_recovered/pilot-runs/native45-v5/out.md` §2) attributes ~79 % +of a native 1/1000 run to repeated admission and verification of 3.48 GiB of +staged source: node execution is 1,143 s of a 5,362 s run. Make source +authentication and identity verification happen once per run instead of once +per accessor use and once per executed node, without weakening any refusal and +without moving any digest value. + +**Done:** read the evidence base; located all five mechanisms at base HEAD. + +**Done (2026-09-16):** + +- `docs/us-native-verification-once.md` — the design note, written before any + capsule changed. +- **Mechanism 5**: `_object_stream` builds each plain float/int/bool column's + context-digest bytes with numpy. Byte-identical against a verbatim copy of the + pre-change body over every column kind, the float specials, non-canonical NaN + payloads, both int64 endpoints and a 200-frame random sweep. 0.47 s -> 0.09 s + on a 6,928 x 240 frame. +- **Mechanism 4**: the ACS record fence now uses `bytes.find`/`bytes.count` with + cached terminator cursors. Proven against the byte loop on 46,655 exhaustive + short strings across five ceiling settings, 4,500 random strings, and the real + staged `csv_pus.zip`: 3,422,890 records, identical digests, 226.9 s -> 4.6 s + (49x). Re-pins `_ACCEPTED["acs_person_coverage_authentication.py"]`. +- **Mechanism 3**: `run_graph` carries a `_SourceIdentities` cache keyed on stat + signatures, and re-derives every source in full before building the manifest. + `RunManifest.source_identities` records it without moving any existing value. + The refusal it protects had no test at all; `test_graph_executor_source_identity` + now pins it. Graph suite 108 s -> 72 s. +- **Mechanisms 1 and 2**: `survey_population_preparation.verification_epoch()`, + an opt-in scoped memo. Cheap tier every borrow (live authority, attached + payloads, producer encoding); expensive tier skipped only while a signature + over every path it reads — the roster stat identities among them — and every + live buffer it digests is unchanged; unconditional full re-validation on + leaving the epoch. The two moved inventory contracts are re-derived (below). + (The roster stat identities were compared in the cheap tier when this line + was first written; the fourth session moved them into the signature so a + memo miss raises the code an unmemoised borrow raises. See below.) + +**Done (2026-09-16, second session):** + +- Draft PR [#935](https://github.com/PolicyEngine/microcosm/pull/935), base + `microcosm-us-launch-integration-20260909`. +- Main-only split branch `graph-verify-once-main`, worktree + `~/PolicyEngine/_worktrees/microcosm-graph-verify-once-main`, branched from + `origin/main` (51c314382). It carries the graph-shard change only: the three + commits re-applied, plus a changelog fragment. Its one deliberate difference + from this branch is that `_update_scalar` has no exact-float `struct.pack` + shortcut on main -- that belongs to PR #893 -- so the parity test's reference + copy was aligned to main's actual pre-change body, which makes it a literal + verbatim copy there too. +- Before/after measurement staging in `.measure/` (gitignored): a parameterised + copy of the v5 pilot probe that takes the source tree, the staged run inputs + and the output directory from the environment, asserts every imported + `microcosm` module resolves inside the measured tree, and enforces a 16 GiB + resident ceiling alongside the CPU one. Baseline worktree + `~/PolicyEngine/_worktrees/microcosm-verify-once-baseline` is detached at + `f7bb88525`. + +**Done (2026-09-17, fourth session) — the verification findings:** an +adversarial static verification of the branch returned four medium and seven low +findings and no high one +(`~/PolicyEngine/_recovered/scratch-backup/893/lanes/verify-once-findings-20260916.json`). +Ten commits answer them, on a branch rebased onto the base tip `363a9033b`: + +- rebased onto `363a9033b` and force-pushed, so two-dot and three-dot diffs + agree again (26 files, +5,422 / −819); +- a mutation test per memoised capsule that changes file **content** while the + cheap tier's view of it stays byte-identical, so the memo-miss branch itself + refuses rather than a stat comparison in front of it; +- the roster stat identities moved out of the refusing cheap tier into the + signature, so an in-epoch refusal carries the code an unmemoised borrow + carries (behaviour, not documentation: the design note promised it and the + fix is smaller than the divergence, and it deletes a redundant `_file_stats` + walk per borrow); +- `verification_epoch()`'s record is bound by both runners and attached to the + manifest as `RunManifest.verification_epoch` — outside the key, the JSON, + every node receipt and every cache record. A real nine-node run over invented + sources reports `{"capsules": 1, "hits": 20, "misses": 3, + "final_validations": 1}`; +- `_source_stat_signature` follows a member symlink, as `_directory_identity` + does, so a target rewritten mid-run refuses at the next node rather than at + run end; `_object_stream` packs floats in native order, as the loop it + replaces does; +- the dead `_stat_or_absent` is deleted; the four `_path_stat` copies and the + two `_stat_identity` definitions stay, with the pin cost of consolidating + them re-derived through the modules' own generator and stated in the report; +- the design note's five inaccuracies are corrected, the nested finalizer + chains its refusal instead of dropping it, and the report's stale items are + historicized. + +Re-run afterwards, all rc 0: `769 passed, 1 skipped in 44.56s` +(`packages/microcosm-graph/tests`), `128 passed in 188.39s` (the four new test +files), `1811 passed, 1 skipped, 38 warnings in 11200.54s (3:06:40)` (the 46 +build test files that reach a changed module), plus `ruff check` clean, +`17 files already formatted`, `uv lock --check`, both `ci_test_groups` checks. + +**Done (2026-09-17, second fix pass) — the two residuals:** a re-verification of +the ten commits above left two low findings, and two commits answer them. (1) +Only the nine-node runner had a test that read `RunManifest.verification_epoch` +back off a real run, so +`test_nineteen_node_financial_cold_and_required_replay` now reads it off both +manifests the financial runner returns — its own epoch's and the nested +population epoch's — for the cold run and the required replay, asserting the +protocol label, `final_validations == capsules >= 1`, and that `to_json` still +cannot see it; and the committed probe +(`experiments/native-verify-once/probe_verify_once.py`) now copies the record +into its output as `verification_epoch`, so the next measured run is the first +that will carry counts. Nothing was re-measured, and the three measurement files +the report quotes predate that line, which the report, the README and the design +note all state. (2) This journal's "Next" and the report's open question 4 +restated #938 at its current head and size, with the mirror recorded as landed. +No `packages/*/src` file changed in either commit. + +Re-run, both rc 0: `7 passed in 119.98s (0:01:59)` +(`test_us_graph_atomic_survey_financial.py`) and +`24 passed in 157.61s (0:02:37)` (`test_us_native_verify_once_epoch.py`), plus +`ruff check` and `ruff format --check` clean on the two changed Python files. + +**Done (2026-09-17, third fix pass) — one medium and two low:** a second +re-verification found that the 2026-09-16 refusal-code fix had weakened one +at-borrow refusal, and three commits answer it and the two low findings. +(1) `_finalize_epoch` recorded its memo signature *after* validating, so at an +inner nested close — whose memo survives into the outer epoch — a roster file +whose stat moved during the trailing part of `_validate` was absorbed as the +new normal and the outer borrows were hits until the outermost close. Each +close now takes the signature before validating and again after and records +none when they differ, so the next borrow is a miss that refuses; +`asec_2024_native_population._epoch_exit` had the same ordering and is fixed +the same way. Two new tests move a source from a profile hook as that close's +own validation returns and fail with `DID NOT RAISE` against the previous +ordering. `_memoized_validate`'s signature handler also narrowed from +`BaseException` to `Exception`, which is what its own comment describes. +(2) `RunManifest.verification_epoch` and `run_graph(_verification_epoch=)` had +no graph-shard test on either branch: `test_graph_verification_epoch.py` adds +nine, and two build-shard tests cover the signature-cannot-be-taken handler and +the `raise own from nested` branch. (3) The stale cheap-tier descriptions the +2026-09-16 fix left behind — test-file docstrings, the module block comment, the +design note and this report — now say what the code does, and `executor.py:2100` +is corrected to `:2105`. Two `packages/*/src` files changed and no pin moved: +`graph_implementation._dependency_contract` was re-run over both edited modules +and `imports`, `resource_accesses_sha256` and `unbound_uses_sha256` are all +unchanged. + +Re-run, all rc 0: `778 passed, 1 skipped in 37.72s` +(`packages/microcosm-graph/tests`, up from 769 — the new +`test_graph_verification_epoch.py` adds nine), `28 passed in 190.65s (0:03:10)` +(`test_us_native_verify_once_epoch.py`, up from 24), +`1607 passed, 38 warnings in 8311.33s (2:18:31)` (the 34 build test files naming +either changed module), `8 passed in 0.68s` +(`test_us_asec_prepared_resources.py`, which builds the stage implementation +manifests), and `597 passed, 2 skipped in 17.75s` in the +`graph-verify-once-main` worktree. Plus `ruff check .`, `ruff format --check` on +all five changed Python files, `uv lock --check`, both `ci_test_groups` checks. + +**Next:** Max's rulings on the report's remaining open questions. Open question +4 is answered and needs nothing further: the split is draft PR +[#938](https://github.com/PolicyEngine/microcosm/pull/938), opened 2026-09-16 +and now at head `33f3150bb` (base `main`, 7 files +1,430/−8, MERGEABLE, still +draft; `gh pr view 938`, 2026-09-17 07:43 UTC). The graph-shard commits made +after the verification findings were mirrored into it on 2026-09-17 at +06:45–06:46 UTC — the verification epoch on the manifest, the member-symlink +follow, the native-order float cast and the docstring fix — and the third fix +pass's new `test_graph_verification_epoch.py` followed as `33f3150bb`. Diffed +the same morning, its five earlier graph files differ from this branch's only by +the documented `import struct` and exact-`float` hunks that belong to #893's +base. +#935's body already carries the measurement table. Measured: nine-node prefix +before 1,803.87 CPU s without completing (ceiling) against after 1,444.78 CPU s +completing; nineteen-node 5,278.61 -> 2,010.07 CPU s (2.63x) against the v4 cold +receipt, peak RSS 9.31 -> 13.31 GB. + +**Pins re-derived so far:** + +| pin | old | new | +|---|---|---| +| `acs_native_coverage_binding._ACCEPTED["acs_person_coverage_authentication.py"]` | `475aa795…fe85bcff` | `9ec68721…d88e8e49f` | +| `graph_implementation_inventory.json` `survey_population_preparation.py` `unbound_uses_sha256` | `29c09f6f…d296ef91` | `d114117c…dd4005910` | +| `graph_implementation_inventory.json` `asec_2024_native_population.py` `unbound_uses_sha256` | `71463df4…d7f287608` | `7bb20439…40d58cdf4` | + +**Lane notes.** Root `out.md` is a tracked file holding the Amendment 19 lane's +committed report; this lane's report goes to +`experiments/native-verify-once/out.md`. The v5 cold run (`run_v5.py`, pid +81194) is live on this machine, so probe runs wait for the window the lane +brief allows (`pgrep -f run_v5.py` empty, or > 40 GB free). +## Stack reconcile after #938 — 2026-09-18 + +Lane: bring `microcosm-us-launch-integration-20260909` (#893) level with +`origin/main` 8c44daa52 (#938 verify-once merged 2026-09-17), then carry the +merge down the stacked branches `native-verify-once` (#935), +`native-scale-transport` (#945), `native-row-ceilings` (#949) and +`native-retention-seal` (#950). Merges only, never rebases; pins regenerate +through their generators. Report: `experiments/stack-reconcile-20260918/out.md`. + +State: worktree `_worktrees/microcosm-stack-reconcile`, local branches +`reconcile/`, pushed with `git push origin HEAD:`. +Starting heads: main 8c44daa52; integration 8a7e1b12d; native-verify-once +0d04abfcf; native-scale-transport 6fca96f31 (one commit past the brief's +886f777eb); native-row-ceilings c5ac78d2d; native-retention-seal db7c93871. + +Done: survey. Main's delta since the d1196af10 merge base is #938 alone: +nine files, all under `packages/microcosm-graph/` plus one changelog fragment; +uv.lock and every pyproject are unchanged, so the approved-lock digest does +not move. `git merge-tree` predicts one textual conflict, an import-line clash +in `graph/executor.py` (`import struct` on the branch, `import stat` on main). +The four downstream merges preview conflict-free at today's heads. + +Next: step 1 merge, checks, push; then steps 2–4 in order; then the report. diff --git a/changelog.d/20260906-frame-metadata-storage.fixed.md b/changelog.d/20260906-frame-metadata-storage.fixed.md new file mode 100644 index 000000000..dfa6dd022 --- /dev/null +++ b/changelog.d/20260906-frame-metadata-storage.fixed.md @@ -0,0 +1 @@ +Preserve complete Frame metadata in graph content-store format v2, including nested source evidence, and refuse older frame caches that cannot restore it or reuse of one frame key with different metadata. diff --git a/changelog.d/413-snap-final-export-checks.added.md b/changelog.d/413-snap-final-export-checks.added.md new file mode 100644 index 000000000..7b582048c --- /dev/null +++ b/changelog.d/413-snap-final-export-checks.added.md @@ -0,0 +1 @@ +Add scoped SNAP checks that reopen an identified final export, recompute 51-state caseload and benefit estimates, verify county/state consistency, and preserve pending status when runtime evidence or reviewed tolerances are missing. diff --git a/changelog.d/893-age-count-fixture.internal.md b/changelog.d/893-age-count-fixture.internal.md new file mode 100644 index 000000000..1e691423f --- /dev/null +++ b/changelog.d/893-age-count-fixture.internal.md @@ -0,0 +1 @@ +Use the maintained Frame store API in the national age-count cold and fresh-process replay tests after removal of the transfer example module. diff --git a/changelog.d/893-asec-prepared-resources.fixed.md b/changelog.d/893-asec-prepared-resources.fixed.md new file mode 100644 index 000000000..95dc74d99 --- /dev/null +++ b/changelog.d/893-asec-prepared-resources.fixed.md @@ -0,0 +1 @@ +Restore three pinned ASEC resource files omitted during integration so prepared and composed source stages can compute their implementation manifests. Preserve the existing empty engine-default allowlist, source declarations, and reported-income contract. diff --git a/changelog.d/893-ci-reshard-durations.fixed.md b/changelog.d/893-ci-reshard-durations.fixed.md new file mode 100644 index 000000000..5f5581ea7 --- /dev/null +++ b/changelog.d/893-ci-reshard-durations.fixed.md @@ -0,0 +1 @@ +CI: rest lane split six ways and wheels eight ways, each sharded pytest run reporting its 25 slowest tests. diff --git a/changelog.d/893-diagnostic-current-lock.fixed.md b/changelog.d/893-diagnostic-current-lock.fixed.md new file mode 100644 index 000000000..cff257d2e --- /dev/null +++ b/changelog.d/893-diagnostic-current-lock.fixed.md @@ -0,0 +1 @@ +Align the source/seed diagnostic with the lockfile already approved for the primary QRF worker after the UK openpyxl dependency declaration. diff --git a/changelog.d/893-diagnostic-refusal-context.internal.md b/changelog.d/893-diagnostic-refusal-context.internal.md new file mode 100644 index 000000000..c6e330a2f --- /dev/null +++ b/changelog.d/893-diagnostic-refusal-context.internal.md @@ -0,0 +1 @@ +Record bounded code-path context for refused CI spec diagnostics while preserving the existing read restrictions and incomplete result status. diff --git a/changelog.d/893-diagnostic-source-io-lock.fixed.md b/changelog.d/893-diagnostic-source-io-lock.fixed.md new file mode 100644 index 000000000..f192b5b45 --- /dev/null +++ b/changelog.d/893-diagnostic-source-io-lock.fixed.md @@ -0,0 +1 @@ +Update the source identity diagnostic's exact lockfile pin after adding the engine-free source reader extra, and verify that the checked-in lock passes while an incorrect pin still refuses. diff --git a/changelog.d/893-fiscal-measurement-spm-selection.fixed.md b/changelog.d/893-fiscal-measurement-spm-selection.fixed.md new file mode 100644 index 000000000..9e05e6336 --- /dev/null +++ b/changelog.d/893-fiscal-measurement-spm-selection.fixed.md @@ -0,0 +1 @@ +The declared fiscal measurement kernel now binds the release SPM selection into its model contract and passes it to the engine, and a model output that reaches an SPM measurement variable must declare the household county, the unit's tenure and each member's age as leaves, which the static dependency closure does not see. diff --git a/changelog.d/893-historical-zero-provenance.internal.md b/changelog.d/893-historical-zero-provenance.internal.md new file mode 100644 index 000000000..5c7d83751 --- /dev/null +++ b/changelog.d/893-historical-zero-provenance.internal.md @@ -0,0 +1 @@ +Keep the exact historical ASEC zero-origin citations in the dependency scan while continuing to reject other incumbent data-package references in that module. diff --git a/changelog.d/893-merged-uk-identity.fixed.md b/changelog.d/893-merged-uk-identity.fixed.md new file mode 100644 index 000000000..b6a1ef320 --- /dev/null +++ b/changelog.d/893-merged-uk-identity.fixed.md @@ -0,0 +1 @@ +Refresh the UK country-spec identity from the merged source and resource set after incorporating the UK monthly-target correction. diff --git a/changelog.d/893-native-shared-fiscal-solve.changed.md b/changelog.d/893-native-shared-fiscal-solve.changed.md new file mode 100644 index 000000000..8ae6af2a5 --- /dev/null +++ b/changelog.d/893-native-shared-fiscal-solve.changed.md @@ -0,0 +1 @@ +Extract the US release builder's existing dense and L0 calibration calls into a shared helper for prepared support. The legacy entry uses the same helper; exact-k selection, solve settings, loss reporting, and release gates retain their existing behavior. diff --git a/changelog.d/893-native-shared-fiscal-targets.changed.md b/changelog.d/893-native-shared-fiscal-targets.changed.md new file mode 100644 index 000000000..70e074528 --- /dev/null +++ b/changelog.d/893-native-shared-fiscal-targets.changed.md @@ -0,0 +1 @@ +Reuse the maintained US ledger target compilation, reviewed Medicaid substitutions and target-parity checks through a shared helper. Source-specific SSI and target-profile checks retain their legacy order. diff --git a/changelog.d/893-population-observer-isolation.fixed.md b/changelog.d/893-population-observer-isolation.fixed.md new file mode 100644 index 000000000..710af6485 --- /dev/null +++ b/changelog.d/893-population-observer-isolation.fixed.md @@ -0,0 +1 @@ +Give graph population observers detached snapshots so callback mutations and retained references cannot change subsequent calculations or cached populations. Preserve the complete frame, provenance, weights and mass accounting on both cold runs and cache hits. diff --git a/changelog.d/893-population-only-block-source.added.md b/changelog.d/893-population-only-block-source.added.md new file mode 100644 index 000000000..09da0ddba --- /dev/null +++ b/changelog.d/893-population-only-block-source.added.md @@ -0,0 +1 @@ +Normalize pinned population-only Census API responses into atomic block support. Reconcile every positive block against independent state totals and CD/PUMA mappings, retain exact source bytes and request descriptions, and keep publisher qualification separate from byte integrity. diff --git a/changelog.d/893-puf-original-application.added.md b/changelog.d/893-puf-original-application.added.md new file mode 100644 index 000000000..1b7f769c6 --- /dev/null +++ b/changelog.d/893-puf-original-application.added.md @@ -0,0 +1 @@ +Add explicit original-arm PUF application declarations and a strict detached conditioning merger that retain raw draws, qualified fixed values, and separate fit/application seed provenance. diff --git a/changelog.d/893-refusal-expectations.fixed.md b/changelog.d/893-refusal-expectations.fixed.md new file mode 100644 index 000000000..fb7fe18b8 --- /dev/null +++ b/changelog.d/893-refusal-expectations.fixed.md @@ -0,0 +1 @@ +Recognize the earlier graph-node population check in mutation regression tests while retaining the required refusal and mutation-restoration assertions. diff --git a/changelog.d/893-source-demographic-conditioning.added.md b/changelog.d/893-source-demographic-conditioning.added.md new file mode 100644 index 000000000..88bc7fb66 --- /dev/null +++ b/changelog.d/893-source-demographic-conditioning.added.md @@ -0,0 +1 @@ +Add optional source-qualified sex and state predictors to survey financial imputation. Preserve observed-year distinctions and default behavior, refuse unresolved source demographics, and verify complete population preservation through fitting, attachment and required replay. diff --git a/changelog.d/893-spec-identity-diagnostics.internal.md b/changelog.d/893-spec-identity-diagnostics.internal.md new file mode 100644 index 000000000..3eb49f5a0 --- /dev/null +++ b/changelog.d/893-spec-identity-diagnostics.internal.md @@ -0,0 +1 @@ +Capture bounded spec and seed identity evidence after shared CI tests so changed source fingerprints can be reviewed without weakening coverage assertions. diff --git a/changelog.d/893-survey-atomic-prefix.added.md b/changelog.d/893-survey-atomic-prefix.added.md new file mode 100644 index 000000000..30658e355 --- /dev/null +++ b/changelog.d/893-survey-atomic-prefix.added.md @@ -0,0 +1 @@ +Connect qualified survey geography to shared Census-block assignment before cloning and age calibration. Preserve raw sampling allocation separately, reconstruct geography in calibration budgets, and verify complete geography inheritance through cold execution and required replay. Add bounded source-byte normalization controls and record the remaining native-source and release checks. diff --git a/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md b/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md new file mode 100644 index 000000000..9614ce4df --- /dev/null +++ b/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md @@ -0,0 +1 @@ +Add supplied-source UK atomic-area assignment keyed to stable post-clone household identity, and load UK runtime exports only when requested. diff --git a/changelog.d/893-worker-resource-trace.fixed.md b/changelog.d/893-worker-resource-trace.fixed.md new file mode 100644 index 000000000..a48093a90 --- /dev/null +++ b/changelog.d/893-worker-resource-trace.fixed.md @@ -0,0 +1 @@ +Update the worker resource identity regression to follow the actual lazy import closure, and verify that opened JSON resources affect identity while unlisted resources do not. diff --git a/changelog.d/908-calibration-target-snapshots.added.md b/changelog.d/908-calibration-target-snapshots.added.md new file mode 100644 index 000000000..127a2b385 --- /dev/null +++ b/changelog.d/908-calibration-target-snapshots.added.md @@ -0,0 +1 @@ +Shared microcosm-calibrate can emit aggregate-only per-target estimate snapshots from real solver iterations at a configurable bounded or every-epoch cadence, with a versioned schema, stable ordered target identity digest, honest current/best-retained/selected iterate labelling, and an atomic latest-snapshot plus immutable bounded history store (opt-in; off by default). diff --git a/changelog.d/908-target-snapshot-review-fixes.fixed.md b/changelog.d/908-target-snapshot-review-fixes.fixed.md new file mode 100644 index 000000000..373ab23d0 --- /dev/null +++ b/changelog.d/908-target-snapshot-review-fixes.fixed.md @@ -0,0 +1 @@ +Close the four #908 target-snapshot review findings: a closed typed aggregate metadata contract, detached snapshot delivery, atomic history publication, and strict public codec validation. diff --git a/changelog.d/914-snapshot-iterate-identity.fixed.md b/changelog.d/914-snapshot-iterate-identity.fixed.md new file mode 100644 index 000000000..2563244d6 --- /dev/null +++ b/changelog.d/914-snapshot-iterate-identity.fixed.md @@ -0,0 +1 @@ +Use completed optimizer updates consistently in target snapshots, retain the selected budget probe, and encode overflowing relative errors as counted nulls. diff --git a/changelog.d/acs-native-declaration-cache.changed.md b/changelog.d/acs-native-declaration-cache.changed.md new file mode 100644 index 000000000..fa9e57827 --- /dev/null +++ b/changelog.d/acs-native-declaration-cache.changed.md @@ -0,0 +1 @@ +Cache immutable ACS source declaration summaries by exact source bytes and parser context while preserving fresh source reads and live implementation checks. diff --git a/changelog.d/acs-serial-lookup.fixed.md b/changelog.d/acs-serial-lookup.fixed.md new file mode 100644 index 000000000..0f7e90355 --- /dev/null +++ b/changelog.d/acs-serial-lookup.fixed.md @@ -0,0 +1 @@ +Reuse ACS household-key lookups across person archive chunks while preserving source validation and selection. diff --git a/changelog.d/acs-spm-source-assembly.added.md b/changelog.d/acs-spm-source-assembly.added.md new file mode 100644 index 000000000..c3bde0eff --- /dev/null +++ b/changelog.d/acs-spm-source-assembly.added.md @@ -0,0 +1 @@ +Add opt-in ACS SPM source construction with a selected-roster identity registry, preserving tax units and household design weights before source amounts are mapped. Bind explicit native construction requests to captured-source rowwise evidence and reviewed assembler identity without enabling a default or supplying engine roles. diff --git a/changelog.d/asec-farm-allocation-meaning.fixed.md b/changelog.d/asec-farm-allocation-meaning.fixed.md new file mode 100644 index 000000000..ab570a7c3 --- /dev/null +++ b/changelog.d/asec-farm-allocation-meaning.fixed.md @@ -0,0 +1 @@ +Keep farm allocation provenance unresolved when only I_FRMYN has a nonzero code: the published dictionary supplies its range but no code meanings. Preserve allocation established by documented flags. diff --git a/changelog.d/asec-source-domain-contract.fixed.md b/changelog.d/asec-source-domain-contract.fixed.md new file mode 100644 index 000000000..e6b385f9c --- /dev/null +++ b/changelog.d/asec-source-domain-contract.fixed.md @@ -0,0 +1,3 @@ +Prevent returned ASEC amount metadata from mutating cached definitions. Require +unique current-vintage entries, supported interest and child-support domains, +and agreement with the retained current-money owner before source capture. diff --git a/changelog.d/asec-usual-hours-observations.added.md b/changelog.d/asec-usual-hours-observations.added.md new file mode 100644 index 000000000..0cd4bb50f --- /dev/null +++ b/changelog.d/asec-usual-hours-observations.added.md @@ -0,0 +1 @@ +Add an all-age ASEC usual-hours observation recoder that separates source nonwork from survey-universe absence, preserves allocation flags, and leaves unresolved histories unfilled. diff --git a/changelog.d/atomic-property-financial-host.added.md b/changelog.d/atomic-property-financial-host.added.md new file mode 100644 index 000000000..460a98daf --- /dev/null +++ b/changelog.d/atomic-property-financial-host.added.md @@ -0,0 +1 @@ +Add an explicit property-income extension to the checked US financial host, retaining complete legacy populations and validating model receipts, reconstructed outputs and required cache replay. diff --git a/changelog.d/atomic-property-tax-host.added.md b/changelog.d/atomic-property-tax-host.added.md new file mode 100644 index 000000000..f3ef5b12a --- /dev/null +++ b/changelog.d/atomic-property-tax-host.added.md @@ -0,0 +1,5 @@ +Add an explicit optional 38-node survey financial host that retains its complete +19- and 35-node populations, rebases property tax leaves with strict unknowns, +and verifies the new full population and numerical artifacts. Add a conditional +typed PUF gate dependency and refuse PUF qualification when required rebased +inputs remain unknown. The existing 19- and 35-node options remain unchanged. diff --git a/changelog.d/calibrate-parity-repin.fixed.md b/changelog.d/calibrate-parity-repin.fixed.md new file mode 100644 index 000000000..c84ba9eec --- /dev/null +++ b/changelog.d/calibrate-parity-repin.fixed.md @@ -0,0 +1 @@ +Refresh calibration parity implementation and node-key pins after the validated Weights copy-protocol change, preserving all recorded direct outputs and platform entries. diff --git a/changelog.d/child-completion-structural-reads.fixed.md b/changelog.d/child-completion-structural-reads.fixed.md new file mode 100644 index 000000000..e26ec99e7 --- /dev/null +++ b/changelog.d/child-completion-structural-reads.fixed.md @@ -0,0 +1 @@ +Fix child property-income graph compilation by taking entity IDs and membership IDs from the executor's structural view while retaining all source-identity column dependencies. diff --git a/changelog.d/child-property-empirical-candidate.added.md b/changelog.d/child-property-empirical-candidate.added.md new file mode 100644 index 000000000..b5a9be753 --- /dev/null +++ b/changelog.d/child-property-empirical-candidate.added.md @@ -0,0 +1,3 @@ +Add an explicit weighted paired empirical model and a full-source ASEC teenager donor qualifier for an opt-in child property-income candidate, preserving observed values, source-knownness and original household design support. The source-only candidate supplies no released default or native-data acceptance. + +Preserve small supported zero-pattern mass during reference transport, require support for every present pattern, refuse wider floating-point inputs before narrowing, and bind consumed allocation interpretation constants into the live source seal. diff --git a/changelog.d/child-property-income-graph.added.md b/changelog.d/child-property-income-graph.added.md new file mode 100644 index 000000000..829fe308c --- /dev/null +++ b/changelog.d/child-property-income-graph.added.md @@ -0,0 +1 @@ +Add an opt-in source-owned child property-income graph fragment with explicit private model-support schemas, paired empirical draws and independent full-population verification; integration and runtime acceptance remain separate. diff --git a/changelog.d/child-property-support-frame-stamp.fixed.md b/changelog.d/child-property-support-frame-stamp.fixed.md new file mode 100644 index 000000000..5b7b99b94 --- /dev/null +++ b/changelog.d/child-property-support-frame-stamp.fixed.md @@ -0,0 +1 @@ +Seal child-property donor and recipient Frames using their private support schema and complete physical contents across the final owner callback. Keep the receiving US population's strict source identity checks unchanged. diff --git a/changelog.d/child-receiving-fixture-strings.fixed.md b/changelog.d/child-receiving-fixture-strings.fixed.md new file mode 100644 index 000000000..6af7ff29f --- /dev/null +++ b/changelog.d/child-receiving-fixture-strings.fixed.md @@ -0,0 +1 @@ +Declare canonical nullable string dtypes in the invented child-property receiving fixture, preserving source-channel values, census-block leading zeros, exact large IDs, and income knownness. diff --git a/changelog.d/ci-file-shards.fixed.md b/changelog.d/ci-file-shards.fixed.md new file mode 100644 index 000000000..3825691c4 --- /dev/null +++ b/changelog.d/ci-file-shards.fixed.md @@ -0,0 +1 @@ +Split concentrated CI test groups and the full installed-wheel inventory into four parallel file shards per Python version, preserving module fixtures, environment coverage and process isolation, and fail before pytest when file selection fails or is empty. Enforce the flat inventory depth so nested tests are rejected by verification rather than absorbed by Git's ordinary wildcard matching. diff --git a/changelog.d/common-frame-export-contract.added.md b/changelog.d/common-frame-export-contract.added.md new file mode 100644 index 000000000..ea5fadddb --- /dev/null +++ b/changelog.d/common-frame-export-contract.added.md @@ -0,0 +1 @@ +Add supplied-parent comparison for US full, pruned and local frame exports, preserving retained inputs and missingness through checkpoint readback. diff --git a/changelog.d/country-bundle-shared-source-pins.fixed.md b/changelog.d/country-bundle-shared-source-pins.fixed.md new file mode 100644 index 000000000..089ac9751 --- /dev/null +++ b/changelog.d/country-bundle-shared-source-pins.fixed.md @@ -0,0 +1 @@ +Update the AM, BE and UK country-bundle identity checks after the reviewed calibration snapshot metadata changes. Their authored resources and draw-site bindings remain unchanged; their resolved spec digests include the shared solver source attestation. diff --git a/changelog.d/current-acs-immigration-source-projection.added.md b/changelog.d/current-acs-immigration-source-projection.added.md new file mode 100644 index 000000000..bb2cf7309 --- /dev/null +++ b/changelog.d/current-acs-immigration-source-projection.added.md @@ -0,0 +1 @@ +Add an opt-in selected ACS immigration source projection that binds original age, sex, household state and literal entry/birthplace evidence through existing owners, preserves NIU and grouped-code limits, and grants no status or stock admission. diff --git a/changelog.d/current-asec-child-support-source.added.md b/changelog.d/current-asec-child-support-source.added.md new file mode 100644 index 000000000..4b951e9a3 --- /dev/null +++ b/changelog.d/current-asec-child-support-source.added.md @@ -0,0 +1 @@ +Qualify retained ASEC child-support amounts, receipt and obligation answers, preserving paid NIU and unknown voluntary payments alongside original source and allocation metadata. diff --git a/changelog.d/current-asec-interest-source.added.md b/changelog.d/current-asec-interest-source.added.md new file mode 100644 index 000000000..73de9e322 --- /dev/null +++ b/changelog.d/current-asec-interest-source.added.md @@ -0,0 +1 @@ +Qualify ordinary and retirement-account ASEC interest from retained source owners, preserving published components, combined totals, knownness and allocation metadata without assigning tax treatment or balancing observations. diff --git a/changelog.d/current-survey-property-graph.added.md b/changelog.d/current-survey-property-graph.added.md new file mode 100644 index 000000000..45b5f9bd0 --- /dev/null +++ b/changelog.d/current-survey-property-graph.added.md @@ -0,0 +1 @@ +Add an opt-in 16-node survey property-income fragment with original DESIGN donor selection, joint ACS draws, signed reconciliation, explicit unknownness, and exact clone attachment while preserving legacy tax leaves and their stated capital-gains limitation. diff --git a/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md new file mode 100644 index 000000000..8f4a31c03 --- /dev/null +++ b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md @@ -0,0 +1 @@ +Bind fiscal measurement caches to target arithmetic and engine table materialization, refuse missing consumed predicate inputs, and count boolean terms arithmetically in shared sum expressions. diff --git a/changelog.d/full-asec-immigration-donor.added.md b/changelog.d/full-asec-immigration-donor.added.md new file mode 100644 index 000000000..334f0ad00 --- /dev/null +++ b/changelog.d/full-asec-immigration-donor.added.md @@ -0,0 +1 @@ +Add a checked full-original-ASEC immigration donor projection with exact household design-weight anchors, original demographic predictors and strict literal-token qualification, without status assignment or graph attachment. diff --git a/changelog.d/graph-scalar-float-packing.changed.md b/changelog.d/graph-scalar-float-packing.changed.md new file mode 100644 index 000000000..0cb0b67fe --- /dev/null +++ b/changelog.d/graph-scalar-float-packing.changed.md @@ -0,0 +1 @@ +Pack an exact Python float straight into the graph executor's scalar digest framing with `struct.pack("=d", ...)` instead of a one-element float64 array, preserving the payload bytes, the length prefix and every other scalar's behaviour. diff --git a/changelog.d/graph-signed-reconciliation.added.md b/changelog.d/graph-signed-reconciliation.added.md new file mode 100644 index 000000000..96be9e554 --- /dev/null +++ b/changelog.d/graph-signed-reconciliation.added.md @@ -0,0 +1 @@ +Expose signed-total component reconciliation as a graph node with explicit scales, bounds, and row-level adjustment diagnostics. diff --git a/changelog.d/graph-storage-parts-whole-series.changed.md b/changelog.d/graph-storage-parts-whole-series.changed.md new file mode 100644 index 000000000..afe3d3e1c --- /dev/null +++ b/changelog.d/graph-storage-parts-whole-series.changed.md @@ -0,0 +1 @@ +Select whole series with `slice(None)` instead of an all-True mask when folding physical storage parts, and skip the null recomputation that masked storage never uses, preserving the emitted value bytes, null bytes and subset-selection behaviour exactly. diff --git a/changelog.d/grouped-target-snapshot-integration.added.md b/changelog.d/grouped-target-snapshot-integration.added.md new file mode 100644 index 000000000..c5df41593 --- /dev/null +++ b/changelog.d/grouped-target-snapshot-integration.added.md @@ -0,0 +1 @@ +Instrument the grouped/fixed-zero Adam solver with the opt-in calibration target snapshot observer, reusing the estimate tensor each epoch's loss was computed from and the closing float64 diagnostics estimate. diff --git a/changelog.d/invented-enrichment-ss-universe.fixed.md b/changelog.d/invented-enrichment-ss-universe.fixed.md new file mode 100644 index 000000000..f0fa89fb8 --- /dev/null +++ b/changelog.d/invented-enrichment-ss-universe.fixed.md @@ -0,0 +1 @@ +Keep the invented enrichment fixture's Social Security nonreceipt answer consistent when its age-14 donor is deliberately changed to age 15, and check the genuine source qualifier preserves other unknown reporting statuses. diff --git a/changelog.d/joint-empirical-graph.added.md b/changelog.d/joint-empirical-graph.added.md new file mode 100644 index 000000000..668f16fc3 --- /dev/null +++ b/changelog.d/joint-empirical-graph.added.md @@ -0,0 +1 @@ +Add typed private joint empirical fit and keyed draw graph operations with exact source-coordinate lineage and explicit support and transport contracts. diff --git a/changelog.d/medicaid-substitution-hierarchy.fixed.md b/changelog.d/medicaid-substitution-hierarchy.fixed.md new file mode 100644 index 000000000..20e3e17fd --- /dev/null +++ b/changelog.d/medicaid-substitution-hierarchy.fixed.md @@ -0,0 +1,5 @@ +Preserve a complete, truthful calibration hierarchy when substituting Rhode +Island's reviewed November 2024 Medicaid enrollment count for its missing +December snapshot. Rebind the target identity and state labels/dimension while +retaining provider/category and source-substitution provenance; keep legacy +targets without hierarchies supported. diff --git a/changelog.d/native-aca-frame-context.fixed.md b/changelog.d/native-aca-frame-context.fixed.md new file mode 100644 index 000000000..d37658b04 --- /dev/null +++ b/changelog.d/native-aca-frame-context.fixed.md @@ -0,0 +1 @@ +Preserve Frame metadata and mass-change history when refreshing ACA marketplace inputs. diff --git a/changelog.d/native-acs-anchor-iteration.changed.md b/changelog.d/native-acs-anchor-iteration.changed.md new file mode 100644 index 000000000..a6e51289e --- /dev/null +++ b/changelog.d/native-acs-anchor-iteration.changed.md @@ -0,0 +1 @@ +Iterate retained ACS income-anchor columns without allocating full record-dictionary lists, preserving literal checks, exact projections, nullable amount bits, and final source-owner checks. diff --git a/changelog.d/native-acs-health-completion.added.md b/changelog.d/native-acs-health-completion.added.md new file mode 100644 index 000000000..c92ec3eb1 --- /dev/null +++ b/changelog.d/native-acs-health-completion.added.md @@ -0,0 +1 @@ +Add optional graph-based completion of seven ACS health coverage categories from authenticated original ASEC donors, preserving source knownness and drawing once per original person before clone transport. Scientific and release qualification remain separate. diff --git a/changelog.d/native-acs-spm-scope.added.md b/changelog.d/native-acs-spm-scope.added.md new file mode 100644 index 000000000..a43343a34 --- /dev/null +++ b/changelog.d/native-acs-spm-scope.added.md @@ -0,0 +1 @@ +Add a pure ACS 2024 household-only SPM scope classifier that preserves nullable roles, source-evidence labels and household-wide uncertainty, and validates native membership mappings without issuing source authority or running a country model. diff --git a/changelog.d/native-all-person-hours.changed.md b/changelog.d/native-all-person-hours.changed.md new file mode 100644 index 000000000..7270c7a5e --- /dev/null +++ b/changelog.d/native-all-person-hours.changed.md @@ -0,0 +1 @@ +Qualify usual-hours proposals for both original survey arms, retaining ASEC earnings evidence and explicitly labeling child zero completion; transport proposals through checked two-clone ancestry without redrawing or overwriting owned fields. diff --git a/changelog.d/native-asec-immigration-assignment.added.md b/changelog.d/native-asec-immigration-assignment.added.md new file mode 100644 index 000000000..a17e0f33b --- /dev/null +++ b/changelog.d/native-asec-immigration-assignment.added.md @@ -0,0 +1 @@ +Add a retained development immigration assignment over the complete original ASEC donor, using qualified current source evidence, 2025 observation years and original household design weights. Preserve original-key status pairs before selection, disclose unchanged mixed-date control assumptions, and keep national stock alignment unqualified. diff --git a/changelog.d/native-calibration-attachment.added.md b/changelog.d/native-calibration-attachment.added.md new file mode 100644 index 000000000..fd5e1c485 --- /dev/null +++ b/changelog.d/native-calibration-attachment.added.md @@ -0,0 +1 @@ +Add a private dense/L0 calibration bridge that attaches weights to unchanged projected inputs and verifies whole-household retention against the supplied parent without claiming native ancestry or release readiness. diff --git a/changelog.d/native-canonical-state-binding.changed.md b/changelog.d/native-canonical-state-binding.changed.md new file mode 100644 index 000000000..a808c2eed --- /dev/null +++ b/changelog.d/native-canonical-state-binding.changed.md @@ -0,0 +1 @@ +Add an independent graph fragment that binds canonical household state FIPS from existing block-derived geography and checks observed survey state equality. Explicit rewrites replace carried canonical values while preserving other inputs, rows, memberships, and weights. Enrichment-host integration and tenure-source completion remain separate. diff --git a/changelog.d/native-canonical-state-host.changed.md b/changelog.d/native-canonical-state-host.changed.md new file mode 100644 index 000000000..49a777ffe --- /dev/null +++ b/changelog.d/native-canonical-state-host.changed.md @@ -0,0 +1 @@ +Add an explicit default-off canonical state binding to native US enrichment, with a keep-all population boundary, typed geography/terminal dependencies, and complete reconstruction on cold and required replay. diff --git a/changelog.d/native-categorical-gate.added.md b/changelog.d/native-categorical-gate.added.md new file mode 100644 index 000000000..dabac088c --- /dev/null +++ b/changelog.d/native-categorical-gate.added.md @@ -0,0 +1 @@ +Add a separate DESIGN-weighted categorical classifier artifact and typed probability graph adapter with explicit class support, source/model/matrix bindings and no changes to QRF or canonical survey outputs. diff --git a/changelog.d/native-comparison-geography-id-binding.fixed.md b/changelog.d/native-comparison-geography-id-binding.fixed.md new file mode 100644 index 000000000..2aac9d5de --- /dev/null +++ b/changelog.d/native-comparison-geography-id-binding.fixed.md @@ -0,0 +1 @@ +Bind a US replacement-scorecard row's area identifier to the level and declaration that resolved it. The national/state/congressional-district view chose a level from one piece of evidence and then searched an ordered metadata key list that consulted neither, so a row without a view-level hierarchy id would have reported the parent `state_fips` the target compiler derives from a district geoid, or a sub-state hierarchy id as a state's. That search was unreachable for every spec the US compiler can currently emit -- each carries a hierarchy whose geography id is non-empty -- so no scorecard's counts change; this closes the hole before a producer reaches it. An identifier is now read only from a declaration at the row's own level, a row with none keeps an empty identifier, and the artifact reports which declaration named each area. Distinct-area counts are taken over census GEOIDs alone, with bare-form and unidentified rows counted beside them, so one area cannot be counted twice under two spellings. Scorecard schema 4 -> 5. diff --git a/changelog.d/native-comparison-geography-validation.fixed.md b/changelog.d/native-comparison-geography-validation.fixed.md new file mode 100644 index 000000000..39eeb8243 --- /dev/null +++ b/changelog.d/native-comparison-geography-validation.fixed.md @@ -0,0 +1 @@ +Validate geographic identifier encodings before counting census areas, preserve unsupported explicit scopes, and exclude unresolved scopes from the known-level missing-area count in schema-6 US comparison scorecards. diff --git a/changelog.d/native-comparison-national-cd-view.added.md b/changelog.d/native-comparison-national-cd-view.added.md new file mode 100644 index 000000000..b86f39d73 --- /dev/null +++ b/changelog.d/native-comparison-national-cd-view.added.md @@ -0,0 +1 @@ +Decompose the US head-to-head replacement scorecard on a national/state/congressional-district view. Every fiscal row now carries the comparison view its declared target evidence names, each artifact payload and the head-to-head comparison roll up on that axis, and a row no declared evidence places is reported `unresolved` rather than counted as national. This is the previously unproduced `national_and_cd_target_fit` item of the native survey release-evidence list; it adds no threshold, tolerance or verdict. diff --git a/changelog.d/native-completion-retention.changed.md b/changelog.d/native-completion-retention.changed.md new file mode 100644 index 000000000..61c8cb04b --- /dev/null +++ b/changelog.d/native-completion-retention.changed.md @@ -0,0 +1 @@ +Add private opt-in compact observation retention for the native financial and full completion graph paths, preserving the default retained snapshots and replay checks. diff --git a/changelog.d/native-consumer-metadata.added.md b/changelog.d/native-consumer-metadata.added.md new file mode 100644 index 000000000..b9219b33e --- /dev/null +++ b/changelog.d/native-consumer-metadata.added.md @@ -0,0 +1 @@ +Add an optional live-consumer representation check for exact native survey projections, using actual variable dtypes, enum domains, periods and source ownership without filling inputs or claiming release qualification. diff --git a/changelog.d/native-csv-record-scan.changed.md b/changelog.d/native-csv-record-scan.changed.md new file mode 100644 index 000000000..276b3c91f --- /dev/null +++ b/changelog.d/native-csv-record-scan.changed.md @@ -0,0 +1 @@ +Speed up bounded, complete unquoted ASEC CSV record checks while preserving byte-level limits, budget accounting, source authentication and refusal behavior. diff --git a/changelog.d/native-engine-projection-20260919.added.md b/changelog.d/native-engine-projection-20260919.added.md new file mode 100644 index 000000000..a241253fa --- /dev/null +++ b/changelog.d/native-engine-projection-20260919.added.md @@ -0,0 +1 @@ +Add an in-process native development projection that preserves selected source cells and context exactly, while keeping consumer qualification, simulation readiness, and release eligibility false. diff --git a/changelog.d/native-financial-check-epoch.changed.md b/changelog.d/native-financial-check-epoch.changed.md new file mode 100644 index 000000000..3ad59aa87 --- /dev/null +++ b/changelog.d/native-financial-check-epoch.changed.md @@ -0,0 +1 @@ +Reuse authenticated preparation validation within one native financial run check. Every independent check still fully validates its sources at entry and close; checks inside an existing verification epoch join that scope, whose close must succeed before the caller returns. Direct Frame seals and source-change refusals remain enabled. Changes invisible to the memo signature may be refused at the enclosing close instead of an intervening borrow. diff --git a/changelog.d/native-full-original-amount-donors.added.md b/changelog.d/native-full-original-amount-donors.added.md new file mode 100644 index 000000000..cefb18f57 --- /dev/null +++ b/changelog.d/native-full-original-amount-donors.added.md @@ -0,0 +1 @@ +Add optional full-original ASEC DESIGN donor support for native current survey amount fitting, independent of selected receiving support and preserving observed amounts, recipient applicability and existing clones. diff --git a/changelog.d/native-hours-graph.changed.md b/changelog.d/native-hours-graph.changed.md new file mode 100644 index 000000000..e70abd979 --- /dev/null +++ b/changelog.d/native-hours-graph.changed.md @@ -0,0 +1 @@ +Execute authenticated ACS and ASEC usual-hours recoding as native graph operations and attach original-person hours and provenance to exact clones after survey enrichment, with independent replay checks. diff --git a/changelog.d/native-hours-handoff-controls.fixed.md b/changelog.d/native-hours-handoff-controls.fixed.md new file mode 100644 index 000000000..67b0d0272 --- /dev/null +++ b/changelog.d/native-hours-handoff-controls.fixed.md @@ -0,0 +1 @@ +Classify native development handoff and retained population witnesses in the runtime inventory, check hours/provenance checkpoint preservation, and recognize only the exact reviewed SPM receipt transfer from Frame metadata to its source owner. diff --git a/changelog.d/native-hours-response-status.fixed.md b/changelog.d/native-hours-response-status.fixed.md new file mode 100644 index 000000000..a6e4f6638 --- /dev/null +++ b/changelog.d/native-hours-response-status.fixed.md @@ -0,0 +1 @@ +Accept and preserve the Census ASEC complete supplement nonresponse code in usual-hours proposals without treating it as source or donor qualification. diff --git a/changelog.d/native-hours-source-owner.added.md b/changelog.d/native-hours-source-owner.added.md new file mode 100644 index 000000000..6ca1145ff --- /dev/null +++ b/changelog.d/native-hours-source-owner.added.md @@ -0,0 +1 @@ +Add a retained source owner for ACS usual-hours proposals, binding original survey records and the complete current ASEC age-15 donor cohort independently of selected household support. ASEC all-age hours and release qualification remain separate. diff --git a/changelog.d/native-household-role-namespace.fixed.md b/changelog.d/native-household-role-namespace.fixed.md new file mode 100644 index 000000000..868a31435 --- /dev/null +++ b/changelog.d/native-household-role-namespace.fixed.md @@ -0,0 +1 @@ +Preserve legacy prepared headship separately from source-qualified nullable household roles, retaining disagreement diagnostics and strict canonical-role checks. diff --git a/changelog.d/native-immigration-clone-attachment.added.md b/changelog.d/native-immigration-clone-attachment.added.md new file mode 100644 index 000000000..55df31bd7 --- /dev/null +++ b/changelog.d/native-immigration-clone-attachment.added.md @@ -0,0 +1 @@ +Optionally attach a genuine original-survey immigration transfer's realized pair to both existing survey clones through the retained enrichment host, preserving source ancestry, geography, weights and replay checks without another fit or draw. diff --git a/changelog.d/native-immigration-consumed-controls.added.md b/changelog.d/native-immigration-consumed-controls.added.md new file mode 100644 index 000000000..7b89170ba --- /dev/null +++ b/changelog.d/native-immigration-consumed-controls.added.md @@ -0,0 +1 @@ +Allow ACS immigration reconciliation to consume explicitly captured controls, including after target-bank replay, and parse those controls from a captured source stage while preserving packaged defaults. diff --git a/changelog.d/native-immigration-observation-years.added.md b/changelog.d/native-immigration-observation-years.added.md new file mode 100644 index 000000000..9b5859331 --- /dev/null +++ b/changelog.d/native-immigration-observation-years.added.md @@ -0,0 +1 @@ +Adopt the reviewed immigration rule and paired ACS reconciliation surface, with explicit per-person observation years, current labor-force inputs, year-independent absent-arrival features, and feature-bound checkpoint replay. Preserve existing control values and the default scalar observation year; native source assignment remains separate. diff --git a/changelog.d/native-immigration-source-bundle.fixed.md b/changelog.d/native-immigration-source-bundle.fixed.md new file mode 100644 index 000000000..27c0cd932 --- /dev/null +++ b/changelog.d/native-immigration-source-bundle.fixed.md @@ -0,0 +1 @@ +Regenerate the typed immigration source bundle from the reviewed source manifest, declare its closed humanitarian-stock schema, and check source projection and renderer equality without requiring the optional country engine. diff --git a/changelog.d/native-immigration-source.added.md b/changelog.d/native-immigration-source.added.md new file mode 100644 index 000000000..e3de3341f --- /dev/null +++ b/changelog.d/native-immigration-source.added.md @@ -0,0 +1 @@ +Add a native immigration source qualifier that retains the complete current ASEC readset and selected ACS literals, with exact source custody and clone evidence transport. Status assignment, stock alignment and release qualification remain separate reviewed steps. diff --git a/changelog.d/native-immigration-transfer-spec.fixed.md b/changelog.d/native-immigration-transfer-spec.fixed.md new file mode 100644 index 000000000..1e63e7b3c --- /dev/null +++ b/changelog.d/native-immigration-transfer-spec.fixed.md @@ -0,0 +1 @@ +Regenerate the typed ACS transfer declaration from its live v4 contract without country-engine metadata, retain immigration evidence and humanitarian reconciliation semantics, and match existing adult-care and Schedule D activation rules. diff --git a/changelog.d/native-input-name-profile.added.md b/changelog.d/native-input-name-profile.added.md new file mode 100644 index 000000000..34c433089 --- /dev/null +++ b/changelog.d/native-input-name-profile.added.md @@ -0,0 +1 @@ +Add an explicit 159-name native US national/CD diagnostic profile that excludes the canonical prior-year income family and reports those scope exclusions. Preserve the existing 163-name historical and 161-name national/CD profiles and their defaults; native source and release qualification remain separate. diff --git a/changelog.d/native-original-immigration-transfer.added.md b/changelog.d/native-original-immigration-transfer.added.md new file mode 100644 index 000000000..3ebc4ce5b --- /dev/null +++ b/changelog.d/native-original-immigration-transfer.added.md @@ -0,0 +1 @@ +Add a retained paired ACS immigration transfer that uses the complete original ASEC donor and the same native graph parent's original survey allocation, with explicit consumed controls and unchanged source weights. diff --git a/changelog.d/native-other-disability-qualification.fixed.md b/changelog.d/native-other-disability-qualification.fixed.md new file mode 100644 index 000000000..393bf28a0 --- /dev/null +++ b/changelog.d/native-other-disability-qualification.fixed.md @@ -0,0 +1 @@ +Recheck the other-disability adapter's implementation after its final source-owner validation, refusing semantic or qualifier changes before returning observations. diff --git a/changelog.d/native-primary-family-source.added.md b/changelog.d/native-primary-family-source.added.md new file mode 100644 index 000000000..83755b09e --- /dev/null +++ b/changelog.d/native-primary-family-source.added.md @@ -0,0 +1 @@ +Add source-qualified Census primary-family, own-child, household-size, and separate resident-spouse-pair diagnostics with explicit unknown and group-quarters states. diff --git a/changelog.d/native-puf-original-carried-outputs.fixed.md b/changelog.d/native-puf-original-carried-outputs.fixed.md new file mode 100644 index 000000000..6a6622404 --- /dev/null +++ b/changelog.d/native-puf-original-carried-outputs.fixed.md @@ -0,0 +1 @@ +Accept the receiving structural version as the owner of carried arm-one PUF outputs in the original placement cut while proving the carried values are unchanged. diff --git a/changelog.d/native-puf-original-host.added.md b/changelog.d/native-puf-original-host.added.md new file mode 100644 index 000000000..555f4fedd --- /dev/null +++ b/changelog.d/native-puf-original-host.added.md @@ -0,0 +1 @@ +Add opt-in original-arm PUF55 application and conservative placement to the survey enrichment host, preserving the actual selected terminal and independent source, artifact, population, and storage checks. Keep the default path unchanged; full host and release qualification remain separate from bounded component verification. diff --git a/changelog.d/native-puf-original-placement-artifact-replay.fixed.md b/changelog.d/native-puf-original-placement-artifact-replay.fixed.md new file mode 100644 index 000000000..0b24b167f --- /dev/null +++ b/changelog.d/native-puf-original-placement-artifact-replay.fixed.md @@ -0,0 +1 @@ +Persist the input population versions instead of the in-process population seal in the original PUF placement artifact so the required replay reconstructs it byte-for-byte. diff --git a/changelog.d/native-puf-original-placement.added.md b/changelog.d/native-puf-original-placement.added.md new file mode 100644 index 000000000..301e3f638 --- /dev/null +++ b/changelog.d/native-puf-original-placement.added.md @@ -0,0 +1 @@ +Add conservative original-arm PUF55 development placement helpers and typed graph declarations. Preserve known and preexisting values, leave mixed-known units unresolved, and permit five new person values only on complete eligible singletons. The three tax-unit outputs and all 55 upstream application dependencies are explicit; checked-host integration and release qualification remain separate. diff --git a/changelog.d/native-puf-original-source-version.fixed.md b/changelog.d/native-puf-original-source-version.fixed.md new file mode 100644 index 000000000..f6edba787 --- /dev/null +++ b/changelog.d/native-puf-original-source-version.fixed.md @@ -0,0 +1 @@ +Open a private keep-all version for the original PUF arm's recipient and fixed-input nodes so they no longer join the financial version closure and change the authenticated parent's keys. diff --git a/changelog.d/native-row-ceilings.changed.md b/changelog.d/native-row-ceilings.changed.md new file mode 100644 index 000000000..16d60b926 --- /dev/null +++ b/changelog.d/native-row-ceilings.changed.md @@ -0,0 +1 @@ +Raise the seven US native-build row-count ceilings a full-source build meets to four times their measured full-source counts, keeping every refusal code and expression, and re-pin the one implementation digest that moves. diff --git a/changelog.d/native-runtime-helper-classification.fixed.md b/changelog.d/native-runtime-helper-classification.fixed.md new file mode 100644 index 000000000..fe22733a4 --- /dev/null +++ b/changelog.d/native-runtime-helper-classification.fixed.md @@ -0,0 +1 @@ +Classify the shared prior-year income names and logical H5 readback verifier in the exhaustive US runtime inventory, preserving the checks for unreviewed modules and population-treatment boundaries. diff --git a/changelog.d/native-soi-agi-size-targets.added.md b/changelog.d/native-soi-agi-size-targets.added.md new file mode 100644 index 000000000..c5a4ff679 --- /dev/null +++ b/changelog.d/native-soi-agi-size-targets.added.md @@ -0,0 +1 @@ +Adopt the narrow SOI Table 1.1 cross-period size-of-AGI compiler from #960 for native integration, preserving source identities and control-year alignment with additional invented exclusion and aging tests. No feed, target-surface or dataset pins change. diff --git a/changelog.d/native-spm-amount-source.added.md b/changelog.d/native-spm-amount-source.added.md new file mode 100644 index 000000000..c56205172 --- /dev/null +++ b/changelog.d/native-spm-amount-source.added.md @@ -0,0 +1 @@ +Add a source-only SPM amount projection that borrows existing current ASEC money and original membership owners, preserves repeated unit values and unresolved zeros, and leaves missing observations explicit. diff --git a/changelog.d/native-spm-complete-unit-projection.added.md b/changelog.d/native-spm-complete-unit-projection.added.md new file mode 100644 index 000000000..a651e06f1 --- /dev/null +++ b/changelog.d/native-spm-complete-unit-projection.added.md @@ -0,0 +1,4 @@ +Add a pure annual SPM source-input projection that verifies complete original +SPM-unit and household membership through both native clones. Preserve nullable +source roles and require explicit OUTSIDE scope before applying a Boolean +storage placeholder; refuse changed years, ancestry or existing output columns. diff --git a/changelog.d/native-spm-country-source-inputs.fixed.md b/changelog.d/native-spm-country-source-inputs.fixed.md new file mode 100644 index 000000000..776c7124f --- /dev/null +++ b/changelog.d/native-spm-country-source-inputs.fixed.md @@ -0,0 +1 @@ +Honor PolicyEngine-US's declared dataset source inputs in the live adapter and import-free metadata index, including SPM independence roles and annual universe status despite household fallback formulas. Reject malformed or conflicting declarations, retain formula-owned output guards, and omit source observations from generated defaults. diff --git a/changelog.d/native-spm-enrichment-graph.added.md b/changelog.d/native-spm-enrichment-graph.added.md new file mode 100644 index 000000000..8cc467290 --- /dev/null +++ b/changelog.d/native-spm-enrichment-graph.added.md @@ -0,0 +1 @@ +Add an explicitly configured three-node survey enrichment fragment for retained SPM roles and annual universe scope, with complete-unit transport, replay verification and final source/output validation. diff --git a/changelog.d/native-spm-retained-source.added.md b/changelog.d/native-spm-retained-source.added.md new file mode 100644 index 000000000..fa2d3a9de --- /dev/null +++ b/changelog.d/native-spm-retained-source.added.md @@ -0,0 +1 @@ +Add retained-owner qualification of nullable ACS and ASEC SPM roles and explicit annual scope without granting dataset or release certification. diff --git a/changelog.d/native-ss-full-original-basis.added.md b/changelog.d/native-ss-full-original-basis.added.md new file mode 100644 index 000000000..fb5890af5 --- /dev/null +++ b/changelog.d/native-ss-full-original-basis.added.md @@ -0,0 +1 @@ +Add an optional full-original ASEC Social Security report basis with native IDs, original DESIGN weights, source literals and unchanged selected-survey descriptions, leaving donor selection and component modeling separate. diff --git a/changelog.d/native-ss-report-completion.added.md b/changelog.d/native-ss-report-completion.added.md new file mode 100644 index 000000000..49226937b --- /dev/null +++ b/changelog.d/native-ss-report-completion.added.md @@ -0,0 +1 @@ +Add an optional source-qualified Social Security report fragment with full-original ASEC DESIGN donors, real categorical fit/probability artifacts, reason-constrained report completion, and a no-fit empty-recipient path. It preserves source unknowns and does not write canonical beneficiary inputs or qualify a release. diff --git a/changelog.d/native-survey-development-handoff.added.md b/changelog.d/native-survey-development-handoff.added.md new file mode 100644 index 000000000..49a89093b --- /dev/null +++ b/changelog.d/native-survey-development-handoff.added.md @@ -0,0 +1 @@ +Add a checked native survey development handoff in the maintained US release builder, preserving complete Frame checkpoints and source/model flags while reporting missing inputs and outstanding release qualifications without cloning or fabricating pool receipts. diff --git a/changelog.d/native-survey-development-inputs.added.md b/changelog.d/native-survey-development-inputs.added.md new file mode 100644 index 000000000..54a69dab9 --- /dev/null +++ b/changelog.d/native-survey-development-inputs.added.md @@ -0,0 +1 @@ +Add an opt-in graph producer for four source-qualified ASEC development inputs, preserving source unknowns and exact original clone mappings before PUF application. diff --git a/changelog.d/native-survey-race-hispanic-inputs.added.md b/changelog.d/native-survey-race-hispanic-inputs.added.md new file mode 100644 index 000000000..b45253d6b --- /dev/null +++ b/changelog.d/native-survey-race-hispanic-inputs.added.md @@ -0,0 +1 @@ +Add optional source-qualified race and Hispanic-origin graph inputs on original survey persons and existing clones, preserving source literals, allocation histories and unsupported ACS race categories as explicit unknowns. diff --git a/changelog.d/native-survey-sex-inputs.added.md b/changelog.d/native-survey-sex-inputs.added.md new file mode 100644 index 000000000..ee0e309b3 --- /dev/null +++ b/changelog.d/native-survey-sex-inputs.added.md @@ -0,0 +1 @@ +Add opt-in native sex inputs to survey enrichment. The graph exposes qualified ASEC sex/allocation and ACS sex observations, binds nullable `is_female` once per original person, then copies it through the existing clone mapping without filling unknowns or changing weights. diff --git a/changelog.d/native-usual-hours-proposals.added.md b/changelog.d/native-usual-hours-proposals.added.md new file mode 100644 index 000000000..006494317 --- /dev/null +++ b/changelog.d/native-usual-hours-proposals.added.md @@ -0,0 +1 @@ +Add pure native-keyed ACS usual-hours proposals with explicit age15 empirical and under15 modeled-zero policies, preserved source/allocation evidence, and invented validation. Native source and release qualification remain separate. diff --git a/changelog.d/native-verify-once.changed.md b/changelog.d/native-verify-once.changed.md new file mode 100644 index 000000000..60d5f6687 --- /dev/null +++ b/changelog.d/native-verify-once.changed.md @@ -0,0 +1 @@ +verify-once: Verify native sources once per run instead of once per accessor use and once per executed node. A run that the run-end re-derivation refuses now evicts every store object it published, so nothing it wrote survives under the source identity it started with; a run that leaves by any other exception gets the same re-derivation on its way out, keeping its work when its sources held still and evicting it otherwise. A process that stops without unwinding, after a rewrite that left every stat field identical, is not covered. diff --git a/changelog.d/native-veterans-benefits.added.md b/changelog.d/native-veterans-benefits.added.md new file mode 100644 index 000000000..ec11292c4 --- /dev/null +++ b/changelog.d/native-veterans-benefits.added.md @@ -0,0 +1 @@ +Add an opt-in native veterans’ benefits amount route using qualified full-original ASEC donors, preserved receipt/allocation unknownness, original-person ACS draws and explicit canonical clone attachment. diff --git a/changelog.d/native-workers-compensation.added.md b/changelog.d/native-workers-compensation.added.md new file mode 100644 index 000000000..53d87684b --- /dev/null +++ b/changelog.d/native-workers-compensation.added.md @@ -0,0 +1 @@ +Add an opt-in native workers compensation amount group using authenticated ASEC receipt evidence, preserving unresolved reporting status and sharing one ACS draw across support clones. diff --git a/changelog.d/person-status-tax-filter-ownership.fixed.md b/changelog.d/person-status-tax-filter-ownership.fixed.md new file mode 100644 index 000000000..5c3729b63 --- /dev/null +++ b/changelog.d/person-status-tax-filter-ownership.fixed.md @@ -0,0 +1 @@ +Validate descriptive survey-status provenance on the retained pre-tax population while preserving the tax FILTER's current ownership, full-population seals and independent tax reconstruction. diff --git a/changelog.d/pr893-wheel-fixture-contracts.fixed.md b/changelog.d/pr893-wheel-fixture-contracts.fixed.md new file mode 100644 index 000000000..b2831932e --- /dev/null +++ b/changelog.d/pr893-wheel-fixture-contracts.fixed.md @@ -0,0 +1 @@ +Gate pool orchestration tests that require US engine metadata, preserve source-enrichment checkout rejection in wheel tests, and isolate invented UK calibration code-pin fixtures while retaining refusal before I/O. diff --git a/changelog.d/property-completion-host.added.md b/changelog.d/property-completion-host.added.md new file mode 100644 index 000000000..d6d8c1b15 --- /dev/null +++ b/changelog.d/property-completion-host.added.md @@ -0,0 +1 @@ +Add opt-in property completion diagnostics to the existing source projection, with a private typed row artifact, allowlisted aggregate graph receipts and independent replay checks. Preserve default options and numerical property behavior. diff --git a/changelog.d/property-completion-routing.added.md b/changelog.d/property-completion-routing.added.md new file mode 100644 index 000000000..b7e842805 --- /dev/null +++ b/changelog.d/property-completion-routing.added.md @@ -0,0 +1 @@ +Add deterministic original-person property completion routing, independent interest/dividend knownness, exact clone identity diagnostics and original DESIGN support summaries. The pure operation assigns no amounts and grants no source or release authority; host artifact wiring remains separate. diff --git a/changelog.d/property-independent-observation-preservation.internal.md b/changelog.d/property-independent-observation-preservation.internal.md new file mode 100644 index 000000000..cb1fc6a60 --- /dev/null +++ b/changelog.d/property-independent-observation-preservation.internal.md @@ -0,0 +1 @@ +Add a focused invented-source regression confirming that joint property-donor exclusions preserve independently qualified ASEC interest and dividends on both initial clones and through the existing tax split. Production behavior is unchanged. diff --git a/changelog.d/property-model-receipts.added.md b/changelog.d/property-model-receipts.added.md new file mode 100644 index 000000000..de28e20d2 --- /dev/null +++ b/changelog.d/property-model-receipts.added.md @@ -0,0 +1 @@ +Add source-blind verification of the four property fit/apply receipt pairs against authenticated original donor/recipient branches, with complete model history checks and deterministic application replay without refitting. diff --git a/changelog.d/property-survivor-scope.fixed.md b/changelog.d/property-survivor-scope.fixed.md new file mode 100644 index 000000000..4b9258e1c --- /dev/null +++ b/changelog.d/property-survivor-scope.fixed.md @@ -0,0 +1 @@ +Exclude positive survivor receipts from the initial property donor bridge because its two qualified source slots do not cover the additional sources included in the published aggregate. Preserve visible route diagnostics and report the excluded design-weight mass. diff --git a/changelog.d/puf-observer-python313.fixed.md b/changelog.d/puf-observer-python313.fixed.md new file mode 100644 index 000000000..17e72fbcb --- /dev/null +++ b/changelog.d/puf-observer-python313.fixed.md @@ -0,0 +1 @@ +Make PUF graph test observer cleanup compatible with Python 3.13 by explicitly unregistering owned callbacks after disabling events; verify normal and exceptional cleanup and tool reuse. diff --git a/changelog.d/puf55-canonical-donor.added.md b/changelog.d/puf55-canonical-donor.added.md new file mode 100644 index 000000000..10e06ae48 --- /dev/null +++ b/changelog.d/puf55-canonical-donor.added.md @@ -0,0 +1 @@ +Add a canonical59 artifact adapter for the explicit PUF55 survey Social Security profile, retaining return identity and weights and validating the total-carrier convention. Record the scoped passing native seven-node age-development run. diff --git a/changelog.d/puf55-checked-output.added.md b/changelog.d/puf55-checked-output.added.md new file mode 100644 index 000000000..c5fe0d86c --- /dev/null +++ b/changelog.d/puf55-checked-output.added.md @@ -0,0 +1 @@ +Retain PUF55 output handles for source, producer, store, and complete-population rechecks before downstream consumption, revoking a handle after a failed check without rerunning imputation. diff --git a/changelog.d/puf55-survey-host.added.md b/changelog.d/puf55-survey-host.added.md new file mode 100644 index 000000000..f0e410b85 --- /dev/null +++ b/changelog.d/puf55-survey-host.added.md @@ -0,0 +1 @@ +Add the PUF55 host over the combined ACS/ASEC support-clone population, with source-qualified Social Security routes, complete population preservation and required graph replay. diff --git a/changelog.d/puf55-survey-social-security.added.md b/changelog.d/puf55-survey-social-security.added.md new file mode 100644 index 000000000..ba625b57e --- /dev/null +++ b/changelog.d/puf55-survey-social-security.added.md @@ -0,0 +1 @@ +Add an explicit 55-output PUF profile that preserves survey Social Security components and requires a separate total as its ninth conditioning predictor. Validate canonical donor projection and whole-Population attachment/replay while retaining source identities, weights and Social Security unknownness. diff --git a/changelog.d/qrf-observed-matrix-conditioning.added.md b/changelog.d/qrf-observed-matrix-conditioning.added.md new file mode 100644 index 000000000..b1eeeceac --- /dev/null +++ b/changelog.d/qrf-observed-matrix-conditioning.added.md @@ -0,0 +1 @@ +Add an opt-in QRF matrix graph kernel that applies every fitted target, preserves raw draws, and uses identity-bound qualified values to condition later targets. A separate typed conditioning artifact and versioned state record draw, observation, matrix, and producer identities without changing the legacy application protocol. diff --git a/changelog.d/reform-claim-identity.changed.md b/changelog.d/reform-claim-identity.changed.md new file mode 100644 index 000000000..6d4ff055c --- /dev/null +++ b/changelog.d/reform-claim-identity.changed.md @@ -0,0 +1,10 @@ +Add optional benchmark publishers and executable reform definitions to schema-v1 +reform validation reports, ported from PR629 at +`a0441178c9a74bf55dd5aeea62ff9039092c3432`. Preserve scores, years, signs, model +calls, and the `microcosm` payload key; the eight shipped configs gain only the +229 publisher fields from that source. + +Record the actual baseline and reform worlds for simulated comparisons, +including each OBBBA stack step and its merged initial counterfactual when used. +Configured OBBBA repeal definitions remain distinct from those executed +comparisons. Copy definitions so report mutations cannot change input specs. diff --git a/changelog.d/scorer-hdf-engine-gating.fixed.md b/changelog.d/scorer-hdf-engine-gating.fixed.md new file mode 100644 index 000000000..fadf8dfc9 --- /dev/null +++ b/changelog.d/scorer-hdf-engine-gating.fixed.md @@ -0,0 +1 @@ +Mark the four scorer HDF loader tests as requiring the US engine, preserving engine-free installed-wheel source-I/O coverage and execution of the loader tests in the US engine matrix. diff --git a/changelog.d/shared-atomic-geography.added.md b/changelog.d/shared-atomic-geography.added.md new file mode 100644 index 000000000..4eed5d1a9 --- /dev/null +++ b/changelog.d/shared-atomic-geography.added.md @@ -0,0 +1 @@ +Add shared graph operators for atomic-area assignment and geographic derivation. Country declarations choose observed constraints, stable draw identities, sampling stages and versioned mapping relations. Preserve observed locations, reject ambiguous support, and verify mapping consistency on cloned or pruned populations. Existing country builds retain their current geography paths pending native support integration. Normalize US block mappings, qualify observed ACS/ASEC geography against source receipts, and compose shared assignment before the combined-survey clone, with an inherited-mapping gate afterward. diff --git a/changelog.d/signed-income-reconciliation.added.md b/changelog.d/signed-income-reconciliation.added.md new file mode 100644 index 000000000..428496f2c --- /dev/null +++ b/changelog.d/signed-income-reconciliation.added.md @@ -0,0 +1,2 @@ +Add a deterministic weighted projection onto a signed total with explicit +nonnegative component bounds, adjustment diagnostics and floating-point checks. diff --git a/changelog.d/source-io-wheel-extra.fixed.md b/changelog.d/source-io-wheel-extra.fixed.md new file mode 100644 index 000000000..9e8232f55 --- /dev/null +++ b/changelog.d/source-io-wheel-extra.fixed.md @@ -0,0 +1 @@ +Declare an engine-free HDF source extra and install it in the isolated wheel test environment. Verify an invented pandas HDF round trip before source tests, while retaining the base wheel import boundary and excluding the US rules engine. diff --git a/changelog.d/store-verified-bytes.fixed.md b/changelog.d/store-verified-bytes.fixed.md new file mode 100644 index 000000000..d4215726c --- /dev/null +++ b/changelog.d/store-verified-bytes.fixed.md @@ -0,0 +1 @@ +Return the exact verified graph-store byte payload, preventing a file replacement between verification and return from supplying unchecked bytes. diff --git a/changelog.d/survey-final-observation-seal.fixed.md b/changelog.d/survey-final-observation-seal.fixed.md new file mode 100644 index 000000000..bfa186d97 --- /dev/null +++ b/changelog.d/survey-final-observation-seal.fixed.md @@ -0,0 +1 @@ +Recheck the complete detached survey population observations after final source and artifact I/O, so late mutations cannot escape through the values retained for downstream graph composition. diff --git a/changelog.d/survey-social-security-source.added.md b/changelog.d/survey-social-security-source.added.md new file mode 100644 index 000000000..11d76e2dd --- /dev/null +++ b/changelog.d/survey-social-security-source.added.md @@ -0,0 +1 @@ +Add current survey Social Security source qualification with explicit reporting units, question-universe unknownness, allocation literals and component mapping judgments. diff --git a/changelog.d/uk-atomic-household-lineage.added.md b/changelog.d/uk-atomic-household-lineage.added.md new file mode 100644 index 000000000..b40a2a605 --- /dev/null +++ b/changelog.d/uk-atomic-household-lineage.added.md @@ -0,0 +1 @@ +Add a pure UK household-lineage adapter that derives stable atomic-geography keys from explicit source identities and selection/expansion records, preserving exact IDs and refusing ambiguous ancestry without granting source authority or changing the full-build host. diff --git a/changelog.d/us-adapter-spm-context.fixed.md b/changelog.d/us-adapter-spm-context.fixed.md new file mode 100644 index 000000000..792a851b1 --- /dev/null +++ b/changelog.d/us-adapter-spm-context.fixed.md @@ -0,0 +1 @@ +Forward explicit SPM settings to the US adapter's metadata system as well as its simulations, preserving the selected measurement context and the behavior when settings are omitted. diff --git a/changelog.d/us-asec-retirement-annuity-niu.fixed.md b/changelog.d/us-asec-retirement-annuity-niu.fixed.md new file mode 100644 index 000000000..0ddba37c7 --- /dev/null +++ b/changelog.d/us-asec-retirement-annuity-niu.fixed.md @@ -0,0 +1 @@ +Match published annuity NIU literals to the authenticated money owner's normalized zero and NIU status while preserving raw retirement source values. Reject mismatched dollar-zero and NIU encodings before qualification. diff --git a/changelog.d/us-atomic-financial-composition.added.md b/changelog.d/us-atomic-financial-composition.added.md new file mode 100644 index 000000000..a28385f6c --- /dev/null +++ b/changelog.d/us-atomic-financial-composition.added.md @@ -0,0 +1 @@ +Compose survey atomic geography and current financial imputation in a checked twenty-node graph, retaining geography through cloning and financial attachment. Add native Delaware population-only source normalization and complete support readback controls. diff --git a/changelog.d/us-canonical-household-roles-review.fixed.md b/changelog.d/us-canonical-household-roles-review.fixed.md new file mode 100644 index 000000000..6405cf35f --- /dev/null +++ b/changelog.d/us-canonical-household-roles-review.fixed.md @@ -0,0 +1 @@ +Correct the household-role graph test's receipt key, add Boolean-storage and actual source-coordinate/roster refusal controls, and document operative source/artifact capacity limits. diff --git a/changelog.d/us-common-survey-housing.added.md b/changelog.d/us-common-survey-housing.added.md new file mode 100644 index 000000000..1b73ad67b --- /dev/null +++ b/changelog.d/us-common-survey-housing.added.md @@ -0,0 +1,5 @@ +Connect independent housing-survey observations to participation inputs in the +common ACS–ASEC enrichment graph. Preserve source knownness and modeled origin, +fit unresolved receipt at household grain with original design weights, and +route assistance to the reference person's SPM unit. Leave program awards and +SPM valuation with the country model. diff --git a/changelog.d/us-completion-a50-us-schema-fixture.fixed.md b/changelog.d/us-completion-a50-us-schema-fixture.fixed.md new file mode 100644 index 000000000..1a9272a90 --- /dev/null +++ b/changelog.d/us-completion-a50-us-schema-fixture.fixed.md @@ -0,0 +1 @@ +Construct the completion custody tests' ordinary parent with the shared full US schema, invented group sentinel columns and an explicit empty metadata context. Check emptiness through the mapping interface. Preserve the original generic fixture and private child-support frames, and verify that ordinary US routes reject undeclared private-support metadata. diff --git a/changelog.d/us-completion-support-custody.fixed.md b/changelog.d/us-completion-support-custody.fixed.md new file mode 100644 index 000000000..3921953ce --- /dev/null +++ b/changelog.d/us-completion-support-custody.fixed.md @@ -0,0 +1 @@ +Keep child donor/recipient support observations and attached manifest views under their declared private schema during combined survey completion. Preserve full Population custody, permanent refusal, and the strict US receiving/source boundary. diff --git a/changelog.d/us-current-acs-income-anchor-source.added.md b/changelog.d/us-current-acs-income-anchor-source.added.md new file mode 100644 index 000000000..d8d6fce32 --- /dev/null +++ b/changelog.d/us-current-acs-income-anchor-source.added.md @@ -0,0 +1,3 @@ +Qualify original ACS property and retirement income anchors through the retained +survey source owner, preserving literal missingness, allocation, age universe and +signed values while verifying raw identities and adjusted amount bits. diff --git a/changelog.d/us-current-asec-dividend-source.added.md b/changelog.d/us-current-asec-dividend-source.added.md new file mode 100644 index 000000000..a7d453233 --- /dev/null +++ b/changelog.d/us-current-asec-dividend-source.added.md @@ -0,0 +1 @@ +Qualify current ASEC dividend observations and survivor source routes from the retained survey owner, preserving ambiguous zeros, allocation-codebook conflicts and source flags without assigning tax treatment or selecting donors. diff --git a/changelog.d/us-current-asec-income-routing-source.added.md b/changelog.d/us-current-asec-income-routing-source.added.md new file mode 100644 index 000000000..b4d469ed8 --- /dev/null +++ b/changelog.d/us-current-asec-income-routing-source.added.md @@ -0,0 +1 @@ +Qualify the current ASEC pension/annuity, retirement distribution, net property, farm and other-income source families into a typed reporting/routing projection, keeping printed totals, receipt universes, account and category routing, and allocation provenance distinct from any modelled taxable, private or residual component. diff --git a/changelog.d/us-current-asec-property-basis.added.md b/changelog.d/us-current-asec-property-basis.added.md new file mode 100644 index 000000000..908dff17d --- /dev/null +++ b/changelog.d/us-current-asec-property-basis.added.md @@ -0,0 +1 @@ +Add a pure ASEC property donor basis with separate reported and component totals, explicit retirement-slot routing derivations, exact identity alignment and original-design-weight exclusion diagnostics. diff --git a/changelog.d/us-current-asec-retirement-basis.added.md b/changelog.d/us-current-asec-retirement-basis.added.md new file mode 100644 index 000000000..d7da7254e --- /dev/null +++ b/changelog.d/us-current-asec-retirement-basis.added.md @@ -0,0 +1,3 @@ +Add a pure ASEC retirement candidate ledger with explicit measurement assumptions, source routing and knownness, conservative accounting intervals, preserved unresolved survivor/distribution scope, and original DESIGN-weight support diagnostics. It produces no fiscal inputs or source authority. + +Require exact applicable distribution aggregate balance and readable off-route evidence before admitting candidate intervals; preserve account evidence and signed residuals separately from interval availability. diff --git a/changelog.d/us-current-asec-retirement-detail-source.added.md b/changelog.d/us-current-asec-retirement-detail-source.added.md new file mode 100644 index 000000000..799cbc4b2 --- /dev/null +++ b/changelog.d/us-current-asec-retirement-detail-source.added.md @@ -0,0 +1 @@ +Qualify current ASEC pension, disability and survivor source details with exact original-person joins, preserved unknownness and unallocated aggregate comparisons. New literal fields remain separate from retained money; no retirement regularity, taxability or ACS component mapping is inferred. diff --git a/changelog.d/us-current-property-income-sources.added.md b/changelog.d/us-current-property-income-sources.added.md new file mode 100644 index 000000000..ca64a9350 --- /dev/null +++ b/changelog.d/us-current-property-income-sources.added.md @@ -0,0 +1 @@ +Compose source-qualified original ASEC property donor and ACS adult-anchor recipient branches, preserving DESIGN weights, full origin axes, knownness diagnostics and final source-lifetime checks without fitting or attaching tax leaves. diff --git a/changelog.d/us-current-survey-health-coverage.added.md b/changelog.d/us-current-survey-health-coverage.added.md new file mode 100644 index 000000000..5443947b8 --- /dev/null +++ b/changelog.d/us-current-survey-health-coverage.added.md @@ -0,0 +1,4 @@ +Add a source-qualified health coverage graph fragment for the common US survey +frame and its PUF clones. Preserve literal source and allocation codes, explicit +unknowns, and incoming amount fields; keep broader ACS concepts as documented +gaps instead of manufacturing narrower coverage flags. diff --git a/changelog.d/us-current-survey-household-roles.added.md b/changelog.d/us-current-survey-household-roles.added.md new file mode 100644 index 000000000..0951f4950 --- /dev/null +++ b/changelog.d/us-current-survey-household-roles.added.md @@ -0,0 +1 @@ +Add a source-qualified household reference-person fragment for the combined ACS/ASEC graph, preserving group-quarters unknownness and refusing unsupported or conflicting canonical incumbents. Include exact-clone binding, private provenance, full-population replay verification and bounded source/graph test proposals; country-host and native execution remain separate. diff --git a/changelog.d/us-declared-fiscal-measurement.added.md b/changelog.d/us-declared-fiscal-measurement.added.md new file mode 100644 index 000000000..58d36e459 --- /dev/null +++ b/changelog.d/us-declared-fiscal-measurement.added.md @@ -0,0 +1 @@ +Add a candidate-only US graph measurement stage using declared target bindings, household-aligned sparse measures, explicit national/state/district support checks, and complete input-closure checks for optional model evaluation. diff --git a/changelog.d/us-fiscal-consumer-propagation.added.md b/changelog.d/us-fiscal-consumer-propagation.added.md new file mode 100644 index 000000000..0ee46bacb --- /dev/null +++ b/changelog.d/us-fiscal-consumer-propagation.added.md @@ -0,0 +1 @@ +Allow fiscal materialization, batched reform and source-amount helpers, and written-file diagnostics to receive explicit dataset, simulation and system constructors with copied SPM settings. Existing default constructors and county reform-validation settings remain unchanged; injected dependencies do not establish runtime approval or cache identity. diff --git a/changelog.d/us-fiscal-dense-calibration.added.md b/changelog.d/us-fiscal-dense-calibration.added.md new file mode 100644 index 000000000..497b5415f --- /dev/null +++ b/changelog.d/us-fiscal-dense-calibration.added.md @@ -0,0 +1 @@ +Add a numerical fiscal graph calibration stage using existing grouped Adam, explicit original weight and origin bounds, exact household alignment, and target and origin diagnostics without claiming source or release admission. diff --git a/changelog.d/us-fiscal-formula-metadata.added.md b/changelog.d/us-fiscal-formula-metadata.added.md new file mode 100644 index 000000000..94b1d5339 --- /dev/null +++ b/changelog.d/us-fiscal-formula-metadata.added.md @@ -0,0 +1 @@ +Allow fiscal materialization, reform and weight-attachment helpers to use explicitly supplied period-specific formula metadata while preserving the cached static metadata default. Explicit metadata is checked before target checkpoint reuse and does not establish consumer or release qualification. diff --git a/changelog.d/us-fiscal-leaf-policy.added.md b/changelog.d/us-fiscal-leaf-policy.added.md new file mode 100644 index 000000000..dee0da8d5 --- /dev/null +++ b/changelog.d/us-fiscal-leaf-policy.added.md @@ -0,0 +1 @@ +Add optional pinned fiscal input policies with strict producer defaults, complete-population collision checks and private-copy preparation. Literal assumptions remain non-executable pending checked parent integration; sampled traces cannot exempt inactive inputs. diff --git a/changelog.d/us-fiscal-target-snapshots.added.md b/changelog.d/us-fiscal-target-snapshots.added.md new file mode 100644 index 000000000..61514455b --- /dev/null +++ b/changelog.d/us-fiscal-target-snapshots.added.md @@ -0,0 +1 @@ +Allow host-owned target snapshots on the US dense fiscal calibration kernel while preserving result and cache identity. Align grouped snapshot epochs with completed optimizer updates and keep required cache replay silent. diff --git a/changelog.d/us-head-to-head-explicit-consumer.added.md b/changelog.d/us-head-to-head-explicit-consumer.added.md new file mode 100644 index 000000000..25b29849c --- /dev/null +++ b/changelog.d/us-head-to-head-explicit-consumer.added.md @@ -0,0 +1 @@ +Allow the US head-to-head scorer to use one explicit model consumer for historical H5 variable ownership, formula normalization, batched target calculations and the ephemeral by-origin battery. The existing default path remains available; explicit dependencies do not attest runtime qualification or authorize publication. diff --git a/changelog.d/us-head-to-head-shipped-weights.added.md b/changelog.d/us-head-to-head-shipped-weights.added.md new file mode 100644 index 000000000..c8df19e81 --- /dev/null +++ b/changelog.d/us-head-to-head-shipped-weights.added.md @@ -0,0 +1 @@ +Allow US head-to-head scorecards to compare each file's shipped weights unchanged. The explicit population-weight mode applies to both artifacts and appears in JSON and Markdown; the historical rescaling default and population-scale diagnostics remain available. diff --git a/changelog.d/us-household-role-import-contract.fixed.md b/changelog.d/us-household-role-import-contract.fixed.md new file mode 100644 index 000000000..c76791cb1 --- /dev/null +++ b/changelog.d/us-household-role-import-contract.fixed.md @@ -0,0 +1 @@ +Move the unchanged shared demographic requirements into an import-free contract, preserving historical composed-stage re-exports and source identities while preventing household-role qualification from eagerly loading the legacy composed runtime. Bind both shared declarations at existing source and graph live checks. diff --git a/changelog.d/us-launch-integration.fixed.md b/changelog.d/us-launch-integration.fixed.md new file mode 100644 index 000000000..170634a55 --- /dev/null +++ b/changelog.d/us-launch-integration.fixed.md @@ -0,0 +1 @@ +Preserve grouped calibration closing-state results alongside ordinary best-iterate selection, retain the combined US runtime facade and joint geography validation, and publish complete metadata for lazy graph snapshots. diff --git a/changelog.d/us-launch-store-grouped-validation.fixed.md b/changelog.d/us-launch-store-grouped-validation.fixed.md new file mode 100644 index 000000000..c341c76e2 --- /dev/null +++ b/changelog.d/us-launch-store-grouped-validation.fixed.md @@ -0,0 +1 @@ +Reject nonfinite stored JSON as a store-corruption error and normalize the deprecated calibration method alias before validating grouped constraints. diff --git a/changelog.d/us-national-atomic-support.added.md b/changelog.d/us-national-atomic-support.added.md new file mode 100644 index 000000000..085c1627a --- /dev/null +++ b/changelog.d/us-national-atomic-support.added.md @@ -0,0 +1,4 @@ +Add an opt-in national Census population-only source control covering the fifty +states and DC, with independent block/state reconciliation, complete CD119 and +PUMA joins, and full support serialization/readback. Record the accepted native +normalization separately from household assignment and dataset release checks. diff --git a/changelog.d/us-native-child-support.added.md b/changelog.d/us-native-child-support.added.md new file mode 100644 index 000000000..d6acf3261 --- /dev/null +++ b/changelog.d/us-native-child-support.added.md @@ -0,0 +1 @@ +Add an opt-in native child-support amount group using full-original qualified ASEC receipt donors and one zero-aware ACS-original draw shared by clones. Keep paid support observed-only, retain unresolved source statuses, and explicitly replace only carried canonical child-support columns with lossless supported storage. diff --git a/changelog.d/us-native-other-disability-source.added.md b/changelog.d/us-native-other-disability-source.added.md new file mode 100644 index 000000000..fe74af615 --- /dev/null +++ b/changelog.d/us-native-other-disability-source.added.md @@ -0,0 +1 @@ +Add a qualified ASEC other-disability source adapter that reuses the retired two-slot, non-workers-compensation arithmetic over the existing retirement-detail owner, keeps NIU, missing, under-15 and contradictory slots unknown instead of zero, retains conditional allocation provenance, and copies each original row to its clones without a redraw. diff --git a/changelog.d/us-person-status-graph.added.md b/changelog.d/us-person-status-graph.added.md new file mode 100644 index 000000000..615666a24 --- /dev/null +++ b/changelog.d/us-person-status-graph.added.md @@ -0,0 +1 @@ +Add an optional four-node graph fragment exposing qualified descriptive ACS/ASEC person status after full cloning and atomic geography, with exact clone binding and complete receiving-population verification. diff --git a/changelog.d/us-policyengine-h5-logical-readback.added.md b/changelog.d/us-policyengine-h5-logical-readback.added.md new file mode 100644 index 000000000..41c8200ad --- /dev/null +++ b/changelog.d/us-policyengine-h5-logical-readback.added.md @@ -0,0 +1,6 @@ +Add an opt-in PolicyEngine-US H5 export wrapper that compares persisted values, +integer IDs and memberships, row order, household weights and period with a +supplied retained Frame and common-parent contract. It preserves adapter defaults +and admits only complete SPM roles, explicit scope and complete-input codec +behavior; the receipt does not establish +source ancestry, calibration validity or release eligibility. diff --git a/changelog.d/us-policyengine-h5-string-storage.fixed.md b/changelog.d/us-policyengine-h5-string-storage.fixed.md new file mode 100644 index 000000000..47390b30e --- /dev/null +++ b/changelog.d/us-policyengine-h5-string-storage.fixed.md @@ -0,0 +1,3 @@ +Accept logically identical complete Arrow-backed strings during PolicyEngine H5 +readback and report storage normalization without changing source admission or +the shared HDF codec. diff --git a/changelog.d/us-population-seal-streaming.changed.md b/changelog.d/us-population-seal-streaming.changed.md new file mode 100644 index 000000000..60002223e --- /dev/null +++ b/changelog.d/us-population-seal-streaming.changed.md @@ -0,0 +1 @@ +Stream the US full-PUF placement's in-process population seal into its digest instead of expanding every part into one tuple first, preserving each part, its order, its length prefix and its bytes exactly. diff --git a/changelog.d/us-property-income-graph.added.md b/changelog.d/us-property-income-graph.added.md new file mode 100644 index 000000000..ac92df734 --- /dev/null +++ b/changelog.d/us-property-income-graph.added.md @@ -0,0 +1 @@ +Add a source-blind property-income model graph with original-design QRF fitting, joint draw columns, signed reconciliation and explicit diagnostics. diff --git a/changelog.d/us-property-tax-leaves.added.md b/changelog.d/us-property-tax-leaves.added.md new file mode 100644 index 000000000..fe4905618 --- /dev/null +++ b/changelog.d/us-property-tax-leaves.added.md @@ -0,0 +1,6 @@ +Add a deterministic three-node property tax rebase with explicit receiving +version, separate interest/dividend knownness, maintained fraction splits, +subtraction complements and typed numerical completeness evidence. Preserve +unknowns and retirement-account earnings; refuse receiving shapes that the +current FILTER operation cannot preserve. Country and PUF host integration +remains separate. diff --git a/changelog.d/us-puf55-canonical-and-output-seals.added.md b/changelog.d/us-puf55-canonical-and-output-seals.added.md new file mode 100644 index 000000000..0e6679803 --- /dev/null +++ b/changelog.d/us-puf55-canonical-and-output-seals.added.md @@ -0,0 +1,4 @@ +Add a source-bound PUF55 canonical donor graph node with cold/replay controls, +stable live-code identity checks, and finalized-candidate seals for the shared +two-route numerical finalizer. Complete native survey attachment and release +validation remain separate build steps. diff --git a/changelog.d/us-puf55-qualified-fixed-inputs.added.md b/changelog.d/us-puf55-qualified-fixed-inputs.added.md new file mode 100644 index 000000000..f152c9f43 --- /dev/null +++ b/changelog.d/us-puf55-qualified-fixed-inputs.added.md @@ -0,0 +1 @@ +Add explicit original-arm PUF55 recipient projections and opt-in, source-qualified fixed-input graph artifacts. Preserve current clone-one defaults and expose provisional ASEC mapping assumptions without claiming observed taxability or release eligibility. diff --git a/changelog.d/us-puf55-survey-ss-measurement.added.md b/changelog.d/us-puf55-survey-ss-measurement.added.md new file mode 100644 index 000000000..ebe159455 --- /dev/null +++ b/changelog.d/us-puf55-survey-ss-measurement.added.md @@ -0,0 +1,5 @@ +Add an authenticated filer/joint-spouse Social Security report-sum measurement +for PUF55 conditioning, with an explicit eight-predictor fallback for incomplete +reports and preservation of all survey Social Security cells and masks. Record +31 passing invented controls and the remaining recipient integration and +measurement limitations. diff --git a/changelog.d/us-reviewed-dependency-classification.fixed.md b/changelog.d/us-reviewed-dependency-classification.fixed.md new file mode 100644 index 000000000..1f7fe4d55 --- /dev/null +++ b/changelog.d/us-reviewed-dependency-classification.fixed.md @@ -0,0 +1 @@ +Adopt the reviewed exact-buffer store read and optional consumer-metadata adapter dependency contracts so US implementation manifests validate their current source. Retain scoped dependency tripwires and invalidate affected implementation identities rather than reusing earlier cache keys. diff --git a/changelog.d/us-source-string-seal-efficiency.changed.md b/changelog.d/us-source-string-seal-efficiency.changed.md new file mode 100644 index 000000000..a0c33268e --- /dev/null +++ b/changelog.d/us-source-string-seal-efficiency.changed.md @@ -0,0 +1 @@ +Reuse bounded, column-local encodings for repeated source strings during ASEC frame hashing while preserving the exact digest byte stream and checking all values on every validation. diff --git a/changelog.d/us-structural-id-precision.fixed.md b/changelog.d/us-structural-id-precision.fixed.md new file mode 100644 index 000000000..7422f618c --- /dev/null +++ b/changelog.d/us-structural-id-precision.fixed.md @@ -0,0 +1,3 @@ +Require exact integer identity for entity IDs and memberships even when the +selected PolicyEngine variable stores floats. Metadata validation rejects lossy +float32/float64 conversions without changing source IDs or ordinary amount rounding. diff --git a/changelog.d/us-survey-catalogue-digest.changed.md b/changelog.d/us-survey-catalogue-digest.changed.md new file mode 100644 index 000000000..96d1c8d13 --- /dev/null +++ b/changelog.d/us-survey-catalogue-digest.changed.md @@ -0,0 +1 @@ +Stream eligible ACS catalogue seals through bounded record JSON encoding while preserving canonical digest bytes, full-value fallback, and repeated source and owner validation. Bind the additional JSON provider calls into the existing preparation authority checks. diff --git a/changelog.d/us-survey-completion-host.added.md b/changelog.d/us-survey-completion-host.added.md new file mode 100644 index 000000000..ef1e61661 --- /dev/null +++ b/changelog.d/us-survey-completion-host.added.md @@ -0,0 +1,4 @@ +Add an explicit optional US child-property completion host with qualified +household roles, retained issued-parent custody, a separate completion version, +and child-verification-bound tax rebasing. Source implementation and acceptance +tests are authored; combined-host runtime acceptance remains unrun. diff --git a/changelog.d/us-survey-completion-seams.added.md b/changelog.d/us-survey-completion-seams.added.md new file mode 100644 index 000000000..9ea3f64a7 --- /dev/null +++ b/changelog.d/us-survey-completion-seams.added.md @@ -0,0 +1 @@ +Add a dedicated all-row survey completion receiving operation and an optional typed child-verification dependency for property tax rebasing. Default declarations and split calculations remain unchanged; source and complete-parent authority stay with the country host. diff --git a/changelog.d/us-survey-copy-and-identity.fixed.md b/changelog.d/us-survey-copy-and-identity.fixed.md new file mode 100644 index 000000000..76d9520fa --- /dev/null +++ b/changelog.d/us-survey-copy-and-identity.fixed.md @@ -0,0 +1 @@ +Keep frame metadata copying and pickling from changing producer identity on Python 3.14. Refresh ACS source pins after verified formatting-only changes, and reduce per-cell survey identity encoding overhead while preserving canonical bytes, mutation checks, and the current-wage projection. diff --git a/changelog.d/us-survey-current-amount-enrichment.added.md b/changelog.d/us-survey-current-amount-enrichment.added.md new file mode 100644 index 000000000..dab37b769 --- /dev/null +++ b/changelog.d/us-survey-current-amount-enrichment.added.md @@ -0,0 +1 @@ +Add a checked US survey enrichment host that preserves the PUF population while attaching source-qualified unemployment, medical costs and current health coverage. Keep reporting unknowns, original observations and source-to-clone provenance explicit. diff --git a/changelog.d/us-survey-diagnostic-options.fixed.md b/changelog.d/us-survey-diagnostic-options.fixed.md new file mode 100644 index 000000000..e8668b7e0 --- /dev/null +++ b/changelog.d/us-survey-diagnostic-options.fixed.md @@ -0,0 +1 @@ +Reconstruct the grouped calibration solver's closing-state selection options during survey diagnostic verification, preserving exact diagnostic comparison and late-mutation checks. diff --git a/changelog.d/us-survey-interest-conservation.fixed.md b/changelog.d/us-survey-interest-conservation.fixed.md new file mode 100644 index 000000000..b58d850b5 --- /dev/null +++ b/changelog.d/us-survey-interest-conservation.fixed.md @@ -0,0 +1 @@ +Retain the tax-exempt remainder when the survey financial graph splits observed ASEC or modeled ACS interest totals, preserving both parts on the original survey support before PUF enrichment. diff --git a/changelog.d/us-survey-person-status-source.added.md b/changelog.d/us-survey-person-status-source.added.md new file mode 100644 index 000000000..ffb592c55 --- /dev/null +++ b/changelog.d/us-survey-person-status-source.added.md @@ -0,0 +1 @@ +Add a retained-source qualifier for descriptive ACS/ASEC difficulty and enrollment observations, preserving literal codes, item universes, edit provenance and survey periods while leaving statutory and annual eligibility inputs unresolved. diff --git a/changelog.d/us-whole-series-selection-sites.changed.md b/changelog.d/us-whole-series-selection-sites.changed.md new file mode 100644 index 000000000..e14368e80 --- /dev/null +++ b/changelog.d/us-whole-series-selection-sites.changed.md @@ -0,0 +1 @@ +Take the whole series by slice at the remaining US in-process seal sites (atomic geography, input coverage, origin budget, current survey predictors and PUF55 recipients), preserving each seal's emitted bytes exactly. diff --git a/changelog.d/weights-copy-protocol.fixed.md b/changelog.d/weights-copy-protocol.fixed.md new file mode 100644 index 000000000..4a29fdf93 --- /dev/null +++ b/changelog.d/weights-copy-protocol.fixed.md @@ -0,0 +1 @@ +Reconstruct base `Weights` through validation during shallow copy, deep copy and pickle round trips, preserving exact values and read-only storage without changing class metadata. Preserve existing subclass copy protocols. diff --git a/changelog.d/worker-identity-current-lock.fixed.md b/changelog.d/worker-identity-current-lock.fixed.md new file mode 100644 index 000000000..66e497e13 --- /dev/null +++ b/changelog.d/worker-identity-current-lock.fixed.md @@ -0,0 +1 @@ +Update the primary QRF worker's approved lock identity for the reviewed source-I/O packaging metadata change while preserving strict rejection of other locks and the explicit legacy campaign boundary. diff --git a/docs/acs-spm-source-assembly.md b/docs/acs-spm-source-assembly.md new file mode 100644 index 000000000..d883d373f --- /dev/null +++ b/docs/acs-spm-source-assembly.md @@ -0,0 +1,179 @@ +# ACS SPM construction before source mapping + +`microcosm.build.acs_spm_source_assembly` supplies an explicit development +partition for an ACS source Frame whose SPM table contains only `spm_unit_id`. +It preserves tax units, all other memberships and tables, household DESIGN +weights, strata and the mass log. It does not regroup amounts or run a country +model. + +```python +from microcosm.build.acs_spm_partition import ACS_SPM_DEVELOPMENT_POLICY +from microcosm.build.acs_spm_source_assembly import AcsSpmSourceAssemblyOptions +from microcosm.build.us_runtime.acs_pums import build_acs_pums_unit_frame + +frame, metadata = build_acs_pums_unit_frame( + source, + spm_construction=AcsSpmSourceAssemblyOptions( + policy=ACS_SPM_DEVELOPMENT_POLICY, + minor_partner_role=True, + ), +) +``` + +The default `spm_construction=None` retains the existing constructor path. An +explicit request checks canonical assembler capability before opening source +archives and raises `UnsupportedAssembler` if unavailable or incompatible. The +locked calculator 1.0.0 lacks the required behavior; the passing synthetic test +environment uses exact calculator PR45 source +`bcf45768003bb79addfafb0e6d9c7d2d5e547d9d`, with `units.py` SHA256 +`ce0d328d856ca81862b4e80947b6f6269a89bbd842cbdbb1569411b319da5a33`. +This change does not adopt a dependency or change a lockfile. + +## Registry and evidence + +The pure `assemble_acs_spm_source` function returns a Frame, partition projection, +numeric identity registry, crosswalk, unit evidence and aggregate receipt. Its +required `id_ceiling` must be an exact nonnegative int64 integer, excluding +Booleans. The ACS constructor supplies the existing reviewed clone-safe ceiling. +Old IDs survive unchanged member sets; split components receive distinct IDs +above the selected old-ID maximum, ordered by native person keys. Overflow is an +error. + +Construct one registry from the whole selected roster. Pass the same registry +when applying the adapter to complete-household subsets. The registry binds +native `(SERIALNO, SPORDER)` keys, the supplied person and household identities, +raw relationship/age/marital/universe evidence, household counts, construction +policy and actual assembler hash. It rejects changed source records, incomplete +households, partition changes and identity collisions. Row order may differ; +identities do not become stable across different selection domains. These are +generated component identities, not published Census SPM identifiers. + +The adapter temporarily joins household TYPEHUGQ and normalizes SPORDER for the +partition helper. It does not change the original stored tokens. Uncertainty +in secondary relationships remains visible in every affected household's unit +evidence, including a unit containing an observed reference person. The returned +nullable roles retain their original rule/source labels. Relationship allocation +provenance remains unresolved. No engine `is_spm_independent_minor_role` or annual +`spm_unit_spm_universe_status` is inferred or written. + +The source constructor retains the aggregate receipt in its metadata and the +Frame's immutable metadata. The pure adapter returns the separate rowwise +projection for callers that need it; the constructor does not publish those rows +in its aggregate receipt. The receipt binds supplied tables, not authenticated +source files, and does not grant source issuance, consumer or release authority. + +## Fresh SPM inputs and captured native construction + +The hook runs after the one tax-unit construction and household source attachment, +before `map_acs_native_inputs`. Existing tenure mapping follows the new SPM +membership. Existing housing-participation routing awards only the actual +household head's SPM unit. Neither copies an old SPM monetary table. Any existing +SPM amount, flag or provenance column makes this adapter refuse; the dense-parent +regroup helper is a separate workflow. + +The captured housing and native coverage owners accept an explicit +`spm_construction` option. They invoke the evidence-returning constructor against +their private, authenticated archive captures. The public constructor retains its +two-value return contract. Native owners retain canonical bytes produced from +the actual membership, links, partition crosswalk, regrouping summary, registry, +native crosswalk and unit uncertainty tables, including declared column rosters +and dtypes. They bind those bytes to the captured archive identities, options, +assembler and implementation identities. A 256 MiB limit bounds the development +evidence payload after serialization; it is not an allocation preflight or a +full-source memory qualification. + +After capturing the actual evidence, the housing owner removes exactly the +`acs_spm_source_assembly` aggregate key from its mapped graph Frame and records +that transport in its receipt. All other metadata stays intact. The aggregate +receipt remains in the owned evidence, and direct constructor callers retain it +in Frame metadata. It becomes neither graph-context operator authority nor an +engine role. Native verification checks the retained bytes; the preparation's +nested seals also retain them. Missing or changed evidence refuses even though +the mapped Frame no longer carries that aggregate key. Defensive decoded views +are values, not new detached authority. + +A version 1 `selection-request.json` keeps its exact closed shape and imports no +optional assembler. Version 2 requires the additional closed field: + +```json +{ + "acs_spm_construction": { + "policy": "acs_spm_development_reconstruction_v1", + "minor_partner_role": true + } +} +``` + +Use the exported `ACS_SPM_DEVELOPMENT_POLICY` value for the policy field and +`microcosm.us.survey-population-request.v2` for the protocol, retaining the existing +declaration, fraction and seed fields. No mode is enabled by default. The request +bytes bind the construction option into source and graph identities. Existing +survey callers read that request; financial/completion and retention host APIs +are unchanged. Source preparation checks capability before opening population +archives. The native owner additionally requires the exact reviewed PR45 defining +file and verifies the loaded callable during native verification. No global +package dependency or lockfile changes. + +`AuthenticatedSurveyPopulationPreparation.acs_spm_construction_evidence` returns +a defensive view of the checked native records. Its IDs belong to the whole +selected native roster. The existing per-entity `receipt["origins"]` map relates +native person, household and SPM IDs to assembled IDs; cloning remains a later, +separate operation. No rowwise roles or universe status are inferred into model +columns. + +The isolated branch updates the accepted parser/housing bytes and reviewed +helper inventory. These revisions change source/implementation identities even +for default requests; old receipts are not claimed byte-identical. The earlier +first slice at `fec3f9033ab47b752b2d5f8890507bcf06ad3428` intentionally left native +admission blocked. Its separate review and test receipts remain historical +source-only evidence. Frozen native performance and retention checkouts are not +changed by this continuation. + +## Invented validation + +At the first-slice source revision `fec3f9033ab47b752b2d5f8890507bcf06ad3428`, +the focused five-file battery passed 199 tests on exact PR45. Absent-assembler +and installed-1.0.0 environments each passed 151 tests with 48 strict expected +failures restricted to `UnsupportedAssembler`; neither had skips or unexpected +failures. The new assembly file contributes 42 passing PR45 cases, or 23 passing +and 19 expected unsupported cases. The imported partition/probe tests and existing +ACS parser/housing-participation tests retain their assertions. + +Tests cover tax/non-SPM conservation, role sensitivity without membership drift, +GQ preservation, source refusals, ID overflow/collisions, native-key binding, +shuffled and complete-household chunk application, exact large integer IDs, +and real constructor/tenure/head-only participation functions. Guards recorded +zero country imports, network requests and population-file reads. All source +records are invented fixtures. Earlier unsuccessful attempts remain in the local +evidence; this battery does not replace historical PR962 helper-matrix or actual +512A reports. No actual population, 512B, full-source performance, calibration or +release qualification follows from these tests. + +## Native request and issuer validation + +The continuation passes 218 focused adapter/native tests on exact PR45, plus +95 existing ACS literal/native coverage regressions. The 19 new cases cover +actual invented archive issuance, exact options and capability refusal before +source access, the default path without importing the assembler, actual retained +rows and native-to-assembled identity maps, non-SPM/weight conservation, missing +or changed evidence, copied-token refusal, source mutation, candidate replay, +and cold/required graph replay. Changing the explicit option changes the graph +key and refuses reuse of the old required result. The existing `return_values` +path retains the checked preparation and its defensive evidence accessor. + +Absent-assembler and installed-1.0.0 environments each pass 163 tests with +55 strict expected failures restricted to `UnsupportedAssembler`. Of those, +the new native file contributes 12 passing refusal/default cases and seven +unsupported cases. No test is skipped; unexpected exceptions remain failures. +Final completed runs record zero country-import, network and unapproved-file +read guard events. All archives, HDF fixtures and graph populations in these +tests are invented. The implementation and caller tests do not initialize a +country model. + +Earlier failing fixture/assertion attempts, the graph metadata-boundary refusal +and a combined synthetic run stopped by its initial 120-second CPU limit remain +in the local evidence. The final focused runner uses a 600-second CPU ceiling +and segregated temporary fixture directories. These are development test +receipts, not source-scale memory or runtime measurements. No actual population +application, country512B, engine role delivery, annual-universe qualification, +dependency adoption or release claim follows from this continuation. diff --git a/docs/asec-usual-hours-observations.md b/docs/asec-usual-hours-observations.md new file mode 100644 index 000000000..fed00661d --- /dev/null +++ b/docs/asec-usual-hours-observations.md @@ -0,0 +1,65 @@ +# ASEC usual-hours observations + +`current_asec_usual_hours.recode_asec_usual_hours` interprets original March 2025 +work-history literals across published ages 0-85. It returns source observations +without source admission, imputation, weights, graph execution or Frame mutation. + +The [2025 Census person dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +defines `HRSWK` on printed page 6C-17 (PDF page 38) as usual weekly hours during +weeks worked. Its universe is positive `WKSWORK`. Zero is NIU, and 99 represents +99 hours or more. Printed page 6C-20 (PDF page 41) distinguishes initial `WORKYN`, +the temporary/part-time follow-up `WTEMP`, and final recode `WRK_CK`. The work +questions cover people aged 15 and over; the follow-up applies to initial no +answers. Both pages were visually checked. The downloaded PDF SHA-256 is +`5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. + +These definitions support three distinct outcomes: + +| Source evidence | Result | +| --- | --- | +| Positive hours and weeks, coherent final work history | Preserve source hours and raw top code | +| Age 15+, zero hours/weeks and coherent final nonwork | Zero-hours completion labeled as source nonwork | +| Under 15 with NIU history, or adult with entirely NIU history | Unresolved hours with distinct universe/unanswered labels | + +The recoder rejects contradictory histories and invalid codes. An initial no +answer alone does not establish final nonwork. A NIU follow-up alone is not an +explicit no. Shared history and allocation-code checks keep the separate age-15 +donor recoder aligned without disguising an older person as a teenager. + +Original allocation flags and supplement response status remain separate from +the result's provenance. The [Census variable metadata](https://api.census.gov/data/2025/cps/asec/mar/variables/FL_665.json) +includes complete supplement nonresponse as `FL_665=0`; accepting the code does +not mean a respondent supplied the value. A zero person weight does not erase a +source observation. Positive weight remains required by the separate empirical +donor qualification routine. + +The all-NIU adult outcome is a conservative missing-information classification, +not a claim that NIU is a complete valid adult answer. This behavior must be +audited against the full source before graph integration. No earnings value, +last-week hours or historical engine default fills a gap here. In particular, +under-15 zero imputation needs a separately explicit assumption and earnings +conflict checks. This routine is not the all-person release input producer. + +`propose_asec_usual_hours` adds a separate, explicit completion operation. It +requires the original wage, nonfarm self-employment and farm self-employment +literals. It preserves observed adult hours independently of earnings. For an +under-15 person with NIU work history, it produces zero only when the caller +selects `us_hours_under15_zero_completion_v1` and all three earnings fields are +known zero. Nonzero earnings, missing columns or unknown earnings refuse that +completion. An unanswered adult remains unresolved and cannot produce a complete +proposal. The child's result stays labeled `under15_explicit_modeled_zero`. + +An aggregate diagnostic of the complete current ASEC person member found +142,125 records: 72,460 with positive source hours, 41,986 with source nonwork, +and 27,679 under-15 records with NIU history. No work-history contradiction or +zero/missing-hours record with nonzero earnings appeared in those three fields. +The diagnostic source hash matches the maintained 2025 member pin: +`06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5`. +These are unweighted source counts, not population estimates. A source scan does +not issue source authority or qualify a release. + +The version-2 retained source owner binds this observation/completion code and +its protocols, preserves raw earnings alongside work history, and returns a +complete selected-original-person hours projection for both survey arms. +Transport to clones, graph integration and all-person engine input qualification +remain separate. Public release flags remain false. diff --git a/docs/calibration-target-snapshots.md b/docs/calibration-target-snapshots.md new file mode 100644 index 000000000..b48fd32a9 --- /dev/null +++ b/docs/calibration-target-snapshots.md @@ -0,0 +1,155 @@ +# Calibration target snapshots + +An opt-in observer exposes target estimates already computed during an Adam +solve. It also records the estimates on the weights actually returned. Use it +to inspect fit over time without adding another model evaluation or changing +the optimization steps. + +```python +from pathlib import Path + +from microcosm.calibrate import ( + TargetSnapshotCadence, + TargetSnapshotObserver, + TargetSnapshotWriter, + calibrate, +) + +observer = TargetSnapshotObserver( + sink=TargetSnapshotWriter(Path("runs/example/targets"), history_limit=256), + run_id="example", + cadence=TargetSnapshotCadence(every=25), +) +result = calibrate(frame, targets, method="adam", target_snapshots=observer) +``` + +The caller supplies its existing frame and targets. No observer is enabled by +default. The example cadence is a caller choice, not a measured country-build +default. A sink exception propagates, matching the existing progress callback. + +Each snapshot has an ordered target identity, target values, achieved estimates, +relative errors, sequence, phase and iterate labels. Duplicate target names +remain distinguishable by ordered position. Honest nonfinite intermediate +estimates are represented by nulls and counts. Intermediate float32 estimates +and final float64 estimates are labeled with their actual iterate semantics. +Budget probes and subsequent selection/refit phases share one emission sequence. +`epoch` counts completed optimizer updates: zero is the starting state. An +in-loop estimate precedes that iteration's update; its selected counterpart uses +the same convention. Cadence counts loss evaluations starting at one, so an +every-25 cadence can emit epoch 24. The selected budget-search snapshot retains +the winning probe's identity, even when another probe ran afterward. If computing +a relative error overflows, the error is null and `non_finite_rows` counts it; +finite operands remain visible. + +Metadata accepts bounded flat scalar mappings and checked string identifiers; +it does not accept nested payloads or record vectors. `best_retained` must be a +complete `{available, epoch, loss}` mapping in a serialized snapshot. The emitter +can fill that mapping when no best iterate exists; the public validator rejects +a serialized null in its place. Each delivered payload is detached from the +caller's metadata and subsequent snapshots. + +The local writer atomically replaces `latest.json`. History chunks become +visible only after their complete bytes are written and synced, and an existing +chunk cannot be overwritten. Retention and dropped-history counts are explicit. +Use a fresh run directory unless intentionally adopting existing history. +Atomic history publication requires a filesystem that supports hard links and +directory synchronization. Unsupported filesystem errors propagate to the caller; +the writer does not expose partially written history as a fallback. + +## The grouped US solver + +The grouped/fixed-zero Adam path (`grouped_upper_bounds`, optionally +`grouped_preserve_zeros`) is instrumented at the same two seams and with the +same guarantees: the in-loop snapshot reads the exact float32 estimate tensor +the epoch's loss was computed from, and the closing `selected` snapshot reads +the same float64 `problem.estimates` the final diagnostics use. Enabling the +observer on a grouped run adds no matrix evaluation, no model evaluation and no +RNG advance, and returns bit-identical weights and trajectory. + +Two grouped specifics a consumer must read correctly: + +- Grouped Adam is a **closing-state** algorithm. It never runs the retain-best + rule, so every grouped snapshot carries + `best_retained = {"available": false, "epoch": null, "loss": null}` and the + closing snapshot is stamped `epoch == epochs`. That is not the same as an + ordinary run whose retain-best rule was switched off, so grouped snapshots + also carry bounded `selection` labels — `rule: "closing_state"`, + `constraint_mode: "grouped_upper_bounds"` and `grouped_preserve_zeros` — + which mirror the run's `options["iterate_selection"]`. +- In-loop snapshots are **pre-update loss evaluations**, the same convention the + ungrouped Adam loop uses. They are deliberately not the post-update projected + accepted vector that the private `_post_projection_observer` proof seam + reports after the next completed update: those are different vectors, and + producing target totals for the accepted one would require an extra matrix evaluation. + That private seam carries record-length weights, household IDs, the group map + and the absolute-bound vector; none of it reaches a snapshot, and the public + codec rejects record-level key names outright. + +Grouped runs keep every existing guard: fixed zeros stay in full-population +coordinates, the accepted-weight byte equality and ordered-household-ID checks +still run *after* the closing snapshot sink, and the unsupported grouped modes +(scalar cap, conserved mass, prox, L0/exact-k, gate initialization) are still +refused before the optimizer is constructed. Snapshot support does not turn any +of them into a permitted numeric path. + +## US fiscal host + +The actual dense fiscal kernel accepts the shared observer through an optional, +keyword-only constructor argument: + +```python +from microcosm.build.us_runtime.graph_fiscal_dense_calibration import ( + FiscalDenseCalibrationKernel, +) +from microcosm.calibrate import TargetSnapshotCadence, TargetSnapshotObserver + +observer = TargetSnapshotObserver( + sink=host_owned_sink, + run_id="fiscal-run", + candidate_id="candidate-under-review", + cadence=TargetSnapshotCadence(every=25), +) +kernels.register(FiscalDenseCalibrationKernel(target_snapshots=observer)) +``` + +The host owns the sink and any local writer directory. The kernel retains the +observer only on its instance and forwards it to the existing `calibrate` call. +Omitting the argument or passing `None` disables observation. Observer values, +callbacks and paths are absent from node parameters, implementation identity, +cache keys, graph artifacts and receipts. Changing this source implementation +changes its fingerprint normally; changing observer configuration does not. + +The solver binds snapshot identity to the actual compiled target order and +values from the fiscal measurement. The existing final matrix/target, household +ID, weight and measurement checks run after the sink completes. Sink exceptions +propagate, as with the ordinary progress callback. Each delivered dictionary is +detached: mutating its aggregate metadata or target rows cannot change the next +payload or optimizer result. This does not make an arbitrary callback safe to +mutate unrelated live application state. + +A required graph cache hit does not execute the optimizer and emits no target +snapshots, even when a new observer is supplied. Hosts should retain the previous +run's diagnostics and history with their identities; a replay must not be shown +as a new optimizer run. Complete-population and source admission remain the +country host's responsibility. + +This implements the solver, local storage and dense fiscal-host portions of +[issue 908](https://github.com/PolicyEngine/microcosm/issues/908). Staging upload, +a dashboard consumer and native cadence/performance acceptance remain separate +work. See [the fiscal-host integration evidence](../experiments/fiscal-target-snapshot-host-20260912.md). + +The reviewed repair branch passed 307 calibration tests before the final strict +null correction. All 52 snapshot tests then passed after that one-line correction, +including passive solver parity tests. Independent review closed the metadata, +detachment, partial-publication and codec findings. See the +[invented benchmark evidence](../experiments/908-target-snapshot-bench-receipts.md) +for overhead measurements; those are not native US or UK benchmarks. + +Fable's later PR review identified a mixed epoch convention, a missing winning +budget-probe label and a relative-error overflow case. Four new counterexamples +failed first; all 56 snapshot tests and all 311 calibration tests then passed +after those corrections. The 26 affected build identity and graph parity checks +also passed after recalculating the calibration source pins; fit and simulation +pins were preserved. These changes affect diagnostics only. The public writer +still validates each incoming payload because callers can supply serialized +snapshots independently of an observer. diff --git a/docs/categorical-probability-artifacts.md b/docs/categorical-probability-artifacts.md new file mode 100644 index 000000000..7e660e480 --- /dev/null +++ b/docs/categorical-probability-artifacts.md @@ -0,0 +1,48 @@ +# Categorical probability artifacts + +`microcosm.fit.categorical.fit_categorical` fits a real scikit-learn +`HistGradientBoostingClassifier` with directly supplied typed DESIGN weights. +It does not resample donors. Every declared class must have strictly positive +finite weighted support; missing or extra labels refuse. At least two classes +are required because the supported estimator's binary probability interface +does not provide a one-column single-class result. There is no invented mass +or fallback. The caller owns donor selection, scientific encoding and source +qualification. + +Inputs use a unique ordered int64 entity index and finite float64 predictors. +The label Series must have the same index. `CategoricalConfig` exposes the +supported iteration, learning-rate, tree-size, regularization and bin controls. +All constructor parameters are serialized, including the seed, disabled early +stopping, absent class weighting and absent categorical encoding. Probabilities +are taken from the actual fitted estimator and reordered by its `classes_` into +the declared class order. Application neither fits recipients nor draws random +values. + +The immutable fitted artifact binds ordered feature/class schemas, exact donor +matrix, labels, DESIGN weights, source projection hash, configuration, seed, +implementation and runtime identity. It contains the actual serialized fitted +estimator. Its loader is explicitly for trusted local producers/content stores: +the versioned envelope and expected digest detect corruption, but cannot make +an untrusted pickle safe. No fitted mutable object is returned to callers. + +`graph_categorical.categorical_fit_node` consumes a donor Slice and typed source +projection. `categorical_probability_node` consumes its model/metadata pair, +the maintained typed recipient matrix, and a separate recipient source +projection. The nodes return artifacts only. Exact graph declarations, +producer-key pairing, platform, content hashes and training metadata are checked +before prediction. `read_probabilities` checks the caller's expected bindings, +recipient IDs/order, class order and probability values. Those descriptive +checks cannot grant survey-source authority. + +This adapter changes no QRF behavior and attaches no Social Security amounts. +A future SS host must use the authenticated full-original source basis, +explicitly approved predictors and imported component order, then apply the +reason mask and conservation operator separately. The interpretation of +conditional category probabilities as shares, source-window transport, +beneficiary ownership and real-data statistical acceptance remain separate +development/release decisions. + +The tiny invented tests exercise the real weighted estimator, typed graph +cold/required replay, class order, dependency invalidation and refusals. They +do not establish calibration, source admission, a full host replay or release +acceptance. diff --git a/docs/child-property-income-graph.md b/docs/child-property-income-graph.md new file mode 100644 index 000000000..4d08d945b --- /dev/null +++ b/docs/child-property-income-graph.md @@ -0,0 +1,98 @@ +# Child property-income graph candidate + +This source-only fragment connects the source-qualified ASEC age 15–17 donor +projection to the real paired empirical model and original-coordinate draws. +It depends on separately reviewed numerical, source-qualifier and generic graph +adapter revisions. Its tests are authored and unrun. It supplies no production +support default and is not connected to the default financial, tax or PUF host. + +Six declared nodes expose the complete model-support schema: + +1. `child_property.donors` creates a private donor support population. Original + source coordinates are separate from its minted index IDs. The explicit + original-household design-weight column enters the fit; unit index weights + exist only to satisfy the support Frame contract. +2. `child_property.recipients` creates a private population containing every + selected original recipient, its exact coordinates, source age, component + knownness, eligibility and reason. Keeping this population nonempty permits + zero eligible children without a sentinel or an empty weight vector. It + emits the separate recipient projection and complete scenario artifact. +3. `child_property.fit` is the reusable typed empirical-fit node. Full-source + donors are independent of target sampling and clone expansion. The donor + projection excludes the selected preparation digest; changing target + sampling can change custody keys but not the fitted paired donor law. +4. `child_property.draw` is the reusable keyed-draw node. Its declared Boolean + eligibility input selects the original recipients internally. The full + source coordinate plus `child-property-v1` and `pattern`/`donor` suffixes + determine uniforms. Clone IDs, row order and scenario hashes do not. +5. `child_property.attach` reconstructs the real model and draws from the + retained source projections, verifies exact typed artifacts, and fills only + jointly unknown eligible O/D cells on both initial clones. It writes + `child_property_completion_status` and `child_property_imputed`; a model zero + has status `imputed_zero`, never source nonreceipt. Existing source-knownness, + all nonrecipient amounts, other columns, groups and geography are preserved. +6. `child_property.verify` requalifies the source again and compares all table + values/dtypes/missingness, effective weights and strata. It emits typed + verification evidence that still requires the host's complete-population + check, including owners and the ledger. + +`ChildPropertyBoundary` requires the actual authenticated preparation, retained +receiving `Population`, explicit options, typed host edges and exact artifact +pins. Its `require_parent` callback must recheck the host's issued parent and +return that exact Population. The fragment checks the callback's code, defaults +and closure identity, retained source issuance, complete source projections, +configuration and parent physical seals before and after callbacks. The real +source qualifier executes on every fresh check; serialized projections alone +cannot create a boundary. A detected execution/materialization failure revokes +the boundary for future checks. + +The country host must call `verify_materialized` on both cold and required +replay. Required cache hits skip kernel bodies. This method requires the actual +receiving population, both support populations, all verification-node input +`ArtifactValue`s and its output verification `ArtifactValue`. It re-executes the +real empirical fit and keyed draw independently of those artifacts, compares +all three complete populations with maintained replay equality, repeats source +and parent verification, and checks final output seals. Caller-owned artifact +descriptors and the verification output are detached before the first owner +borrow, then compared after the first and final borrows; their complete fields +and exact support-mapping keys must remain unchanged. The reconstructed artifact +and verification contracts are checked again after the final callback. The returned document +is descriptive evidence; it is not an issued financial, tax-complete or release +handle. + +Root-owned host integration remains explicit: insert after the complete survey +clones, atomic geography and current-property attachment, before property-tax +rebasing/gates and PUF. The receiving version must have a structural base so +the two amount writes are declared rewrites. Supply the real preparation, +allocation, frame-context and ordering edges/pins; retain the actual parent +owner in the callback and in the common host's live/source/replay seals. Bind +the actual run manifest's producer keys, implementation identities and source +identities when resolving artifact descriptors. This fragment reconstructs +payloads and checks typed paired producers; it does not independently issue +the common host's parent or validate an arbitrary claimed run manifest. Bind +the checked completion evidence into the tax gate without changing its +all-person completeness requirement. No unresolved adult class is completed by +this child candidate. The disabled/default path must remain unchanged. + +Tests separate pure descriptive fixtures from actual owner execution. Pure +fixtures test the real numerical law and graph patch, exact large IDs, clone +joins, zero stress, zero eligible children, known-incumbent refusal, exact +artifact bindings and drift anywhere in the receiving state. Actual-owner tests +reuse tiny authenticated survey fixtures whose sole teenager donor is absent +from the selected target sample. They run an invented receiving prefix and the +actual six-node fragment, including cold/required replay and first/final-owner +mutation controls for receiving values, artifact replacement, support-key +addition/removal, and verification payload/producer metadata. Refusal permanently +revokes the boundary. They do not claim actual financial/tax/PUF host acceptance. + +The accepted receiving Frame profile has no explicit links; the maintained +whole-population comparison refuses nonempty links rather than dropping them. + +Source projections and draw/completion JSON are private and bounded at 64 MiB; +selected originals are capped at 600,000. The generic model also has its 64 MiB +limit. JSON size checks follow object construction. No measured bytes-per-row, +peak memory or national capacity is claimed. This first implementation repeats +source requalification and model reconstruction at validation boundaries; a +separately reviewed sizing ladder and possible compact artifact representation +are required before a large candidate. Source-only tests cannot establish that +runtime performance or scientific donor support is adequate for release. diff --git a/docs/ci-file-sharding.md b/docs/ci-file-sharding.md new file mode 100644 index 000000000..eaa213790 --- /dev/null +++ b/docs/ci-file-sharding.md @@ -0,0 +1,77 @@ +# CI file sharding + +The fast `rest` inventory runs across six independent GitHub Actions matrix jobs +per Python version, the installed-wheel inventory across eight, and the engine +`us-am` inventory across four. The initial +partition uses sorted file paths round-robin. It balances file counts, not +measured runtime; an expensive single module can still dominate a job. + +Files are the smallest scheduling unit. All tests in a module, its module-scoped +fixtures, cold/required pairs and mutation controls stay in the same pytest +invocation. `us-am:build` and `us-am:other-shards` remain separate processes +inside each of the four `us-am` jobs, preserving import isolation. The US `us-qs` and +UK process divisions also remain unchanged. + +The environment tiers keep their original scope: + +- Fast covers the existing inventory except the explicit `ENGINE_ONLY` list. +- Engine groups partition the full inventory, with the existing country change + filters and US/UK extras. Both Python versions stay unchanged. +- Wheels partition the complete inventory, including `ENGINE_ONLY` files. Their + engine markers/import guards still skip when appropriate. Each matrix job + builds every real wheel, installs into a clean constrained venv, checks import + and spec identity, installs the same source extras, performs the HDF smoke and + confirms engine absence before its portion of the full suite. + +No tests are renamed, newly marked, skipped or replaced by sharding. Only the +partition regression file is added. The independent seed-diagnostic job and +the required `ci-ok` aggregation are unchanged. More jobs repeat setup and +wheel-building work; shorter wall time is not guaranteed by this configuration. +Every sharded pytest invocation prints its 25 slowest tests (`--durations=25`) +so an imbalance shows in the job log. `rest` went from four to six jobs and +`wheels` from four to eight on 2026-09-14 after one `rest` job and both `wheels` +2/4 jobs hit the six-hour runner limit; the partition is by file count, so a +few heavy modules dominate a job and the durations report is what shows which. + +## Selection and verification + +`tools/ci_test_groups.py` owns the file partition. For example: + +```sh +python3 tools/ci_test_groups.py --list rest --shard 1/6 +python3 tools/ci_test_groups.py --list us-am:build --shard 2/4 +python3 tools/ci_test_groups.py --list wheels --shard 8/8 +``` + +The inventory includes only `packages//tests/test_*.py` at that exact +depth. Git's explicit `:(glob)` pathspec and a matching path-depth check keep +nested test files out of the flat inventory. A separate tracked-file scan makes +`--verify` reject nested tests instead of silently dropping their coverage. + +`INDEX/COUNT` is 1-based, with `1 <= INDEX <= COUNT <= 64`. Selection sorts +the files and takes `files[INDEX-1::COUNT]` after the group/process filter. +Duplicate inventory entries, invalid ranges and an inventory too small to fill +every requested shard fail. Omitting the option preserves the original full +group selection. `wheels` is deliberately separate from the fast classifier. + +The workflow uses a checked command substitution and rejects empty output +before turning it into a bash array. It does not hide selector failures inside +process substitution, which can otherwise produce an empty pytest invocation +and inadvertently run the entire suite. + +```sh +python3 tools/ci_test_groups.py --verify +python3 -I -B -S packages/microcosm-build/tests/test_ci_test_groups.py +``` + +These commands need only the standard library and Git; they do not import +Microcosm, collect data tests, build wheels or run a microsimulation. The first +proves the nonempty, disjoint unions across groups, processes and configured +shards and rejects nested tests. The second exercises invalid/empty/duplicate +selection and verifies the actual workflow matrix covers each tier exactly +once while retaining the Python versions, process isolation and wheel boundary. + +Adding a file can shift subsequent round-robin assignments. The coverage proof +must still pass on the current checkout. Future tuning should use separately +reviewed module-level timing evidence; file counts are not timing evidence and +must not justify weaker checks or longer timeouts. diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md new file mode 100644 index 000000000..a4351354a --- /dev/null +++ b/docs/current-survey-puf59-progress.md @@ -0,0 +1,72 @@ +# PUF59 and current survey predictor implementation status + +The PUF59 profile fits and transfers 56 person outcomes and three tax-unit outcomes. Six detailed mortgage fields remain owned by the separate SCF producer. The historical full65 default remains available. PUF59 conditions on the source-specific PUF2015 filing-status class and capped return-size proxy, plus six monetary predictors. Its self-employment predictor includes ordinary and SSTB Schedule C income. Return-level Boolean outputs use an explicit incidence capacity of one; that is not a physical person count. + +The local PUF59/full65 regression run passed 128 ordinary tests. Those tests execute real weighted QRF fits, sequential target conditioning, draws, finalization, whole-Population preservation and required cache replay. They include excluded mortgage incumbent preservation and source-specific predictor checks. The run took 93.9 seconds and peaked at 622.7 MB RSS. These are invented mechanism fixtures, not held-out fit-quality or launch acceptance. + +Current survey financial completion passed 26 ordinary tests, including 15 ASEC demographic projection controls. The financial graph uses original ASEC design weights to fit interest, dividends and capital-gains outcomes for native ACS people. It preserves ACS observed wage/self-employment amounts and source-owned age-15 universe zeros. A draw is shared with its clone by source identity. The graph retains typed preparation, allocation and context evidence while separating the donor branch from survey allocation, avoiding a structural dependency cycle. Cold execution and required replay passed with the actual fitted model artifacts. The combined run took 121.4 seconds and peaked at 524.0 MB RSS. + +The source projection records the distinct survey observation, income and price periods. ASEC sex and state come from the retained original person and household sources, with allocation flags and unknownness preserved. Attaching those demographic projections to the graph is a separate pending integration step. The current financial model conditions on age and two earnings fields; sex/state conditioning and held-out fit quality remain explicit modeling work. + +Social Security handling remains a launch blocker. The historical PUF59 donor transports its total through the retirement component with modeled zero carrier columns for the other components. Those zeros do not establish absent recipient benefits. Its reconciliation code can treat unknown recipient components as zero and use equal component shares. The explicit `PUF55_SURVEY_SS` profile now excludes these four outcomes from PUF enrichment and conditions the remaining 55 outcomes on an additional Social Security total predictor. It preserves existing survey component fields, including unknownness. This profile is not yet wired into the native build. + +The new profile requires a known, finite, nonnegative `tax_unit.puf_conditioning_social_security_total` supplied by an upstream source owner. It does not infer that total from unknown components or move it between entity grains. All eight historical PUF59 predictors retain their order, with the total appended as predictor nine. The profile has a distinct train/apply phase and retains return-level incidence-capacity validation. Historical full65/PUF59 defaults and their 65/59 outcome contracts remain unchanged. + +Eleven new controls pass with real 55-target QRF fits, finalization, raw-draw validation and required cache replay. They check exact Social Security incumbent bit patterns, including signed zero and NaN, and refuse changes to the conditioning total after fitting. The run took 28.5 seconds and peaked at 631.6 MB RSS; receipt SHA256 is `8d5beacf15d8eb7bf568705d4e12bd71d810f3e748571b669a6ec44921ebf0cf`. All 128 historical profile cases also pass against the identical production module. An earlier combined run retained a failure in the new test's comparison of Boolean storage types; the corrected eleven-case run preserves exact values and missingness while treating the requested PUF Boolean output's nullable storage explicitly. Native recipient total qualification, beneficiary completion and whole-population PUF55 attachment remain separate pending checks. The native donor projection now passes as described below. + +The additive current-source qualifier now verifies original ASEC total, reason and allocation literals against the live survey preparation and retained original member. It also binds the ACS 2024 issuance and SSP price adjustment. Both surveys preserve under-15 question-universe unknownness. ASEC may record combined family payments on one person's report; a reason code is not a separately observed component amount or an assignment to individual beneficiaries. See [Social Security source semantics](survey-social-security-source.md) for the source contract, judgment boundaries and unresolved model work. + +The qualifier and numerical basis pass 52 controls: 50 numerical/literal cases and two actual maintained source issuances on invented original inputs. They include reordered source identities, ambiguous reasons, allocation flags, exact requalification, unknownness and no input mutation. The corrected run took 67.6 seconds and peaked at 428.9 MB RSS; the receipt SHA256 is `2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. An earlier run's two source tests exposed an incorrect capture-budget argument; the corrected caller uses the source owner's existing decoded-body bound while retaining exact CSV byte and digest checks. No native Social Security completion or beneficiary allocation has been accepted. + +The combined current source also passes all **225 component integration controls**: 34 shared geography cases, 52 survey Social Security cases, 11 PUF55 cases and all 128 historical full65/PUF59 cases. The exact combined run completed in 205.6 seconds at 687.8 MB peak RSS with no skips or unexpected access refusals. Before/after/current source, model-source and code-resource checks all agree; the maintained checkout matches the tested source map. Receipt SHA256: `63ef0ad0602a66d40f0736490d4c1e29f57723ba938c58d317802b387f947a87`. This remains an invented-input component integration, not a native population certification. + +The consolidated source passes a 314-case selected integration and identity suite, including the 128 profile cases, 26 financial/demographic cases, two canonical/growth wrappers, five metadata regressions and 153 identity controls. A subsequent diagnostic-option reconstruction correction passes all 16 current seven-node age-development tests, including late-mutation refusals. All source, model-source and resource pins survived external postchecks for each run. See the [review guide](us-launch-review.md) for exact revisions, receipts and exclusions. + +The new canonical59-to-PUF55 donor adapter also passes the maintained invented-source construction and artifact wrapper on 64 and 2,048 returns. It verifies the supplied artifact identity and the exact Social Security carrier convention before projecting the grown total into predictor nine. All 55 retained outputs, source RECIDs and weights survive unchanged. Internally rehashed artifacts with altered carrier semantics refuse. The corrected wrapper passed in 9.4 seconds at 486.9 MB peak RSS; receipt SHA256: `db3e9d8aa91863973de5d3201efcbabe31cbc1067c8c3f7fd72be79d887ffc27`. The initial knownness-column mismatch is preserved as a failed control; the fix supplies the owner's exact nonstructural column roster. This adapter grants no source authority. A subsequent separately guarded native projection passes for all 207,692 returns, retaining every target, source RECID, weight and incidence capacity and appending the modeled Social Security total as predictor nine. It took 11.05 seconds at 1.43 GB peak RSS. The original canonical artifact remained unchanged; source/control/model/code checks agree, with no unexpected access refusals. Receipt SHA256: `ce09455947f0efbca16014a23baa19037ac6450876c639568af6584c81bfc45c`. The earlier attempt passed its value assertions but lacked collection accounting and was refused by the guard; that failure is preserved. This native projection ran in memory and produced metadata only. It did not fit a recipient model or attach results to the survey population. + +The genuine seven-node survey age-development run now passes at the unchanged 1/1000 sampling fraction. It prepares 1,584 households and 3,464 people (ACS: 1,529 households/3,324 people; ASEC: 55/140), then retains 3,168 household records and 6,928 person records after the support clone. Export/readback, final source-owner and target checks completed under the frozen `a9895d8a5` source; the final manifest was written only after those checks. All 473 source/control/runtime and 12 code-resource postchecks agree, with no unexpected access refusals. The run took 4,268.8 seconds and peaked at 8.26 GB RSS. Receipt SHA256: `2aa76b5daca7bb0df1ac85d4db23d6b5482073d31d437418d46442ef01b002ca`. This run does not include the subsequent Social Security, PUF55 or atomic-geography integration. + +This is accepted native age-development evidence, not a national or congressional-district release, native enrichment fit-quality result or country-engine result. Complete enrichment, geography, calibration and release-export acceptance remain pending. + +The PUF55 profile now also passes three invented whole-Population attachment +controls. They execute all 55 fitted outcomes through the actual graph, typed +artifact loader and materialized population verifier, followed by Frame storage +and required replay. Every non-owned field, Social Security bit pattern and +unknown cell, membership, design anchor and household weight remains intact. +Two negative cases reject changed Social Security values and unknown-to-zero +conversions even when predictor matrices remain identical. The run took 21.03 +seconds at 631.3 MB peak RSS; receipt SHA256: +`ad8519d867279ebce9ee78bf8f495f3799afb45ec1b3fcda1be649f8a129af30`. +All 482 source/control, 5,983 model-source and 12 code-resource postchecks agree, +with no skips or unexpected access refusals. A subsequent independent review +required a direct finalizer comparison, because the attachment and its verifier +share a column-construction helper. The strengthened three-case run also passes: +all 55 attached outputs agree with the direct finalizer after independent entity +ID alignment, on cold and required replay. It took 20.73 seconds at 630.5 MB; +receipt SHA256 is +`e3a4d548f0209cfacd7d9e3beddb63c1b46204a3168eb9869d2b2eafe9cf2291`. +All 486 source/control, 5,983 model-source and 12 code-resource postchecks agree. +Native recipient qualification, measurement comparability and actual survey +attachment remain pending. + +The observed-geography source qualifier passes 11 invented original-source +controls. It binds current ASEC state and ACS state/PUMA to literal source +household keys, preserves unknown locations, and keeps draw keys stable across +sampling fractions. The initial ten-case run correctly refused a smaller sample +with no positive ASEC support; its corrected fixture supplies positive support +before issuance. Review also added a receipt-digest check against mutation of +the detached ASEC projection. The passing run took 182.50 seconds at 430.3 MB; +receipt SHA256 is +`1900801b1bbcea89a3747f656410e500b1328519694c3351100562bba61cab08`. +The 484 source/control, 5,983 model-source and 12 code-resource postchecks agree, +with no skips or unexpected refusals. + +The observed-geography graph node separately passes 14 controls over the same +kind of invented original sources. Actual three-node execution and required +replay preserve inputs, weights and unknown geography. Declaration, source +receipt, typed-artifact and final-mutation cases refuse invalid output. The run +took 319.44 seconds at 439.1 MB; receipt SHA256 is +`28a6b5c58a258c7b3129a48054a3360647c788f8b76d781cc3bc8fc35a73f997`. +All 486 source/control, 5,983 model-source and 12 code-resource postchecks agree. +Full geography-before-clone composition, native block-source integration and +complete calibration remain separate work. diff --git a/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json b/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json new file mode 100644 index 000000000..79974b85d --- /dev/null +++ b/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json @@ -0,0 +1,447 @@ +{ + "schema": "microcosm.us.puf2015_target2024_growth/1", + "input_money_year": 2015, + "output_money_year": 2024, + "output_profile": "puf59_scf_mortgage_v1", + "ordered_targets": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "tax_exempt_interest_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "non_sch_d_capital_gains", + "taxable_private_pension_income", + "taxable_ira_distributions", + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", + "alimony_income", + "alimony_expense", + "salt_refund_income", + "charitable_cash_donations", + "charitable_non_cash_donations", + "real_estate_taxes", + "home_mortgage_interest", + "investment_interest_expense", + "investment_income_elected_form_4952", + "student_loan_interest", + "educator_expense", + "qualified_tuition_expenses", + "casualty_loss", + "unreimbursed_business_employee_expenses", + "traditional_ira_contributions_desired", + "self_employed_pension_contributions_desired", + "rental_income", + "estate_income", + "farm_income", + "farm_operations_income", + "farm_rent_income", + "miscellaneous_income", + "partnership_income", + "s_corp_income", + "partnership_self_employment_net_earnings", + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", + "sstb_self_employment_income_would_be_qualified", + "business_is_sstb", + "qualified_bdc_income", + "qualified_reit_and_ptp_income", + "sstb_self_employment_income_before_lsr", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "unadjusted_basis_qualified_property", + "w2_wages_from_qualified_business", + "domestic_production_ald", + "unrecaptured_section_1250_gain", + "health_savings_account_ald" + ], + "money_fields": { + "employment_income_before_lsr": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "w2_wages_from_qualified_business": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "sstb_w2_wages_from_qualified_business": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "self_employment_income_before_lsr": { + "positive_family": "business_profit", + "negative_family": "business_loss", + "positive_factor": "1.222517420922031096142334753675412822692", + "negative_factor": "1.718898116890934040328182671660914217539", + "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition." + }, + "sstb_self_employment_income_before_lsr": { + "positive_family": "business_profit", + "negative_family": "business_loss", + "positive_factor": "1.222517420922031096142334753675412822692", + "negative_factor": "1.718898116890934040328182671660914217539", + "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition." + }, + "taxable_interest_income": { + "positive_family": "taxable_interest", + "negative_family": "taxable_interest", + "positive_factor": "2.599750625636417399355437523686408804619", + "negative_factor": "2.599750625636417399355437523686408804619", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "tax_exempt_interest_income": { + "positive_family": "tax_exempt_interest", + "negative_family": "tax_exempt_interest", + "positive_factor": "0.9295395824924337172788943063597248917385", + "negative_factor": "0.9295395824924337172788943063597248917385", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "qualified_dividend_income": { + "positive_family": "qualified_dividends", + "negative_family": "qualified_dividends", + "positive_factor": "1.409891045424454325551259680558160347719", + "negative_factor": "1.409891045424454325551259680558160347719", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "non_qualified_dividend_income": { + "positive_family": "nonqualified_dividends", + "negative_family": "nonqualified_dividends", + "positive_factor": "2.483795459689869824933006427197633652403", + "negative_factor": "2.483795459689869824933006427197633652403", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "non_sch_d_capital_gains": { + "positive_family": "non_schedule_d_distributions", + "negative_family": "non_schedule_d_distributions", + "positive_factor": "1.120350061633613230084419890940102481805", + "negative_factor": "1.120350061633613230084419890940102481805", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "taxable_private_pension_income": { + "positive_family": "taxable_pensions", + "negative_family": "taxable_pensions", + "positive_factor": "1.327588034973940867230285994253866499704", + "negative_factor": "1.327588034973940867230285994253866499704", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "taxable_ira_distributions": { + "positive_family": "ira_distributions", + "negative_family": "ira_distributions", + "positive_factor": "1.510871073557979130794209158828995676924", + "negative_factor": "1.510871073557979130794209158828995676924", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "alimony_income": { + "positive_family": "alimony_income", + "negative_family": "alimony_income", + "positive_factor": "1.542033706590010885068081742929926580179", + "negative_factor": "1.542033706590010885068081742929926580179", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "salt_refund_income": { + "positive_family": "state_tax_refund", + "negative_family": "state_tax_refund", + "positive_factor": "0.9296526775720357466732616002244135476521", + "negative_factor": "0.9296526775720357466732616002244135476521", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "short_term_capital_gains": { + "positive_family": "short_current_gain", + "negative_family": "short_current_loss", + "positive_factor": "2.042865587775988756504710527637911842775", + "negative_factor": "1.738139193325432816803529458382722438642", + "assumption": "Current-year gross gains/loss components per all return proxy for signed net PUF amount; carried losses are excluded." + }, + "long_term_capital_gains_before_response": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "long_term_capital_gains_on_collectibles": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "unrecaptured_section_1250_gain": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "social_security_retirement": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_disability": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_dependents": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_survivors": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "rental_income": { + "positive_family": "rental_royalty_profit", + "negative_family": "rental_royalty_loss", + "positive_factor": "1.476663829685885394438774259073237903799", + "negative_factor": "1.690793641559507564235995133214906035185", + "assumption": "Related rental/royalty reporting-return mean proxy; published total family includes farm rent and does not exactly equal E25850/E25860 component definitions." + }, + "estate_income": { + "positive_family": "estate_profit", + "negative_family": "estate_loss", + "positive_factor": "1.476331638358104936021886589313386353228", + "negative_factor": "1.582810027948011789708036602150526302352", + "assumption": "" + }, + "farm_operations_income": { + "positive_family": "farm_profit", + "negative_family": "farm_loss", + "positive_factor": "1.444720208215047581256304812994467826979", + "negative_factor": "1.728383897337776476443532509645504817056", + "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income." + }, + "farm_income": { + "positive_family": "farm_profit", + "negative_family": "farm_loss", + "positive_factor": "1.444720208215047581256304812994467826979", + "negative_factor": "1.728383897337776476443532509645504817056", + "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income." + }, + "farm_rent_income": { + "positive_family": "farm_rent_profit", + "negative_family": "farm_rent_loss", + "positive_factor": "1.629452810459537374541656575493532799084", + "negative_factor": "1.150378978298943195689490419607276484593", + "assumption": "" + }, + "miscellaneous_income": { + "positive_family": "other_property_gain", + "negative_family": "other_property_loss", + "positive_factor": "1.738242164922618767195190248113271033066", + "negative_factor": "1.411645744283618865226902643798237455207", + "assumption": "Canonical source E01200 is other property gains/losses, not Table1.4 other income." + }, + "partnership_income": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "s_corp_income": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "partnership_self_employment_net_earnings": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "qualified_bdc_income": { + "positive_family": "nonqualified_dividends", + "negative_family": "nonqualified_dividends", + "positive_factor": "2.483795459689869824933006427197633652403", + "negative_factor": "2.483795459689869824933006427197633652403", + "assumption": "Modeled BDC component follows its parent nonqualified dividend pool; no independent source observation." + }, + "qualified_reit_and_ptp_income": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real proxy for the combined modeled REIT/PTP output, which mixes nonqualified-dividend and pass-through components. No claim that this is a pure dividend pool." + }, + "alimony_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "charitable_cash_donations": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "charitable_non_cash_donations": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "real_estate_taxes": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "home_mortgage_interest": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "investment_interest_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "investment_income_elected_form_4952": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "student_loan_interest": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "educator_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "qualified_tuition_expenses": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "casualty_loss": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "unreimbursed_business_employee_expenses": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "traditional_ira_contributions_desired": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "self_employed_pension_contributions_desired": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "domestic_production_ald": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "health_savings_account_ald": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "unadjusted_basis_qualified_property": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "sstb_unadjusted_basis_qualified_property": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + } + }, + "return_incidence_fields": [ + "business_is_sstb", + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", + "sstb_self_employment_income_would_be_qualified" + ], + "source_pins": { + "NATIONAL-GROWTH-EXTRACT.json": "775d89d3fe7a3b1eb9a09c50f6077d41bf838cdb80d9e4aaa9f0ab80ada4e0c8", + "INDEX-VALUES.json": "221f3786a462fe105d3da15df3624de5ad3e3f921b4d0205e096067685fd59b3", + "extract_national.py": "b6c3a8e4fea52653c01d801d6efe9620cdba4a489d6d7ae8c95cf00ee6dfbe1b", + "ACQUISITION.json": "e9a2c34c2ca06a229b2a790ed2987f25d31996653490f342b4b54af37d5f098c" + }, + "sensitivity_cpi_factor": "1.323487344789614247079323424058189918867", + "provenance": "All growth applications are modeled distribution transport from observed national series; source observed/derived/modeled flags remain separate.", + "operation_order": "Decode source statistical2015 \u2192 source canonical transformations \u2192 QBI2015 model (all16 outputs) \u2192 this growth transform once \u2192 2024 canonical donor. Monetary recipient predictors remain 2024.", + "noninterference": "RECID, S006, raw FLPDYR/FLPDMO, raw money, count source fields and actual survey membership are separate inputs and are not accepted or mutated by this transform.", + "sensitivities": "cpi_only uses exactly the same source cohort, zeros, signs and return incidence; no new policy evaluation or calibration authority." +} diff --git a/docs/evidence/puf2015-target2024/INDEX-VALUES.json b/docs/evidence/puf2015-target2024/INDEX-VALUES.json new file mode 100644 index 000000000..5dd19e557 --- /dev/null +++ b/docs/evidence/puf2015-target2024/INDEX-VALUES.json @@ -0,0 +1,8 @@ +{ + "schema": "microcosm.public_index_value_transcript/1", + "retrieved_date": "2026-09-09", + "retrieval": "Official URLs successfully read with web retrieval; exact values transcribed and independently checkable. These are not hashes of original publisher HTML/PDF bytes. Direct SSA urllib acquisition returned HTTP403 before writing any document.", + "awi": {"url": "https://www.ssa.gov/oact/cola/AWI.html", "series": "National Average Wage Index", "2015": "48098.63", "2024": "69846.57", "measurement": "Wages subject to federal income tax and deferred compensation; average-worker index, used as an explicit proxy for PUF wage amount growth."}, + "cola": {"url": "https://www.ssa.gov/oact/cola/colaseries.html", "cash_timing_url": "https://www.ssa.gov/cola/", "december_payable_percent": {"2015": "0.0", "2016": "0.3", "2017": "2.0", "2018": "2.8", "2019": "1.6", "2020": "1.3", "2021": "5.9", "2022": "8.7", "2023": "3.2"}, "timing": "December benefits are received in January of the following year. Product covers January2016 through January2024 cash increases relative to calendar2015 cash benefits. This transports a fixed entitlement; it does not forecast the recipient mix or new awards."}, + "cpi": {"url": "https://www.bls.gov/cpi/tables/supplemental-files/historical-cpi-u-202412.pdf", "series": "CPI-U, US city average, all items, annual average, 1982-84=100", "table": "Historical CPI-U index averages, annual average column", "2015": "237.017", "2023": "304.702", "2024": "313.689", "pages_one_based": [4,5], "monthly_2024_check": ["308.417","310.326","312.332","313.548","314.069","314.175","314.540","314.796","315.301","315.664","315.493","315.605"]} +} diff --git a/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json b/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json new file mode 100644 index 000000000..e24f8d37d --- /dev/null +++ b/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json @@ -0,0 +1,4439 @@ +{ + "schema": "microcosm.us.puf_observed_growth_sources/1", + "scope": "exact headers and national row only; no AGI rows used", + "tables": { + "15in14ar.xls": { + "url": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "sha256": "9f41bee263f5a8bb971df39a9673b55136c94954131a95bfd83b73a22801ec89", + "sheet": "TBL14", + "year": 2015, + "title": "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, \nby Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 9, + "all_returns": { + "cell": "B9", + "value": 150493263, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns" + ] + }, + "families": { + "taxable_interest": { + "count": { + "cell": "H9", + "value": 42636696, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I9", + "value": 95881223, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "tax_exempt_interest": { + "count": { + "cell": "J9", + "value": 5827038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K9", + "value": 61871455, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Amount", + "Amount" + ] + } + }, + "ordinary_dividends": { + "count": { + "cell": "L9", + "value": 27607044, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M9", + "value": 260252720, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "qualified_dividends": { + "count": { + "cell": "N9", + "value": 25755976, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "O9", + "value": 203187788, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Amount", + "Amount" + ] + } + }, + "state_tax_refund": { + "count": { + "cell": "P9", + "value": 20256512, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Q9", + "value": 31110732, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "alimony_income": { + "count": { + "cell": "R9", + "value": 414420, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "S9", + "value": 10077086, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "business_profit": { + "count": { + "cell": "T9", + "value": 18791200, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 391975736, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "business_loss": { + "count": { + "cell": "V9", + "value": 5935725, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 60161435, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "non_schedule_d_distributions": { + "count": { + "cell": "X9", + "value": 4323250, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 11563203, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "other_property_gain": { + "count": { + "cell": "AD9", + "value": 1063576, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 33036562, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "other_property_loss": { + "count": { + "cell": "AF9", + "value": 1153117, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 21093508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "ira_distributions": { + "count": { + "cell": "AH9", + "value": 14159018, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 253213041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "taxable_pensions": { + "count": { + "cell": "AL9", + "value": 28199160, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AM9", + "value": 689991999, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_profit": { + "count": { + "cell": "AZ9", + "value": 6768234, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA9", + "value": 103058883, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_loss": { + "count": { + "cell": "BB9", + "value": 4531666, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC9", + "value": 46245560, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_rent_profit": { + "count": { + "cell": "AV9", + "value": 394273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AW9", + "value": 5205330, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_rent_loss": { + "count": { + "cell": "AX9", + "value": 101041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 710953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "estate_profit": { + "count": { + "cell": "BH9", + "value": 630256, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 32453002, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "estate_loss": { + "count": { + "cell": "BJ9", + "value": 57802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 5033199, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_profit": { + "count": { + "cell": "BL9", + "value": 520802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 13533867, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_loss": { + "count": { + "cell": "BN9", + "value": 1278825, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 27497651, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "other_income_profit": { + "count": { + "cell": "BX9", + "value": 6121770, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 46769290, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "other_income_loss": { + "count": { + "cell": "BZ9", + "value": 332708, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 6693961, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "partnership_s_corp_profit": { + "count": { + "cell": "BD9", + "value": 6044409, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE9", + "value": 755622761, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "partnership_s_corp_loss": { + "count": { + "cell": "BF9", + "value": 2699817, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG9", + "value": 126618186, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + } + }, + "23in14ar.xls": { + "url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "sha256": "b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af", + "sheet": "TBL14", + "year": 2023, + "title": "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, \nby Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 9, + "all_returns": { + "cell": "B9", + "value": 160602107, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns" + ] + }, + "families": { + "taxable_interest": { + "count": { + "cell": "T9", + "value": 55260238, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 313812674, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "tax_exempt_interest": { + "count": { + "cell": "V9", + "value": 6893860, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 66091992, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Amount", + "Amount" + ] + } + }, + "ordinary_dividends": { + "count": { + "cell": "X9", + "value": 33718460, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 504224981, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "qualified_dividends": { + "count": { + "cell": "Z9", + "value": 31106344, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AA9", + "value": 336070296, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Amount", + "Amount" + ] + } + }, + "state_tax_refund": { + "count": { + "cell": "AB9", + "value": 3109096, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AC9", + "value": 4311977, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "alimony_income": { + "count": { + "cell": "AD9", + "value": 183582, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 6686429, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "business_profit": { + "count": { + "cell": "AF9", + "value": 21411713, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 530380019, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "business_loss": { + "count": { + "cell": "AH9", + "value": 9067174, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 153441387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "non_schedule_d_distributions": { + "count": { + "cell": "AJ9", + "value": 3209131, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AK9", + "value": 9340820, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "other_property_gain": { + "count": { + "cell": "AP9", + "value": 945122, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AQ9", + "value": 49567896, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "other_property_loss": { + "count": { + "cell": "AR9", + "value": 775387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AS9", + "value": 19448929, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "ira_distributions": { + "count": { + "cell": "AT9", + "value": 16694154, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AU9", + "value": 438147938, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "taxable_pensions": { + "count": { + "cell": "AX9", + "value": 29541284, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 932130236, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_profit": { + "count": { + "cell": "BL9", + "value": 6346357, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 138609251, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_loss": { + "count": { + "cell": "BN9", + "value": 3196282, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 53570273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_rent_profit": { + "count": { + "cell": "BH9", + "value": 325953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 6811206, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_rent_loss": { + "count": { + "cell": "BJ9", + "value": 67356, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 529586, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "estate_profit": { + "count": { + "cell": "BX9", + "value": 648583, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 47892046, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "estate_loss": { + "count": { + "cell": "BZ9", + "value": 36592, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 4898828, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_profit": { + "count": { + "cell": "CB9", + "value": 484162, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CC9", + "value": 17656301, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_loss": { + "count": { + "cell": "CD9", + "value": 1258869, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CE9", + "value": 45444491, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "other_income_profit": { + "count": { + "cell": "CN9", + "value": 7468434, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CO9", + "value": 60816508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "other_income_loss": { + "count": { + "cell": "CP9", + "value": 318839, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CQ9", + "value": 12213628, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "partnership_profit": { + "count": { + "cell": "BP9", + "value": 3266345, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ9", + "value": 452077038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "partnership_loss": { + "count": { + "cell": "BR9", + "value": 2103634, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BS9", + "value": 168884100, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "s_corp_profit": { + "count": { + "cell": "BT9", + "value": 4138040, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BU9", + "value": 797450861, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "s_corp_loss": { + "count": { + "cell": "BV9", + "value": 1415974, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BW9", + "value": 89543715, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + } + }, + "15in14acg.xls": { + "url": "https://www.irs.gov/pub/irs-soi/15in14acg.xls", + "sha256": "7e128eb06e14371a15baa72060e91d74e4c55ba8596610ddd047c9ef168cbbd1", + "sheet": "Sheet1", + "year": 2015, + "title": "Table 1.4A Returns with Income or Loss from Sales of Capital Assets\nReported on Form 1040, Schedule D: Selected Items, by Size of\nAdjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 10, + "families": { + "short_term_gain": { + "count": { + "cell": "F10", + "value": 3900677, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "G10", + "value": 38164189, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Amount", + "Amount" + ] + } + }, + "short_term_loss": { + "count": { + "cell": "H10", + "value": 7820586, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I10", + "value": 250592206, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Amount", + "Amount" + ] + } + }, + "long_term_gain": { + "count": { + "cell": "BJ10", + "value": 11735160, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK10", + "value": 733313255, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Amount", + "Amount" + ] + } + }, + "long_term_loss": { + "count": { + "cell": "BL10", + "value": 6375392, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM10", + "value": 312723716, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Amount", + "Amount" + ] + } + }, + "short_assets_gain": { + "count": { + "cell": "J10", + "value": 3923409, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K10", + "value": 22196394, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_assets_loss": { + "count": { + "cell": "L10", + "value": 6269981, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M10", + "value": 68624843, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_other_gain": { + "count": { + "cell": "AZ10", + "value": 231076, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA10", + "value": 5948464, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "short_other_loss": { + "count": { + "cell": "BB10", + "value": 336867, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC10", + "value": 1980907, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_gain": { + "count": { + "cell": "BD10", + "value": 483349, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE10", + "value": 17194108, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_loss": { + "count": { + "cell": "BF10", + "value": 725964, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG10", + "value": 16056081, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_carryover": { + "count": { + "cell": "BH10", + "value": 2000156, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI10", + "value": 171105171, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Amount", + "Amount" + ] + } + }, + "long_assets_gain": { + "count": { + "cell": "BN10", + "value": 8440065, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO10", + "value": 282493545, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_assets_loss": { + "count": { + "cell": "BP10", + "value": 4441176, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ10", + "value": 62343837, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_other_gain": { + "count": { + "cell": "DD10", + "value": 2177243, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DE10", + "value": 226671889, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "long_other_loss": { + "count": { + "cell": "DF10", + "value": 296456, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DG10", + "value": 2562723, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_gain": { + "count": { + "cell": "DH10", + "value": 1721361, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DI10", + "value": 216227585, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_loss": { + "count": { + "cell": "DJ10", + "value": 552552, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DK10", + "value": 10694019, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_distributions": { + "count": { + "cell": "DL10", + "value": 9733033, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DM10", + "value": 62496866, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Amount", + "Amount" + ] + } + }, + "long_carryover": { + "count": { + "cell": "DN10", + "value": 4371225, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DO10", + "value": 291699768, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Amount", + "Amount" + ] + } + } + } + }, + "23in14acg.xls": { + "url": "https://www.irs.gov/pub/irs-soi/23in14acg.xls", + "sha256": "5b35c2f007f16503c0dcf6cc085d3a6b049ca523d99b7d7ae29afdbdd2d02e70", + "sheet": "Sheet1", + "year": 2023, + "title": "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets\nReported on Form 1040, Schedule D: Selected Items, by Size of\nAdjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 10, + "families": { + "short_term_gain": { + "count": { + "cell": "F10", + "value": 6215519, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "G10", + "value": 70482448, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Amount", + "Amount" + ] + } + }, + "short_term_loss": { + "count": { + "cell": "H10", + "value": 11370982, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I10", + "value": 673910232, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Amount", + "Amount" + ] + } + }, + "long_term_gain": { + "count": { + "cell": "BJ10", + "value": 11888516, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK10", + "value": 971279947, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Amount", + "Amount" + ] + } + }, + "long_term_loss": { + "count": { + "cell": "BL10", + "value": 11230240, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM10", + "value": 450065138, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Amount", + "Amount" + ] + } + }, + "short_assets_gain": { + "count": { + "cell": "J10", + "value": 7140149, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K10", + "value": 58927145, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_assets_loss": { + "count": { + "cell": "L10", + "value": 8377604, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M10", + "value": 135549749, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_other_gain": { + "count": { + "cell": "AZ10", + "value": 174491, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA10", + "value": 8619769, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "short_other_loss": { + "count": { + "cell": "BB10", + "value": 401474, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC10", + "value": 3833687, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_gain": { + "count": { + "cell": "BD10", + "value": 495780, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE10", + "value": 28464220, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_loss": { + "count": { + "cell": "BF10", + "value": 560758, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG10", + "value": 16759596, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_carryover": { + "count": { + "cell": "BH10", + "value": 5151589, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI10", + "value": 543302365, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Amount", + "Amount" + ] + } + }, + "long_assets_gain": { + "count": { + "cell": "BN10", + "value": 8891406, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO10", + "value": 405281941, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_assets_loss": { + "count": { + "cell": "BP10", + "value": 9496262, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ10", + "value": 149558927, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_other_gain": { + "count": { + "cell": "DD10", + "value": 1979380, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DE10", + "value": 316779349, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "long_other_loss": { + "count": { + "cell": "DF10", + "value": 338620, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DG10", + "value": 5211519, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_gain": { + "count": { + "cell": "DH10", + "value": 1219800, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DI10", + "value": 282281744, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_loss": { + "count": { + "cell": "DJ10", + "value": 766861, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DK10", + "value": 22526709, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_distributions": { + "count": { + "cell": "DL10", + "value": 9487999, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DM10", + "value": 44095015, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Amount", + "Amount" + ] + } + }, + "long_carryover": { + "count": { + "cell": "DN10", + "value": 5622923, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DO10", + "value": 349944357, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Amount", + "Amount" + ] + } + } + } + } + }, + "factors": { + "taxable_interest": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "H9", + "value": 42636696, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I9", + "value": 95881223, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "T9", + "value": 55260238, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 313812674, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "2.525269343625908636956987730976528139607", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.599750625636417399355437523686408804619" + }, + "tax_exempt_interest": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "J9", + "value": 5827038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K9", + "value": 61871455, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "V9", + "value": 6893860, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 66091992, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "0.9029088360274333448808012169263853497016", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "0.9295395824924337172788943063597248917385" + }, + "ordinary_dividends": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "L9", + "value": 27607044, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M9", + "value": 260252720, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "X9", + "value": 33718460, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 504224981, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.586285145427430620786783837166923431342", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.633071660126895406016322292262784767590" + }, + "qualified_dividends": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "N9", + "value": 25755976, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "O9", + "value": 203187788, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "Z9", + "value": 31106344, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AA9", + "value": 336070296, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.369498520263452278862567470282453558367", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.409891045424454325551259680558160347719" + }, + "state_tax_refund": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "P9", + "value": 20256512, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Q9", + "value": 31110732, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AB9", + "value": 3109096, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AC9", + "value": 4311977, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "0.9030186910014518713848306957259555062394", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "0.9296526775720357466732616002244135476521" + }, + "alimony_income": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "R9", + "value": 414420, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "S9", + "value": 10077086, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AD9", + "value": 183582, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 6686429, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.497855374161636196047724476262267688168", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.542033706590010885068081742929926580179" + }, + "business_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "T9", + "value": 18791200, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 391975736, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AF9", + "value": 21411713, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 530380019, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.187493036701270108472919624578501757792", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.222517420922031096142334753675412822692" + }, + "business_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "V9", + "value": 5935725, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 60161435, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AH9", + "value": 9067174, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 153441387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.669652726148833347538733957583542565766", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.718898116890934040328182671660914217539" + }, + "non_schedule_d_distributions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "X9", + "value": 4323250, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 11563203, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AJ9", + "value": 3209131, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AK9", + "value": 9340820, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.088252710423015210712466518141315463440", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.120350061633613230084419890940102481805" + }, + "other_property_gain": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AD9", + "value": 1063576, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 33036562, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AP9", + "value": 945122, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AQ9", + "value": 49567896, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.688442578911762234575993608257254511052", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.738242164922618767195190248113271033066" + }, + "other_property_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AF9", + "value": 1153117, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 21093508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AR9", + "value": 775387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AS9", + "value": 19448929, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.371202948062275806522918206792748706765", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.411645744283618865226902643798237455207" + }, + "ira_distributions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AH9", + "value": 14159018, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 253213041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AT9", + "value": 16694154, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AU9", + "value": 438147938, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.467585531705808482641269279807429143993", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.510871073557979130794209158828995676924" + }, + "taxable_pensions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AL9", + "value": 28199160, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AM9", + "value": 689991999, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AX9", + "value": 29541284, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 932130236, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.289553441251142788324750319651443404751", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.327588034973940867230285994253866499704" + }, + "rental_royalty_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AZ9", + "value": 6768234, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA9", + "value": 103058883, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BL9", + "value": 6346357, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 138609251, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.434358304667835504133977902598222238470", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.476663829685885394438774259073237903799" + }, + "rental_royalty_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BB9", + "value": 4531666, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC9", + "value": 46245560, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BN9", + "value": 3196282, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 53570273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.642353427026338423846026443645930519505", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.690793641559507564235995133214906035185" + }, + "farm_rent_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AV9", + "value": 394273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AW9", + "value": 5205330, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BH9", + "value": 325953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 6811206, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.582769973612852083106490319603270854083", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.629452810459537374541656575493532799084" + }, + "farm_rent_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AX9", + "value": 101041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 710953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BJ9", + "value": 67356, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 529586, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.117421316799902418041369349372073484912", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.150378978298943195689490419607276484593" + }, + "estate_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BH9", + "value": 630256, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 32453002, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BX9", + "value": 648583, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 47892046, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.434035630420548027555129084975779987827", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.476331638358104936021886589313386353228" + }, + "estate_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BJ9", + "value": 57802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 5033199, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BZ9", + "value": 36592, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 4898828, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.537463478591264240530009559622650667952", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.582810027948011789708036602150526302352" + }, + "farm_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BL9", + "value": 520802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 13533867, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CB9", + "value": 484162, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CC9", + "value": 17656301, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.403329848619305835091312061082920777637", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.444720208215047581256304812994467826979" + }, + "farm_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BN9", + "value": 1278825, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 27497651, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CD9", + "value": 1258869, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CE9", + "value": 45444491, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.678866744726831887395787683833365558775", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.728383897337776476443532509645504817056" + }, + "other_income_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BX9", + "value": 6121770, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 46769290, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CN9", + "value": 7468434, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CO9", + "value": 60816508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.065879634464584874641519579136196913601", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.097317105419594110775195552571543585636" + }, + "other_income_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BZ9", + "value": 332708, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 6693961, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CP9", + "value": 318839, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CQ9", + "value": 12213628, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.903940252832176698123261573865488200400", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.960095811549227363973941096035769795063" + }, + "short_current_gain": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "short_assets_gain", + "short_other_gain", + "short_pass_through_gain" + ], + "amounts_thousands_usd": [ + 45338966, + 96011134 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.984338725063739328074935070060878814103", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.042865587775988756504710527637911842775" + }, + "short_current_loss": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "short_assets_loss", + "short_other_loss", + "short_pass_through_loss" + ], + "amounts_thousands_usd": [ + 86661831, + 156143032 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.688342557388515472795249540239320768338", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.738139193325432816803529458382722438642" + }, + "long_current_gain": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "long_assets_gain", + "long_other_gain", + "long_pass_through_gain", + "long_distributions" + ], + "amounts_thousands_usd": [ + 787889885, + 1048438049 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.246932856321931105568396313543555115004", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.283710381837894882785950440755768785471" + }, + "long_current_loss": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "long_assets_loss", + "long_other_loss", + "long_pass_through_loss" + ], + "amounts_thousands_usd": [ + 75600579, + 177297155 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "2.197568722014698372473641296575486125444", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.262384673681395979556694950087192296750" + }, + "partnership_s_corp_profit": { + "method": "combined_dollars_per_all_return", + "source": { + "count": { + "cell": "BD9", + "value": 6044409, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE9", + "value": 755622761, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023_components": [ + { + "count": { + "cell": "BP9", + "value": 3266345, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ9", + "value": 452077038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + { + "count": { + "cell": "BT9", + "value": 4138040, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BU9", + "value": 797450861, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + } + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "observed_2015_2023_factor": "1.549554109090802831376907083449129785317", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.595257264233857504616939193376066035754" + }, + "partnership_s_corp_loss": { + "method": "combined_dollars_per_all_return", + "source": { + "count": { + "cell": "BF9", + "value": 2699817, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG9", + "value": 126618186, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023_components": [ + { + "count": { + "cell": "BR9", + "value": 2103634, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BS9", + "value": 168884100, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + { + "count": { + "cell": "BV9", + "value": 1415974, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BW9", + "value": 89543715, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "observed_2015_2023_factor": "1.912533224223542778513579904641714927631", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.968942227400735507640732147170530268700" + }, + "nonqualified_dividends": { + "method": "residual_dollars_per_ordinary_dividend_return", + "observed_2015_2023_factor": "2.412636223005660751243227924409122950293", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.483795459689869824933006427197633652403", + "source_families": [ + "ordinary_dividends", + "qualified_dividends" + ] + } + }, + "modeled_bridge": { + "method": "constant real conditional amounts after 2023", + "cpi_2023": "304.702", + "cpi_2024": "313.689" + }, + "capital_carryover_reconciliation_thousands_usd": { + "short": { + "2015": { + "net_including_carried_loss": -212428017, + "carried_loss": 171105171, + "current_year_net_by_removing_carryover": -41322846, + "current_year_gross_component_gain": 45338966, + "current_year_gross_component_loss": 86661831, + "current_year_net_from_component_difference": -41322865, + "reported_less_component_net_residual": 19, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + }, + "2023": { + "net_including_carried_loss": -603427784, + "carried_loss": 543302365, + "current_year_net_by_removing_carryover": -60125419, + "current_year_gross_component_gain": 96011134, + "current_year_gross_component_loss": 156143032, + "current_year_net_from_component_difference": -60131898, + "reported_less_component_net_residual": 6479, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + } + }, + "long": { + "2015": { + "net_including_carried_loss": 420589539, + "carried_loss": 291699768, + "current_year_net_by_removing_carryover": 712289307, + "current_year_gross_component_gain": 787889885, + "current_year_gross_component_loss": 75600579, + "current_year_net_from_component_difference": 712289306, + "reported_less_component_net_residual": 1, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + }, + "2023": { + "net_including_carried_loss": 521214809, + "carried_loss": 349944357, + "current_year_net_by_removing_carryover": 871159166, + "current_year_gross_component_gain": 1048438049, + "current_year_gross_component_loss": 177297155, + "current_year_net_from_component_difference": 871140894, + "reported_less_component_net_residual": 18272, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + } + } + }, + "notes": [ + "Reporting-return means mix intensive-margin growth and changing composition. This is a distribution transport assumption, not panel growth.", + "Positive/loss magnitudes are distinct series. Source zeros and sign states are retained.", + "No total calibration or reporting-incidence claim; source design weights are unchanged." + ] +} diff --git a/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json b/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json new file mode 100644 index 000000000..23975da76 --- /dev/null +++ b/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json @@ -0,0 +1,34 @@ +[ + { + "url": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "bytes": 83456, + "sha256": "9f41bee263f5a8bb971df39a9673b55136c94954131a95bfd83b73a22801ec89", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "15in14ar.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "bytes": 115712, + "sha256": "b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "23in14ar.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/15in14acg.xls", + "bytes": 64000, + "sha256": "7e128eb06e14371a15baa72060e91d74e4c55ba8596610ddd047c9ef168cbbd1", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "15in14acg.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/23in14acg.xls", + "bytes": 74752, + "sha256": "5b35c2f007f16503c0dcf6cc085d3a6b049ca523d99b7d7ae29afdbdd2d02e70", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "23in14acg.xls" + } +] diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index db6d790c7..155170d73 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "86cecb19281446d2b505fb975c822c60a604e18f44baded5dc9d62d508a8197e", - "protocol_sha256": "55d6b907dd44613a5498dd7156f1ce9e78758835255bc7c336b4c327bb40e9bf" + "map_sha256": "e1be208d9420e39ccf3486eb8aa02ed76e778a5b77cae1615e71ce63562df75a", + "protocol_sha256": "1a8a3c2946b704a0b90f30a3f8e53e360f5dd3012ff2cc2d8e2ca1eb3062d397" }, "failures": [], "observed": { - "map_sha256": "86cecb19281446d2b505fb975c822c60a604e18f44baded5dc9d62d508a8197e", - "protocol_sha256": "55d6b907dd44613a5498dd7156f1ce9e78758835255bc7c336b4c327bb40e9bf" + "map_sha256": "e1be208d9420e39ccf3486eb8aa02ed76e778a5b77cae1615e71ce63562df75a", + "protocol_sha256": "1a8a3c2946b704a0b90f30a3f8e53e360f5dd3012ff2cc2d8e2ca1eb3062d397" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "55d6b907dd44613a5498dd7156f1ce9e78758835255bc7c336b4c327bb40e9bf", + "implementation_sha256": "1a8a3c2946b704a0b90f30a3f8e53e360f5dd3012ff2cc2d8e2ca1eb3062d397", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "55d6b907dd44613a5498dd7156f1ce9e78758835255bc7c336b4c327bb40e9bf", + "implementation_sha256": "1a8a3c2946b704a0b90f30a3f8e53e360f5dd3012ff2cc2d8e2ca1eb3062d397", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "e058159f89a01b6541e98ee57ffb0169eeff8ed2ea4fb1c664b3f3fb0c60a675" + "spec_sha256": "fa6d62e783446ada4998edaad7d182fc15e4b56763c9bf4778e2cf0594b1c1de" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "e058159f89a01b6541e98ee57ffb0169eeff8ed2ea4fb1c664b3f3fb0c60a675" + "spec_sha256": "fa6d62e783446ada4998edaad7d182fc15e4b56763c9bf4778e2cf0594b1c1de" }, "status": "pass" } diff --git a/docs/fiscal-leaf-policy.md b/docs/fiscal-leaf-policy.md new file mode 100644 index 000000000..823603506 --- /dev/null +++ b/docs/fiscal-leaf-policy.md @@ -0,0 +1,114 @@ +# Fiscal input policies + +The declared fiscal measurement stage requires an input producer for every +leaf in its static model closure. Missing and nonfinite values still refuse +evaluation. The optional leaf policy records more precise intent without +weakening that default. + +An explicit policy can describe producer ownership or a proposed literal +scenario assumption. **Assumption-bearing executions currently refuse with +`FISCAL_MEASUREMENT_ASSUMPTION_PARENT_ADMISSION_UNSUPPORTED`.** No US assumption +resource is enabled, and no dataset column or producer requirement changes. + +## Exact policy identity + +`load_fiscal_leaf_policy(payload, expected_sha256=...)` accepts bounded, +canonical JSON bytes and an explicit SHA-256 pin. It performs no file or +network I/O. The same `FiscalLeafPolicy` goes to `fiscal_measurement_node` and +`FiscalMeasurementKernel`. Its pin is rechecked when used; the parsed document +is detached. Omitting the argument, or passing `None`, preserves the previous +all-producer declaration shape and runtime behavior. + +A policy has schema version 1, artifact kind +`microcosm.us.fiscal_leaf_policy`, one positive integer `period`, an exact +ordered list of model `roots`, and an `entries` mapping covering the static +closure exactly. Missing entries and stale entries refuse. Each entry's +entity must agree with the independently derived model metadata. + +| Kind | Required entry fields | Behavior | +| --- | --- | --- | +| `producer` | `kind`, `entity`, `producer` | The declared input column must exist and every cell must be known and finite. | +| `assumption` | `kind`, `entity`, `value`, `interpretation`, `period`, `roots`, `affected_roots`, `affected_programs`, `rationale`, `reviewer`, `source_issue`, `reform_sensitivity` | Validated and inspectable; execution remains unsupported. | +| `inactive` | No supported shape | Always refused. | + +An assumption's interpretation is explicitly `baseline_behavior` or +`scenario_parameter`. Neither category means the program is insensitive to +the choice. The affected roots must include every declared root whose static +closure contains that leaf, in declaration order. Scope must exactly match +the policy's period and roots; program names and review fields cannot be +empty. These fields describe claimed review and intent. They do not prove +reviewer authorization, establish a source observation, or make an observed +attribute an acceptable assumption. + +V1 supports finite Boolean, signed 64-bit integer and floating-point literals +with explicit engine type checks. Strings, enums, null, containers and an +`engine default` reference refuse. The actual engine metadata and whether a +default is available (with its literal value when present) are recorded +beside the proposed value. The host never chooses a value by default lookup. +The exact policy and engine/default record enter `model_contract`, so policy, +scope, ownership, review or engine-default changes alter the declared node and +its cache key. The same declaration is embedded in a successful measurement +artifact. In this version, only producer policies can produce that artifact. + +A column constant at an engine default remains admissible as a producer at +this boundary. The label does not establish observation or modeled variation. +The source owner and existing release input-coverage/variation gates retain +that responsibility. Behavioral take-up and filing propensities are not +automatically classified as innocuous scenario settings. No prior wages are +added by this infrastructure. + +## Complete-parent admission still required + +The graph kernel receives only its declared input slices. It cannot prove a +column is absent from the complete parent when a caller omitted that column +from the slice. Declaration checks reject an assumption name present on any +declared entity, but this is insufficient to enable execution. + +The future country host must retain its independently checked complete +population and use `validate_fiscal_leaf_policy_population(frame, policy)` +before projection, around relevant I/O, and during cache admission. The +validator rejects any incumbent assumption name on any entity, including +wholly unknown or partially unknown columns. Passing a projection or a +caller-authored column roster is not absence proof. The validator itself +does not authenticate or issue a parent. + +`private_fiscal_assumption_frame(frame, policy)` is a detached preparation +helper for that future integration. It repeats the collision check and +creates engine-only tables. It preserves the supplied frame's values, +membership, weights, strata and metadata. The current fiscal kernel does not +call it. A future host must validate the exact policy against the admitted +engine, retain source and complete-parent checks before and after I/O, bind +the engine/default record, evaluate only the private copy, and keep assumption +columns out of the population and measurement adapter. A cached artifact +cannot supply missing source or parent authority. + +## Why inactivity is unsupported + +`validation_input_coverage._provision_input_leaves` traces a single empty +one-person simulation. It observes the leaves read by that execution. A +conditional formula or `defined_for` branch may read additional leaves for +other households. Absence from that trace cannot establish universal +inactivity, even at the same period and roots. + +Future inactivity support needs exhaustive engine-, period- and root-scoped +dependency proof, including conditional paths and declared state roots. The +static closure remains conservative; neither it nor a sampled trace supplies +this exemption today. + +## Bounded implementation evidence + +The local source/mock test run passed 91 cases, including the existing +invented fiscal measurement, calibration and observer graphs. The new controls +check strict-default compatibility, malformed policies, exact scope and +closure coverage, type mismatches, private-copy isolation, omitted-column +collisions, late policy mutation, graph key changes and required replay with +a forbidden model call. There were no country engine imports, network +connections or subprocess operations; no native population or source data +was used. This is infrastructure validation, not actual engine-policy +admission or candidate fiscal validation. + +The source and selected test files were sealed before and after the run. +Receipt SHA-256: +`c7aed643333638460d7464091be0c068b910baba2eefbb76cfa1208af6f8265f`. +JUnit SHA-256: +`54215b02c7a8a85488db48b2fee339b0ddbad907a0985502019c185b9a1c3adb`. diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md new file mode 100644 index 000000000..5793b0b1f --- /dev/null +++ b/docs/geography-assignment.md @@ -0,0 +1,261 @@ +# Geography assignment in the population graph + +Design decision, corrected 10 September 2026: finish constructing the full survey +multispine, including its initial support and PUF clones, then assign one small +census area to each resulting household. Derive larger geographies from that +area using identified, versioned mappings. Distinct clones may receive different +areas subject to their observed source constraints. The assigned location then +remains fixed through subsequent enrichment, calibration and analysis views. + +The earlier block-before-clone implementation and its passing controls remain +historical evidence. They do not demonstrate this corrected ordering. Geographic +assignment must use a stable post-clone identity, including a clone discriminator +where original-source identity alone is shared by multiple households. + +The corrected order now passes 70 controls on invented inputs, plus a separate +default-path compatibility check. The nine-node survey prefix expands the initial +clones before assignment; its nineteen-node financial and twelve-node age +extensions retain the assigned geography. Draw identity combines +`survey_geography_origin_key` with `household_support_clone_index`, so renumbering +households does not redraw their location. A typed validation artifact orders +financial enrichment after the geography gate. See the +[scoped acceptance](../experiments/us-postclone-geography-70-controls-20260910.json). +Native execution of this corrected order and population-quality acceptance +remain pending. The older results below retain their original source scope. + +## Declarative country capability + +Geography is a declarative country capability backed by shared graph operators. +A country declares its atomic area type, code system and vintage, source mapping +artifacts, observed-geography constraints and sampling-weight convention. The UK +declaration can select an area system by nation. Adding a country should not +require writing a country-specific assignment or derivation kernel. + +The shared assignment operator selects the atomic area using the declared +support and stable household identity. Shared lookup/join operations derive +larger geographies from identified mappings. If an input already has a qualified +atomic area, validate and retain it. Source acquisition and normalization may +still require publisher-specific adapters; they produce the common support and +mapping contracts, rather than owning a separate assignment algorithm. + +Sampling and derivation remain executable graph operations with typed inputs, +outputs and replay identity. A country configuration does not itself execute +them. Mapping edges retain their relation type, including exact nesting or an +explicit best-fit convention. Prefer existing graph lookup/join primitives where +they support these contracts; introduce only the shared behavior they lack. + +The existing country-spec geography declaration is a starting point, but its +legacy clone-and-assign contract and country-specific runtime references do not +yet implement this shared atomic-area contract. Preserve those compatibility +paths while the shared operators and country adapters receive their own checks. + +```mermaid +flowchart LR + S[Complete survey multispine including clones] --> A[Assign one small area per resulting household] + C[Observed survey geography constraints] --> A + L[Versioned small-area support] --> A + A --> D[Derive larger geographies] + M[Versioned geographic mappings] --> D + D --> E[Subsequent enrichment retains location] + E --> R[Rules and calibration] + R --> P[Pruned analysis file retains location] +``` + +## Country anchors + +| Country | Assigned household anchor | Derived geographies | +| --- | --- | --- | +| US | Census block, with census vintage | Block group, tract, county, state, PUMA and congressional district using the declared mapping/boundary vintages | +| England and Wales | 2021 Output Area | LSOA, MSOA, local authority, ward, constituency and region | +| Scotland | 2022 Output Area | Scottish statistical areas, council area, ward, constituency and region | +| Northern Ireland | 2021 Data Zone | Super Data Zone, local government district, constituency and nation | + +Northern Ireland's 2021 Data Zones replaced the 2011 Small Areas. Scottish +Output Areas use the 2022 census. These are different source systems; the graph +must retain the original area type and vintage rather than imply identical units. +See [NISRA census output geography](https://www.nisra.gov.uk/statistics/census-2021-results/census-output-geography) +and [Scotland's census general report](https://www.scotlandscensus.gov.uk/about/scotlands-census-2022-general-report/). + +Some larger boundaries do not nest exactly. England and Wales small-area estimates +use best-fit mappings for wards and parliamentary constituencies. The derived +edge must identify whether its mapping is exact, best-fit or another explicit +approximation, with its source and vintage. A mapping convention cannot be +presented as observed household location. See the +[ONS methodology](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationestimates/methodologies/smallareapopulationestimatesqmi). + +For US congressional districts, the Census Bureau's CD119 block equivalency +file assigns whole original 2020 tabulation blocks to districts. Colorado's +block `080010096072000` crosses the enacted CD07/CD08 boundary but is assigned +to CD08 for tabulation. Represent this as an official tabulation mapping, +not exact spatial containment. The same source distinguishes original 2020 +blocks from subsequently adjusted geometry and includes undefined `ZZ` areas. +See the [Census Bureau's CD119 documentation](https://www.census.gov/geographies/mapping-files/2025/dec/rdo/119-congressional-district-bef.html). +By contrast, the same-vintage tract-to-PUMA relation is nested, as described in +the [Census PUMA guidance](https://www.census.gov/programs-surveys/geography/guidance/geo-areas/pumas.html). + +## Required behavior + +- Respect observed source constraints, such as ACS PUMA and FRS region. Record + inferred geographic detail as modeled. Preserve observed source geography + separately from the assigned location. +- Bind each draw to a stable household identity, seed and assignment definition. + Reordering rows or selecting an existing household must not redraw its location. + Changed source support or boundary mappings produce a new identified revision. +- Assign household members consistently with their resulting household. Initial + clones have distinct draw identities and may receive different anchors. + Materialized county, district and other fields must agree with that household's + selected anchor. Source geography remains a separate observed constraint. +- Show assignment and derivation as separate operations in the graph. Expose the + source constraints, support, sampling weights, mapping conventions and judgment + annotations in the inspector. +- Refuse missing, ambiguous or incompatible mappings unless an explicit, + source-supported resolution rule is part of the assignment definition. +- Verify inheritance and mapping consistency again on every pruned export. + Engine input profiles and geographic build invariants are separate contracts. + +## Existing implementation and next changes + +The shared implementation now lives in `microcosm.build.atomic_geography` and +`microcosm.build.graph_atomic_geography`. It provides four country-neutral nodes: +`geography.support_import@1`, `geography.assign_atomic@1`, +`geography.derive@1` and `geography.gate@1`. Countries supply declarations and +normalized, pinned support files. The graph interfaces and legacy country +operators are unchanged. Country configuration and native support integration +remain necessary before these nodes can assign locations in a release build. + +The support contract uses deterministic NPZ bytes with no object arrays. It has +one unique string code per atomic area; every mapping column identifies its +source, vintage and relation, and every integer sampling-weight column identifies +its source and basis. Nonnegative weights have a bounded exact total. A source +adapter must resolve any nonfunctional crosswalk before admission; the shared +lookup never multiplies households through a many-to-many join. + +Assignment conditions on all nonmissing observed constraints jointly. Declared +stages can use different weights—for example, household counts to select a +constituency followed by population counts to select its small area. Each draw +uses the existing `keyed_uniform` protocol with stable source identity, the +declared stream, system, stage, assignment definition and support digest. +Selection uses an integer inverse CDF. Reordering, subsetting or adding unrelated +households preserves draws. The numerical declaration remains `platform_bitwise`; +cross-platform equivalence has not been established. + +The version-1 declaration lists `identity`, `stream`, three assignment `outputs` +(`area`, `system`, `basis`), and `systems`. Each system supplies its atomic +identity and source, a selector, observed constraints, an optional observed-area +column, sampling stages and derived layers. Geographic codes use nullable +strings throughout, retaining leading zeroes. An engine that needs integer +storage must declare that conversion separately. A system without a particular +layer produces missing values for that layer, which the integrity gate checks. +The graph stores the canonical declaration in its normative `definition` +parameter and exposes the stream separately; no graph parameter grammar changed. + +The 34 invented-data controls include real graph execution, cold and warm reuse, +source-change invalidation, exact lookup metadata, independent sampling-boundary +checks, three-system routing, observed-area retention, row/subset stability and +clone/prune validation. All passed under the isolated control runner on +9 September 2026. These checks establish shared operator behavior; they do not +admit a Census/ONS source or certify any population's geographic fit. The gate +checks mapping integrity, while the executor and build own structural lineage +and population-quality acceptance. The country's chosen identity still needs +verification across actual sampling rungs. + +The US development composed graph attaches geography after harmonization, but +`us_runtime/graph_geography.py` selects a joint tract/congressional-district cell +and emits PUMA, county and district. It does not assign a Census block. Its prior +national source and joint-support checks therefore do not establish block-first +acceptance. The separate atomic survey graph now provides that block assignment +and derivation connection before enrichment. Its population-only national block +support and source review are recorded below. + +The UK already has an area-based ladder in `uk_runtime/geography_ladder.py`. +Its sampler selects a constituency within FRS region using household counts, +then an OA within constituency using population; one selected ladder row supplies +the other geographies. It currently consumes a shared seeded random stream, so +stable household-keyed assignment under reordering and subsets still needs work. +The graph should expose the final OA/Data Zone assignment and the mapping +derivations explicitly, including that sampling convention. + +The Northern Ireland source builder currently infers Data Zone constituencies +using the modal active-postcode constituency. Review the official +[NISRA constituency aggregation and lookup resources](https://www.nisra.gov.uk/publications/census-2021-output-geography-information-papers) +before choosing or replacing that approximation. Source existence is not +acceptance of a particular lookup or its application to the population. + +The US block adapter in `us_runtime/atomic_block_support.py` now preserves each +supplied block, leading zeroes, population weight and the source-labeled +tract-to-PUMA and block-to-district mappings. Its declaration uses the shared +operators, preserves observed state/PUMA separately, and labels CD119 as +`official_tabulation`. Its sampling proxy is 2020 persons; this does not imply +household counts or support for subsequent construction in unpopulated 2020 +blocks. All 13 invented tests passed, including missing/inconsistent mapping +refusals and subset-stable assignment. Receipt SHA256: +`404deb7666b3ac9b8a654181228c545ab95c131f30a1fbe65a2e15d5a08a4c4c`. +These initial adapter tests do not establish native geographic fit. Subsequent +national support and native pilot results are recorded below. + +The earlier `us_runtime/graph_atomic_survey_clone.py` declared this sequence: +shared import, assignment, derivation and integrity gate, then the existing +combined-survey support clone and an inherited-mapping gate. The graph compiler +makes the clone depend on every member of its base version, including the +pre-clone gate. The post-clone gate never draws another location. An actual +executor control passes cold execution and required replay: four invented +households become eight, every location column is inherited, all entity rows +are copied, and each household weight is split equally across the pair. +Subset mapping verification also passes. Receipt SHA256: +`1bb75a5ee53edcbdd199a6d0163cbe75c32e54c80513cb3eb8d13021c3c2b1c3`. +A separate source-only Codex review found no actionable defects. + +The current survey path now qualifies observed state and ACS PUMA from the +retained original-source preparation. Eleven source controls and fourteen graph +controls passed, covering missing geography, source changes, replay and stable +household identity across sampling rungs. These controls use invented originals +through the actual source issuers. Their receipts are recorded in +`experiments/us-survey-geography-source-controls-20260909.json` and +`experiments/us-survey-geography-graph-controls-20260909.json`. + +The earlier `us_runtime/graph_atomic_survey_population.py` connected that projection +to block assignment, geographic derivation, the integrity gate and the combined +survey clone. It retains the raw allocation separately from the enriched +pre-clone population, even though both share the allocation version identifier. +The complete ten-node prefix passed five controls: fresh execution, required +replay, full geography inheritance, support-byte mismatch and late mutation +refusals. The test population contains six survey households before cloning and +twelve afterward. The exact tested revisions and remaining checks are recorded +in `experiments/us-atomic-survey-population-controls-20260909.json`. + +The earlier calibration budget and age runner consumed this optional prefix +and independently reconstructed it from the raw allocation. Eight budget +controls passed, as did the complete thirteen-node age graph through fresh +execution and required replay. A separate compatibility control verifies that +the existing predictor and PUF-host qualifiers still accept the default survey +prefix. See the corresponding `us-atomic-budget-semantic-8`, +`us-atomic-age-v2-1`, and `us-budget-predictor-compatibility-1` records in +`experiments/`. + +Replay can canonicalize the hidden backing values of missing numeric cells. +Persistent budget identities therefore bind logical Frame values, population +version, complete ownership, weight kinds, mass ledger and exact design-weight +bytes. They exclude only the helper receipt's physical population stamp. +Same-object mutation checks retain physical identity. The initial physical-stamp +replay failure remains recorded; the corrected budget controls pass. + +The byte-source adapter passed thirty-six invented-source controls through the +maintained PL, CD and PUMA parsers. This does not authorize acquisition of native +PL archives, which also contain housing segments. Native support preparation +will instead request only Census `P1_001N` block populations and independent +state totals, then join CD119 and tract-to-PUMA mappings. Delaware is the first +source control; national survey assignment requires complete admitted support +for the fifty states and DC. No native acquisition is accepted by these tests. + +`us_runtime/atomic_block_api_sources.py` implements that population-only response +adapter. All thirty-five invented controls pass, including independent state +totals, exact block preservation, missing mappings, malformed response fields, +byte limits and late source-record mutation. Request descriptors preserve the +repeated Census `in` parameters and omit credentials. The adapter does not infer +the response's origin from its contents; acquisition and publisher qualification +remain separate. Evidence: +`experiments/us-atomic-block-api-sources-35-controls-20260909.json`. + +Publisher-qualified native block support, national and district calibration, +and pruned export verification remain necessary for a release. Source integrity +checks and successful invented-data runs do not establish geographic fit. diff --git a/docs/graph-signed-reconciliation.md b/docs/graph-signed-reconciliation.md new file mode 100644 index 000000000..8cba5aceb --- /dev/null +++ b/docs/graph-signed-reconciliation.md @@ -0,0 +1,57 @@ +# Signed component reconciliation in the graph + +`microcosm.fit.graph_signed_reconciliation.signed_reconciliation_node` exposes +the existing weighted Euclidean projection as an ordinary graph calculation. +Its parameters name the anchor, input draws, output components, nonnegative +bounds, positive scales and numerical tolerances. The country model must +declare and justify those choices. The operator supplies no default scale or +tax interpretation. + +For the proposed ACS property-income family, the components are ordinary +interest, retirement-account interest, dividends and signed property income. +The first three have nonnegative bounds; property income can offset them. +A zero reported total therefore does not imply that every component is zero. +Retirement-account interest remains a separate survey component and is not +relabeled as taxable or tax-exempt interest. Source qualification, a defensible +ASEC-to-ACS measurement bridge, fitting and tax conversion remain separate +upstream/downstream operations; this numeric node alone does not complete them. + +The graph retains the anchor and raw draw columns. It adds the reconciled +components, each component's adjustment and bound activity, the sum residual +and the projection objective. A typed summary artifact records the complete +rule, row count, adjusted-row count, maximum absolute residual and bound counts. +It describes the numerical result and grants no source or release authority. +These declarations and columns make the rule inspectable in the shared graph +viewer without another viewer implementation. + +All input amounts must be finite float64 values. Unknown anchors or draws +refuse; the country host must declare how it selects or models those cases. +Inputs cannot share names with the new outputs, and no original observation +is overwritten. The kernel never reads survey origin, weights or an RNG and +returns no membership or weight changes. Its numerical contract is bitwise on +one platform; the explicit projection tolerances test constraints within each +run and are not a claim of cross-platform equality. + +## Verification on 12 September 2026 + +Twenty-four guarded tests pass, including direct comparison to the independently +tested pure projection; negative and zero-net anchors; exact IDs above 2^53; +row permutation; missing, infinite and incorrectly typed inputs; invalid +bounds/scales and column collisions; and declaration/context disagreement. +An actual two-node graph executes cold, retains the complete input Frame, +metadata and zero-weight record, then reopens the store under required replay. +The replay hits both nodes and preserves artifact identity. Changing scales +reuses the source node and invalidates the reconciliation node. + +The closed run used 3.69 seconds wall time and 417,775,616 bytes peak RSS. +All 1,000 source and eight owned hashes, plus 15 resource hashes, were unchanged; +there were no unexpected denied operations or child processes. These are +invented-data graph checks, not native Microcosm acceptance. Receipt SHA-256: +`7e0ed8fefc911b9d7969f48e0d2c382c189e40a0427a120a163a8ce9c8ffae01`. +JUnit SHA-256: +`914745d61c768ae4f445df408db89601d4a4268374ba4f0e6caba6085ea6be6c`. + +The first run preserved 22 passing tests and two fixture failures: the test +attempted an empty `Weights` object and constructed unsorted group IDs. +Correcting those fixture inputs produced the result above; production source +was unchanged between the two runs. Independent review is recorded separately. diff --git a/docs/joint-empirical-graph.md b/docs/joint-empirical-graph.md new file mode 100644 index 000000000..bccfee1a2 --- /dev/null +++ b/docs/joint-empirical-graph.md @@ -0,0 +1,118 @@ +# Joint empirical graph operations + +The reusable adapter fits a weighted paired empirical law and draws from it with +the maintained keyed-uniform protocol. It uses `joint_empirical.py`; it does not +fit QRF or independently sample two marginals. The source host selects and +authenticates donors and recipients, supplies an explicit support policy, and +independently verifies draws before attaching them to a survey population. + +This source implementation depends on the numerical operator introduced in +`aaa8b071ab14a97e514deb05d367f74a964abb29`. Its tests are authored but unrun. + +## Private support branches + +The 2026-09-13 root decision clarifies the initial child implementation plan: +independent private model-support CREATE branches are allowed so their exact +column schemas appear in graph Slices. They do not change the receiving survey +population, original spine, clones or geography. There is no new EXPAND operation. +The US source owner creates those branches from its retained qualified source. +The reusable adapter owns only the fit and draw operations. + +Side-table entity IDs must be exact unique int64 values. They identify rows when +results are joined back and are excluded from the declared random coordinates. +Original source coordinates occupy separately declared int/string columns. No +float or Boolean source coordinate is accepted. The host must validate both the +original coordinate and the original-person join; a matching side-table ID alone +does not confer source authority. + +## Fit contract + +`joint_empirical_fit_node` declares one all-row Slice with the two ordered target +columns, original donor and household key columns, and explicit weight column. +Targets and weights are nonnullable float64; original keys retain their integer +or string types. Donor keys must be unique, while household keys may repeat. +All numeric validation and support rules execute in the real numerical model. +The input weight kind is recorded as `explicit`; the host separately documents +and authenticates any original household design-weight provenance. + +The node consumes a typed `source_projection` artifact whose payload hash is an +explicit parameter. This is the full donor projection, distinct from the selected +recipient projection. The complete support document is a canonical JSON string +plus its SHA256 in normative parameters. No production support default exists. + +The node emits the numerical operator's exact private `model` bytes and a typed +`model_metadata` artifact with model/support/donor-source hashes, target names, +explicit weight kind and aggregate diagnostics. Model bytes contain paired donor +values and exact tagged private coordinates. Metadata contains no donor IDs. +Changing selected recipient sampling does not enter the numerical model bytes. + +## Draw contract + +`joint_empirical_draw_node` declares the original-recipient coordinate Slice. It +can also declare an exact nonnullable Boolean `eligibility_column`. The support +Frame then contains all real original recipients; the kernel selects eligible +coordinates internally, with candidate/draw counts in its receipt. An all-false +column produces a legitimate zero-row draw while retaining the nonempty support +Frame. This uses no sentinel person and does not relax Frame or Weights: a +row-masked empty Slice would itself require forbidden empty projected weights. +An empty direct DataFrame test is only a numerical context test, not this graph +path. Invalid or duplicate unselected coordinates are still refused. + +The node +consumes model and metadata from the same producer and the separate typed +recipient `source_projection`. It checks platform scope, artifact store identity, +the numerical model's complete plain-data encoding, support agreement, donor +source hash and recipient projection hash. + +Parameters include the complete stream `(algorithm, experiment_id, replicate, +base_seed)`, a coordinate suffix, the complete canonical transport document and +hash, and explicit source/support/scenario revisions. The US child candidate +uses suffix `('child-property-v1',)`. The adapter appends `pattern` and `donor` +separately and calls the real `keyed_uniform`. It declares `SeedSource.KEYED` and +does not consume the node-key-seeded sequential generator. + +Only the original source coordinates and explicit suffix enter the stream. +Side-table IDs, clone numbers, row position, batch boundaries, model node keys, +recipient sample size and scenario hashes do not. The scenario hash is a host +revision reference, not authority over its semantics; the actual transport and +stream parameters are independently normative. Scenario comparisons share +uniforms while remaining distinct content-addressed graph operations. + +The private `draws` artifact records exact tagged original coordinates, decimal +int64 side IDs, paired float64 values, patterns, positive donor references and +all model/support/source/scenario/transport bindings. Rows have canonical +coordinate ordering. Pattern zero has no sampled donor identity; it remains a +model zero rather than an observed nonreceipt. No survey columns are written. + +`read_joint_empirical_draw` checks exact supplied side IDs and original keys plus +the expected hashes and parameter documents. It returns values in the supplied +order as bytes-backed readonly arrays. This validates the serialization and join +contract, not the draw law: a host must reconstruct the actual model and keyed +draws from retained qualified source before accepting canonical attachment. + +## Verification and limits + +The invented tests cover actual numerical fit/draw calls, unequal-weight paired +probabilities, exact large IDs, integer/string distinctions, row permutations, +partitions, recipient additions, side-ID changes, common scenario uniforms, +typed artifact/producer/hash changes, malformed private joins, explicit zero +stress and empty recipient batches. A real four-node cold/required replay checks +both model-support Frames and scenario-specific draw keys. No test grants survey +authority, and no native fit or production support policy is claimed. + +The numerical model is capped at 64 MiB and 1,048,576 donors. Private draw JSON is +also capped at 64 MiB and 1,048,576 recipients; the byte check occurs after Python +object construction. These are ceilings, not measured national capacity. Before +large use, measure bytes per row, working-memory peaks and repeated validation +cost on small samples. A separately reviewed compact transport may be needed. +Implementation identity includes this adapter, the numerical model, weight +resolver, keyed-randomness module and its canonical serializer implementation, +with NumPy/Pandas platform-bitwise scope. Serialized metadata and draw-header +bindings compare canonical bytes, preserving integer/Boolean/float distinctions. + +The corrective source tests add valid changed-model/stale-metadata refusal, +noninteger encoded stream/count refusal, canonical source identity and actual +compiler-key changes for source/support/transport/stream revisions. The actual +four-node runtime test now covers all, some and zero eligible recipients with +cold/required replay and refusal of corrupted invented cache payloads. These +tests are authored and UNRUN pending a separately approved bounded runtime. diff --git a/docs/native-hours-source-owner.md b/docs/native-hours-source-owner.md new file mode 100644 index 000000000..bbdabbfb6 --- /dev/null +++ b/docs/native-hours-source-owner.md @@ -0,0 +1,109 @@ +# Native hours source ownership + +`qualify_current_survey_hours(preparation, age15_policy=..., under15_policy=...)` +connects the pure usual-hours proposals to the original ACS and ASEC source +records held by an authenticated survey preparation. Version 2 returns complete +selected-original-person hours for both arms. It preserves source observations +and labels explicit completion and empirical imputation separately. The source +qualifier does not modify a Frame; the graph fragment described below owns +attachment to the receiving population. + +The function captures the preparation's pinned ACS person archive and current +ASEC person member, checks their sizes and hashes, exhausts every CSV record +and archive member, and joins selected people by original survey coordinates. +Original ages must match the preparation. It derives the complete income-2024 +age-15 donor key set from the issued ASEC coverage before selection and requires +the captured member's donors to match that exact set. The production roster has +2,174 people; matching that count alone does not qualify a different roster. + +The age-15 empirical and under-15 zero-completion policies are explicit arguments. +The pure proposal module supplies their versioned names, seed, native-keyed draw +and interpretation of raw codes. Source allocation and response flags remain +distinct from modeled provenance, including valid `FL_665=0` supplement +nonresponse. Both future clones should receive the same original-person draw. +This source owner neither creates clones nor assigns hours to them. + +The returned `QualifiedSurveyHoursProposals` holds private raw tables, proposals, +original source owners and an immutable receipt. Call `validate()` before +borrowing the result and after final relevant I/O. It checks implementation, +retained source identities, exact projection objects, table storage and nested +proposal contents, including ASEC earnings and the combined person projection. +A copied object, arbitrary callback, matching detached table +or receipt alone does not replace the retained object. Raw records and donor +assignments are private evidence; public diagnostics should use aggregate counts +and provenance summaries. + +The receipt qualifies selected original-person hours and the ASEC own arm. It +deliberately leaves `all_person_engine_input_qualified`, `source_admission_issued` +and `release_eligible` false. The pure batches' scope flags remain false as well; +the enclosing live source owner supplies the checked relationship rather than +changing those flags. `person_hours` preserves the original-person index order +and includes the target, provenance and policy columns. The ACS `proposals` +and separate `asec_proposals` retain native keys and literals for inspection. + +ASEC hours use the [separate observation and completion rules](asec-usual-hours-observations.md). +The under-15 zero assumption requires explicit policy and known-zero source +earnings; it is never relabeled observed zero. Missing adult hours refuse complete +projection construction. Historical engine defaults, last-week hours and prior +wages are not substitutes for current source evidence. + +`borrow_cloned_hours_columns(qualified, receiving_people)` reuses the existing +exact two-clone attachment contract. It transports each original proposal and +its provenance to both descendants without redrawing hours. It checks original +IDs, native IDs, survey channels, complete clone pairs and output ownership. +It validates the retained source owner before and after borrowing, then checks +that source I/O changed neither the recipient table nor the borrowed output. +The returned Series are independent copies. This helper does not issue receiving +population authority: the graph host retains and authenticates its own receiving +owner and binds these columns to execution and replay. + +## Graph execution and replay + +`graph_current_survey_hours` declares four ordinary operations on the receiving +population. `survey_hours.source` retains the authenticated source projection in +a private artifact. `survey_hours.asec_recode` executes the ASEC recoder and +explicit under-15 completion. `survey_hours.acs_recode_impute` executes ACS +recoding, the original-keyed age-15 empirical draw and under-15 completion. +`survey_hours.attach` checks the two column artifacts against the independent +source-owner result and attaches hours, provenance and policy to both clones. + +The attachment reads only the four ancestry and survey-channel columns. It owns +`weekly_hours_worked_before_lsr`, `hours_provenance` and `hours_policy`; existing +ownership refuses before execution. Public graph parameters contain source +hashes, survey periods, policy identifiers and the seed. Raw literals, identifiers +and donor assignments remain in private artifacts. + +`graph_us_survey_enrichment` appends these operations after housing attachment. +Its final population is the hours attachment output. On cold execution and +required-cache replay it compares every hours artifact and intermediate +population with expectations derived from the independent retained source +owner. The host checks current source and implementation ownership after final +store I/O. Its receipt records the hours source and attachment hashes and both +policies, while release eligibility remains false. Prior wages are not inputs. + +## Verification boundary + +The tests use the real source issuers, preparation, private capture and owner +validation with tiny invented archives. A selected sample deliberately excludes +the sole age-15 donor household while the full donor cohort remains available. +Tests compare full versus selected proposals and cover explicit policy choices, +malformed/duplicate records, detached owners, replaced callbacks and receipts, +table and nested-proposal mutation, retained coverage mutation, changed seed, +and projection mutation during final source I/O. The small graph tests also run +cold and required-cache execution, check clone equality after reordering and +preserve existing tables, memberships and weights. They refuse changed private +artifacts, copied or revoked owners, output collisions and mutation during the +final host callback, and check that policy changes invalidate downstream keys. +The fixture's one-donor limit +is a private pre-issuance test seam; production retains the 2,174-key requirement. + +This change continues the native branch based on `fee4aac9d`, including pure +hours and response-status correction `4f34495e5`. Fresh main was inspected and +lacks these native APIs, so it is not an independent main-branch replacement. +Actual full-source owner and integrated-host execution, calibrated release +comparisons and publication remain separate. The hours fragment has been tested +without rerunning the financial and full PUF fits; that bounded test does not +establish native production qualification or David's hours-distribution gate. +A complete +all-age ASEC source diagnostic supports the recoder but does not replace the +retained source owner or qualify its production execution. diff --git a/docs/native-primary-family-source.md b/docs/native-primary-family-source.md new file mode 100644 index 000000000..682b70078 --- /dev/null +++ b/docs/native-primary-family-source.md @@ -0,0 +1,89 @@ +# Native source household classification + +`current_survey_primary_family.classify_primary_household` classifies one complete +original household. `current_survey_primary_family_source.qualify_current_survey_primary_family` +borrows the existing authenticated survey preparation and householder projection, +reads the pinned original members, and returns a detached household table and +descriptive receipt. It grants no source admission, population admission, graph +attachment, calibration, or release eligibility. Future graph integration must +requalify against the live source after relevant I/O and use the maintained exact +householder/source-to-clone transport. + +## Definitions and official source mappings + +The [2024 ACS subject definitions](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf) +define own children as never-married biological, adopted, or stepchildren of the +householder under 18 (printed page 89). A household has at most one primary +family, and its married-couple classification requires the householder and spouse +to reside together (page 90). A related subfamily does not add another family to +that count (page 91). This classifier follows that household concept, rather than +the broader employment-tabulation child universe. The spouse relationship includes +both opposite-sex and same-sex couples. + +The [2024 ACS PUMS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.txt) +maps `RELSHIPP` 20 to the householder, 21/23 to the spouse, and 25/26/27 to +biological/adopted/stepchildren. `MAR=5` establishes never-married status; +`AGEP` supplies age. `TYPEHUGQ=1` identifies housing units and 2/3 group quarters. +`NP` must match the complete retained roster. Spouse and unmarried-partner +relationships appearing together are unresolved. ACS does not publish the general +resident spouse-line pointer needed here to count secondary couples; that result +is explicitly unavailable, even when the primary couple is known. + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +maps `A_EXPRRP` 1/2 to the reference person, 3/4 to the spouse, and 5 to own +children (printed page 6C-2, PDF page 23). Code 6 is unnamed and stays unresolved. `A_MARITL=7` +supplies never-married status; `A_AGE` supplies age. `A_SPOUSE` is a line number +(page 6C-1), resolved reciprocally within `PH_SEQ` using `A_LINENO`. Missing, +out-of-household, self, asymmetric, and marital-inconsistent pointers stay unknown. +All valid resident pairs are counted separately from the primary family. + +ASEC has no original `TYPEHUGQ` field. `H_LIVQRT` 1–7 identifies housing units; +8–12 identifies other units (page 6A-2). `HRHTYPE` 9/10 denotes group quarters +(page 6A-3); disagreement between those classifications is unavailable. The +classifier also requires an interview (`H_HHTYPE=1`), `H_NUMPER` equal to roster +size, and the independently qualified reference person. Household-type marriage +codes are a consistency check, never a substitute for spouse evidence. + +The [CPS subject definitions](https://www.census.gov/programs-surveys/cps/technical-documentation/subject-definitions.html) +also require never-married status for own children under 18. CPS published counts +may include college-dorm residents associated with a family; this projection +describes retained resident household rosters and does not claim equality with +every CPS published family universe. + +## Outputs and unknowns + +The household table carries nullable integer columns: + +- `census_household_count` and `census_household_population`; +- `census_married_primary_family`; +- `census_married_primary_family_own_child_u18` and + `census_married_primary_family_own_child_u6` (at least one own child under 6); +- `census_primary_married_own_children_u18_count`; +- `census_household_size_1` through `_6`, and `_7_plus`; +- `census_all_resident_spouse_pairs`, a separately qualified ASEC measure. + +Separate household, primary-family, own-child, and spouse-pair status columns +distinguish qualified values, exclusions, and missing or inconsistent evidence. +Known nonhousing units contribute zero to household-family/size indicators; +their pair count is unavailable. Unresolved universe evidence produces unknown +metrics. A known housing unit with an incomplete roster retains a household count +of one, but has unknown size/family metrics and an explicit incomplete-roster +status. Size bins partition the complete qualified housing-unit cohort only. + +For a known nonmarried primary household, married-family child measures are zero. +For a married primary family, missing age or marital status of a potential own +child remains unknown. Grandchildren, in-laws, foster children, and unrelated +children do not become own children through age, tax dependency, SPM membership, +or co-residence. Native identifiers are handled as literal strings or exact Python +integers without float conversion. + +The receipt records exact readsets, source and householder receipt digests, +source-year origins, output digest, and status counts. Published values may include +Census allocation; no unallocated-observation claim is made. Original source +mutation invalidates the existing preparation, and the qualifier reconstructs +all metrics/statuses after its final owner/source I/O. Returned tables are +detached values; their presence is not retained live authority. + +This change adds no targets, tolerances, weighted estimates, status draws, or graph +nodes. It does not establish calibrated household composition or statistical +acceptance against a published table. diff --git a/docs/native-usual-hours-proposals.md b/docs/native-usual-hours-proposals.md new file mode 100644 index 000000000..0b5f2dd57 --- /dev/null +++ b/docs/native-usual-hours-proposals.md @@ -0,0 +1,71 @@ +# Native usual-hours proposals + +`microcosm.build.us_runtime.current_survey_hours` provides a pure first slice +for a future native `weekly_hours_worked_before_lsr` graph leaf. It reads no +files, changes no Frame, authenticates no source and grants no release authority. +The implementation continues reviewed native source +`fee4aac9d7be2643fe02ec6b855d231abe5980a4`; that continuation contains APIs not +yet available on main. No parser, native owner, graph host, benchmark, dependency +or accepted-source pin changes in this slice. + +`recode_acs_usual_hours(row)` accepts literal source fields for one original +2024 ACS person. It preserves positive WKHP, including40 and99; retains FWKHP +allocation status; and distinguishes source-supported nonwork from unavailable +survey-universe values. FTP blank is NIU; numeric0 is not a valid WKHP answer. +Under16 alone does not establish zero work. Current ESR and last-week hours +are not used. Missing required headers and contradictory codes refuse. + +`recode_asec_age15_hours(row)` accepts one supplied original March2025/income2024 +age15 donor. It checks HRSWK/WKSWORK and final WRK_CK/WTEMP history, allowing +temporary work after initial WORKYN2. Source allocation flags stay separate. +The [Census 2025 FL_665 codebook](https://api.census.gov/data/2025/cps/asec/mar/variables/FL_665.json) +defines 0 as complete supplement nonresponse, alongside codes 1, 2 and 3. +All four codes are retained as valid source response-status evidence. This +recoding does not exclude nonrespondents or relabel their values as observed; +donor-cohort qualification remains the source owner's separate responsibility. +WTEMP0 is NIU, not an explicit no: positive hours/weeks and final WRK_CK1 remain +consistent with WORKYN2 unless WTEMP2 explicitly contradicts work. This retains +the reviewed donor rule, without claiming any supplied record actually occurred. +The recoder requires positive raw MARSUPWT integer units; it does not qualify +that weight or donor identity against actual files. This is deliberately not +the all-age ASEC hours producer. + +`propose_acs_usual_hours(rows, donors=..., age15_policy=..., +under15_policy=...)` returns frozen proposals for original source people. +Rows carry explicit raw literals, source/construction labels and optional donor +identity/allocation evidence. Source values are preserved; missing adult hours +refuse. Age15 completion requires explicitly selecting +`acs-age15-empirical-full-asec2025-v1`. Under15 modeled zero separately requires +`us_hours_under15_zero_completion_v1`, refuses positive wages or nonzero +self-employment, and retains unknown earnings as unknown. Neither policy is +enabled by default. No missing-hours40 fallback exists. + +The empirical CDF arithmetic and seed20260914 follow the previously reviewed +pure hours helper. Draws now bind original native source identity, not historical +population row positions. ACS keys contain survey/income year, literal SERIALNO +and numeric SPORDER. ASEC donor keys contain survey/income year, PH_SEQ, PERIDNUM +and A_LINENO. Duplicate keys and inconsistent duplicate ASEC coordinates refuse. +Ties in donor hours use canonical native identity; recipient and donor ordering +or batching do not change draws. These new draws need not match dense historical +draws. A later clone adapter must transport each original person's proposal +through checked ancestry instead of independently drawing for clone identities. + +The helper labels every supplied donor cohort **unauthenticated**, even when it +has2174 rows. The future source owner must bind the complete reviewed March2025 +cohort, actual source bytes and roles, native joins, allocation/history projection, +policies and implementation; counts and caller-provided hashes cannot establish +that authority. The chosen full-source empirical model is not a generic ASEC15+ +fallback, and these proposals do not attest model quality for native recipients. + +This slice preserves the separation between the ACS native value, its allocation +status, source nonwork completion, modeled under15 zero and empirical age15 draw. +WKL allocation status is outside its supplied fields, so no unallocated-answer +claim follows from source nonwork. Prior wages are not consumed. Last-week hours, +entity structure, weights, geography and other inputs are outside this helper. + +Invented tests cover code/universe refusals, source40/99 preservation, explicit +policies, temporary-work history, allocation provenance, empirical boundaries, +duplicate native identity refusal and permutation/batch invariance. Native +source qualification, parser retention, graph materialization/replay, per-origin +recipient signal gates, actual-data validation and release acceptance remain +separate work. diff --git a/docs/puf2015-canonical59-and-growth.md b/docs/puf2015-canonical59-and-growth.md new file mode 100644 index 000000000..36733c8fe --- /dev/null +++ b/docs/puf2015-canonical59-and-growth.md @@ -0,0 +1,74 @@ +# PUF 2015 canonical donor and 2024 transport + +These source owners turn the statistical 2015 PUF into an explicit return-level donor with 59 canonical outputs. They preserve raw fields, source status and documentary decisions separately from modeled values. The six detailed mortgage balance, interest and origination-year outputs belong to the downstream SCF stage. + +```mermaid +flowchart LR + A[Authenticated PUF typed source] --> B[Observed and derived return fields] + B --> C[Eleven explicit baseline models] + C --> D[QBI model in 2015 money] + D --> E[Source-specific 2024 monetary transport] + E --> F[Canonical donor with 59 outputs typed donor] + F --> G[Eight-feature PUF59 donor interface] +``` + +The source codec retains all ordinary returns, including those missing the demographic supplement. Disclosure aggregates retain their source lexemes but are excluded from individual donors. Neither a demographic gap nor zero reported exemptions means that a source return is dropped. + +## Count measurements + +PUF 2015 MARS 2 combines joint and qualifying widow(er) returns. The matching predictor maps the survey's separate widow category into that combined class. It is named `puf_2015_filing_status_code`; generic model filing status is unchanged. + +The corresponding `puf_2015_capped_return_size` equals one plus an extra unit for MARS 2 plus the sum of the four reported dependent-category counts. The source owner validates their disclosure caps by filing class. On the survey side, the matching measurement caps the reported dependent total before adding the filing-class units. The result is a statistical predictor, not a count of physical people or co-residents. Actual survey membership remains unchanged. + +The separate `puf_person_incidence_capacity` is one for return-level modeled zero/one QBI incidence. It is used only to validate the canonical outcome representation and is excluded from the eight fitted predictors. + +## Observed fields and explicit models + +The decoder emits 33 directly mapped or algebraically derived baseline fields. Eleven additional baseline outputs explicitly model source gaps: a normalized interest split, a component-neutral Social Security carrier, desired contributions proxied by realized deductions, incomplete tuition and employee-expense proxies, and active-partnership earnings. These transformations retain their assumptions in the receipt. + +The interest model uses the existing [IRS 2015 Table 2.1 workbook](https://www.irs.gov/pub/irs-soi/15in21id.xls). It normalizes the four published component amounts within the source AGI band. Mortgage interest, points and mortgage insurance share the available home-interest leaf; the investment component alone enters investment expense. Complementary rounding preserves the source total exactly. This does not create a mortgage balance, origination year or observed loan structure. + +The QBI owner reproduces the archived data model's assumptions with RECID-keyed random draws, retaining a reusable full-cohort employee calibration. Its 15 added leaves and Schedule C replacement are modeled inputs. The statutory engine owns deductions and limits. Ordinary and SSTB Schedule C branches are summed for the donor's total self-employment predictor, matching the survey measurement. + +## Source-specific monetary transport + +The publisher's statistical 2015 convention is the input basis. Raw filing-year/month fields remain intact; the transform does not apply another row-year CPI adjustment. + +Wages and modeled W2 amounts use the [SSA average wage index](https://www.ssa.gov/oact/cola/AWI.html). Social Security carriers use [SSA COLAs](https://www.ssa.gov/oact/cola/colaseries.html) received from January 2016 through January 2024. This transports fixed entitlements and does not predict changes in recipient composition. + +Other income families use the observed 2015 and 2023 national amount/count cells of [IRS Table 1.4 and 1.4A](https://www.irs.gov/statistics/soi-tax-stats-individual-statistical-tables-by-size-of-adjusted-gross-income), with positive and loss magnitudes kept separate. The 2023→2024 step uses the [BLS annual CPI-U series](https://www.bls.gov/cpi/tables/supplemental-files/historical-cpi-u-202412.pdf), explicitly assuming stable real conditional amounts for that year. Expenses, UBIA and selected mixed model outputs use an explicitly named CPI proxy. Every monetary field has a declared rule; incidence fields are unchanged. + +Partnership and S-corporation reporting counts overlap, so their dollar components are summed with the common all-return denominator. Dividend residuals use the ordinary-dividend reporting universe and do not subtract overlapping recipient counts. + +The PUF short- and long-term capital fields remove the effect of carried losses. Published net-loss means that include carryovers cannot be substituted directly. The model uses current-year gross gain/loss components per all return as a declared proxy for signed net flows. The evidence retains both the carryover-adjusted reported net and component-difference net, including their unresolved residual. It does not attribute that difference to rounding without evidence. + +The observed series are source facts. Their application to individual donor amounts, the related-family substitutions, and the last-year bridge are modeling decisions. They do not establish forecast accuracy or current reporting incidence. A same-cohort CPI-only sensitivity is available. Source design weights are divided from S006 hundredths once and are never grown. + +## Validation status + +The ordinary verification executes real source decoding, the QBI model, growth, the deterministic typed envelope and the PUF59 donor interface at 64 and 2,048 invented records. It checks all 59 outputs, sign/zero preservation, physical type and knownness refusal, exact interest conservation, SSTB/base identities, source immutability, stable full-fit subset replay, artifact byte/value replay and malformed-input refusal. + +The 59 invented controls passed. The separately reviewed genuine construction then produced all 59 canonical outcomes for 207,692 ordinary returns in 23.07 seconds. The 64-return subset reproduced the full-cohort outputs when it reused the full QBI employee fit, and the typed donor passed byte/value replay through the canonical donor interface. + +Independent review subsequently found that the version 1 envelope could attach +an old descriptive receipt to altered, domain-valid values. The version 2 codec +now checks the row-ordered identity/weight/source-feature digest and all 59 output +values against that receipt on encode and decode. It also pins the actual +interest-band facts, verifies the embedded growth recipe, requires canonical +JSON and physical integer row metadata, and makes CPI-sensitivity acceptance +explicit. The corrected source passed 80 invented controls, including the 21 +new integrity cases. Formatting followed that run; the composed integration +checkout still needs its own execution checks. + +The original genuine donor and evidence remain on version 1. A version 2 genuine +construction or repacking requires separate verification, and the new decoder +refuses old magic. Serialized receipt consistency does not authenticate an +issuer, establish execution or confer release status. + +This result verifies source donor construction. Social Security component reconciliation, downstream SCF mortgage semantics, conditional-fit quality, recipient placement and calibration remain open release requirements. It does not certify a calibrated population or policy-analysis file. + +## Evidence bundled with the source + +The [national extract](evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json) records the exact public workbook cells, units, hashes, family calculations and unresolved capital residuals. The [index transcript](evidence/puf2015-target2024/INDEX-VALUES.json) identifies the SSA and BLS values and retrieval sources; it is a transcription of retrieved values, not a hash of the original web pages. The [growth recipe](evidence/puf2015-target2024/GROWTH-RECIPE.json) assigns each monetary field a factor and interpretation. + +[Public workbook locators](evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json) provide the four official URLs and exact sizes and hashes without local cache paths. The recipe also preserves hashes of the original local acquisition receipt and extraction script. Those two local files are outside this publication bundle; their recorded hashes do not imply that a reader received them. diff --git a/docs/puf55-qualified-fixed-inputs.md b/docs/puf55-qualified-fixed-inputs.md new file mode 100644 index 000000000..15fc9fd3f --- /dev/null +++ b/docs/puf55-qualified-fixed-inputs.md @@ -0,0 +1,13 @@ +# Qualified fixed inputs for PUF55 survey arms + +The existing PUF55 recipient interface continues to select clone one by default. `qualify_puf55_survey_recipients(run, arm=0)` adds a separate original-arm projection and distinct graph node IDs. Both projections validate the complete two-arm identity and role mapping. This changes no default application or finalization behavior. + +`qualify_puf55_survey_fixed_inputs(run, arm=0)` requalifies the retained financial run and produces the eight current financial targets on the exact person and tax-unit axes. These are **qualified survey-preserving values**: some ACS amounts and component splits are modeled. They are not all raw observations. Tax-unit amounts include every modeled member and are known only when every member is qualified. + +The optional `development_rules=DEVELOPMENT_RULES` tuple enables four explicitly provisional ASEC mappings. It preserves the maintained pension fraction, regular-IRA-as-taxable routing, the net-property total as a rental proxy, and the direct farm amount. These require original reporting/amount knownness and exact agreement with the carried person value. NIU, ambiguous zeros, incomplete routing, off-route contradictions and unsupported ACS values remain unknown. A finite inherited column cannot establish knownness. Source totals and unresolved taxable/component-split flags remain in the source-basis artifact. Graph parameters name the assumptions and explicitly say release science is not accepted. + +`puf55_survey_fixed_input_nodes(qualified)` declares one artifact-only node for the selected arm. It depends on the genuine source/financial edges, full receiving slices, recipient projection and each selected recipient matrix. Register `Puf55SurveyFixedInputKernel(run, development_rules=...)` alongside the existing recipient kernels. Its outputs include `qualification`, `source_basis`, and the typed target artifacts named by `observed_artifact_names(qualified, profile)`. + +Those target artifacts bind exact float64 values, boolean masks, row identities, target names, matrix bytes and the actual matrix producer key. The core `legacy_qrf_apply_observed_matrix_nodes` interface can consume them when the later host integration is enabled. It executes every target model, retains each raw draw, and merges qualified values before subsequent targets. This module does not fabricate an application prefix, fit or apply a model, attach columns, prune records, change weights, or issue a population. + +On required replay the host must validate actual producer identities against its complete compiled graph and call `verify_materialized_puf55_fixed_inputs` with the materialized artifacts and matrix edges. The verifier requalifies the real financial and source owners; a receipt, detached dataclass, checkpoint or caller-supplied hash grants no source authority. Whole-host execution, arm placement/finalization, seed policy and scientific release qualification remain separate integration work. diff --git a/docs/puf55-survey-ss-measurement-decision.md b/docs/puf55-survey-ss-measurement-decision.md new file mode 100644 index 000000000..fc12e6b85 --- /dev/null +++ b/docs/puf55-survey-ss-measurement-decision.md @@ -0,0 +1,107 @@ +# PUF55 survey Social Security conditioning decision + +On 2026-09-09 the build owner adopted +`social_security_filer_joint_spouse_report_sum_proxy` for PUF matching. This +supersedes the pending measurement judgment in +[survey-social-security-source.md](survey-social-security-source.md). The +decision authorizes this conditioning measurement and its explicit fallback; +it does not assert donor/recipient equivalence, allocate beneficiaries, complete +Social Security components, approve a fit, or admit a release. + +For each modeled return, include exactly one `HEAD` and, only when the actual +`filing_status_input` is `JOINT`, exactly one `SPOUSE`. Sum their available +nonnegative source report totals. Preserve combined family and child reports +without splitting, deduplicating, or subtracting amounts. Exclude dependents +from this return feature without changing any person's observed or unknown +Social Security cells or component masks. + +The public helper requires the actual authenticated survey preparation and +qualifies current source reports. It uses the exact modeled return roles +retained by that preparation. It does **not** rerun tax-unit construction using +current money. That limitation is accepted for this conditioning stage; later +production integration must qualify the chosen role/current-money relationship +explicitly. A caller-made role table or decoded receipt grants no authority. +The PUF disclosure-capped return-size predictor is not a person count, and the +coarsened PUF filing class 2 cannot identify an actual spouse: surviving-spouse +returns also map to that class. + +## Availability and refusal + +An available report-sum means every included reporter has an in-universe, +finite source amount. It does not mean those reporters legally own all the +reported benefits. Known zero reports are valid. Unknown components do not +invalidate a known total. Family/child ambiguity remains an explicit limitation +and does not itself trigger missingness. + +An unavailable included report, including an under-15 filer outside the source +question universe, leaves the proxy unknown and selects the separately named +`PUF55_SURVEY_SS_NO_TOTAL` profile. This profile has the same 55 outputs and +eight predictors. The existing `PUF55_SURVEY_SS` retains nine predictors. +Neither produces the four Social Security components. There is no partial sum, +dependent-zero convention, component-sum fallback, silent row deletion, or +implicit change to the 59/65-output profiles. + +Missing/invalid roles, duplicate or orphan membership, conflicting source +values, nonnumeric amounts, negative/nonfinite observations, and known amounts +outside the reporting universe refuse. They cannot route to eight predictors: +the first two PUF predictors still require valid roles. A source qualifier's own +refusals remain refusals; the fallback does not override them. Graph integration +must bind both disjoint recipient routes, account for each recipient exactly +once, authenticate inherited clone membership, and execute the corresponding +complete fit/apply/attachment chain. This numerical helper does none of that +graph admission or fitting. + +## Measurement limits + +[IRS Publication 915 (2015), pages 2–6](https://www.irs.gov/pub/irs-prior/p915--2015.pdf#page=2) +assigns benefits to the legal beneficiary, including a child whose parent +receives the check; joint filers combine spouses' benefits. The return amount +uses benefits net of repayments and includes the Social Security equivalent +part of tier-1 Railroad Retirement, excluding other pension portions. Form +1040EZ omits benefits, and some recipients need not file. These rules motivate +the selected return members but do not establish survey report ownership or +PUF editing conventions. [SSA's representative-payee guidance](https://www.ssa.gov/payee/faqrep.htm) +likewise distinguishes the person managing a payment from its beneficiary. + +The [2025 ASEC dictionary, PDF pages 48–49](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=48) +permits combined family payments and identifies payments on behalf of children. +Its source total covers calendar 2024 for a 2025 interview. The +[ACS 2024 questionnaire, question 43](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +uses the prior twelve months, permits jointly received income entirely on one +person's record, and combines Social Security with Railroad Retirement. +[ACS SSP metadata, PDF page 45](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf#page=45) +defines the age-15 universe, rounded/topcoded amounts and ADJINC price +adjustment. The helper preserves those distinct observation windows; it does +not calendarize ACS or estimate repayments/Railroad adjustments. + +The ninth donor predictor remains the existing transported 2015 `E02400` +carrier under its declared 2024 money-growth convention. Carrier zeros do not +prove absent benefits. Exact 2015 publisher editing/omission rules remain +unverified; the [IRS PUF page](https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files) +reports the 2012–2015 files unavailable. Matching is therefore conditional on +this documented proxy assumption, not a newly authenticated observed 2024 +return total. Using eight predictors does not remove donor coverage concerns +for nonfilers or resolve assumptions in the other donor outcomes. + +## Diagnostics required before relying on a fit + +Family payments on selected reporters may overstate their own benefits; +payments reported elsewhere may understate them. An all-member sum includes +dependent reports and cannot solve this attribution problem. Topcoding, +allocation, return editing and transported donor money can distort zeros and +tails. Growth of fixed 2015 amounts does not predict 2024 cohort composition. + +Report route counts, known zero/positive distributions and tails by survey, +actual filing status, age, child-report reasons, allocation provenance, and +multiple-return household status. Preserve original-household holdouts across +clones. Compare design-weighted held-out performance and recipient overlap for +eight versus nine predictors; donor-only validation cannot establish cross-source +measurement validity. An adult in-universe all-member report sum may be a +labelled sensitivity diagnostic, with under-15 unknownness retained. It is not +the default feature. Benefit-component modeling and release checks remain +separate work. + +This change provides source-bound numerical values, explicit route metadata, +and an isolated graph profile declaration. Actual source guards, route-aware +graph integration, donor projection for the fallback, fits and diagnostics are +subsequent verification steps, not results claimed by this decision. diff --git a/docs/shared-constants.md b/docs/shared-constants.md index 02442f68e..3eda45a17 100644 --- a/docs/shared-constants.md +++ b/docs/shared-constants.md @@ -51,3 +51,61 @@ from microcosm.calibrate.geography_constants import ( Do not add another state FIPS/postal mapping to a country runtime or diagnostics module. Extend the shared definition and its consistency test when supported geographies change. + +## US target comparison views + +[`microcosm.build.us_runtime.target_geography_view`](../packages/microcosm-build/src/microcosm/build/us_runtime/target_geography_view.py) +owns the national/state/congressional-district vocabulary that US +candidate-versus-incumbent target comparisons roll up on +(`US_TARGET_GEOGRAPHY_VIEW_LEVELS`, `US_TARGET_GEOGRAPHY_VIEW_ORDER`, +`UNRESOLVED_GEOGRAPHY_VIEW_LEVEL`, `GEOGRAPHY_VIEW_LEVEL_ALIASES`) and the +`us_target_geography_view` / `us_target_spec_geography_view` resolvers that +read it out of a compiled target's declared evidence. + +The same three scopes are advertised by the frozen native measurement kernel +`graph_fiscal_measurement` as its household `FiscalGeographyScope` levels. +That kernel is not edited to import the shared tuple, because doing so would +move a live graph node identity; +`test_us_release_head_to_head_scorer.py::test_view_levels_match_the_native_measurement_kernel_scopes` +asserts the two vocabularies equal instead, and is the consistency test +required by item 5 above. Congressional-district membership itself is not +restated here: it comes from the shared +`microcosm.data.us_critical_targets.is_congressional_district_target`. + +The same module also owns which metadata key identifies an *area* at each +level (`_LEVEL_OWNED_ID_METADATA_KEYS`: `state` → `state_fips`, +`congressional_district` → `congressional_district_geoid`) and which two +sources carry the ledger fact's geography id verbatim +(`_CANONICAL_GEOGRAPHY_ID_SOURCES`). Both matter because the US target +compiler derives a district's parent `state_fips` from its geoid +(`fiscal_targets.py:2825-2830`), so an identifier read without regard to level +would name the wrong area. Read an identifier only through +`us_target_geography_view`; do not re-derive a level-to-key mapping in a +consumer. + +The source groups do not establish an identifier's encoding: hierarchy and +ledger fields accept bare or malformed text too. The view validates canonical +values at their resolved level with the existing fiscal target compiler's +state/CD parsers and `national_age_activation.NATIONAL_GEOGRAPHY_ID`. Bare +values are recognized separately, with the shared state FIPS mapping; other +nonempty values remain unrecognized. Neither validation rewrites values nor +assigns a missing congressional-district vintage. + +The repo has no shared +constant for the state prefix (`"0400000US"` is an unnamed literal in +`fiscal_targets.py`, `congressional_district_vintage.py`, +`congressional_district_geography.py` and `medicaid_take_up.py`). Until one +exists, `target_geography_view` declares no equivalence between them: +`TargetGeographyView.geography_id_is_canonical` is how a consumer counting +distinct areas keeps the encodings apart. `geography_id_is_bare` distinguishes +recognized bare values from unrecognized ones. Scorecard schema 6 reports both +remainders and counts resolved rows without IDs separately from unresolved +scopes; no consumer should convert between encodings on its own. + +If every explicit scope declaration is unsupported, the row stays unresolved +even when its name or other metadata provides CD evidence. The fallback is +reserved for rows without any explicit scope declaration. + +Do not add another geographic-level enumeration to a US comparison, scorer, or +diagnostics module. Extend this definition and both its consistency tests when +a new view is supported. diff --git a/docs/snap-final-export-checks.md b/docs/snap-final-export-checks.md new file mode 100644 index 000000000..d65917856 --- /dev/null +++ b/docs/snap-final-export-checks.md @@ -0,0 +1,90 @@ +# SNAP final-export checks + +`snap_export_acceptance` in `microcosm.build.us_runtime.snap_release_acceptance` +checks an explicitly identified final H5. It returns individual check statuses, +not an overall release certificate. It is not enabled in the builder by default +and has no CLI. + +The interface requires the H5 path and expected SHA-256, an explicit nonempty +`ExportContract`, integer model year, US `TargetRegistry`, and typed +`TargetDiagnostic` rows. Optional `TargetFitRequirement` objects carry existing +reviewed fit decisions. A file hash identifies bytes; it does not authenticate +source custody, calibrated-weight issuance, the registry, or permission to +publish. The release owner must bind those authorities independently. + +The wrapper requires exactly 51 state rows (50 states plus DC) for each of +`snap_households` and `snap_total`, following the current fiscal-target +materializer metadata. It checks diagnostic identities, target values and error +arithmetic. The final estimates come from a simulation reopened from the H5 by +the maintained `default_simulate_factory`, with the release SPM selection. The +interface accepts no precomputed simulation totals or caller-written passing +summary. Input-stage take-up fit and saturation never waive final-export misses. + +## Count and period definitions + +The current `fiscal_targets.py` mapping uses FNS fiscal-year average monthly +participating households. Its model counterpart is the weighted count of SPM +units with positive annual `snap`: the boolean is evaluated at the SPM grain +before summing. It is not a count of physical households with any recipient or +an annual-ever participation measure. Two positive SPM units within a household +count twice at that household's calibrated weight, regardless of how many +persons belong to either unit. + +The benefits counterpart is the weighted sum of annual modeled `snap` at the +same SPM grain. The result records each target's source period, source citation +and model period separately. It does not relabel an older source observation as +a current-year observation or resolve the fiscal-year/calendar-year difference. +Both calculations retain `MicroSeries` weights and use `map_to="spm_unit"`. +Simulation SPM IDs and order must equal those read from the export. + +The [FNS FY2022 eligible-person participation comparison](https://www.fns.usda.gov/research/snap/state-participation-rates/2022) +is explicitly advisory and pending. This slice does not import the old PR's +rate table or infer eligible people from positive-SNAP SPM-unit counts. A future +comparison needs a verified source table and an aligned eligible-person and SNAP +assistance-unit universe; SPM membership alone does not supply that universe. + +## Tolerances and artifact checks + +The wrapper reuses each `TargetSpec.tolerance` as an absolute tolerance and any +matching `TargetFitRequirement.max_abs_relative_error`. If both apply, both must +hold. A row with neither stays pending, even at zero measured error. Existing +fit requirements retain their minimum-match checks. The wrapper supplies no new +statistical tolerance and does not reuse the input-stage take-up tolerance as a +final-export tolerance. + +The H5 passes the maintained denied-pool ingress boundary. Checks cover the +export contract, absence of persisted `snap`, entity IDs and memberships, +nonempty groups, nonnegative finite household weights, the model year, +nonmissing county/state FIPS, nonzero county suffixes, matching county/state +prefixes, and positive-weight support in all 51 areas. County codes are not authenticated against +a particular vintage's county roster. The file hash is checked again after all +engine calculation and teardown I/O, including when teardown raises. A changed file or failed teardown invalidates computed fit. +Missing engine dependencies or an absent final artifact stay pending; malformed +available evidence produces a failed check. Consumers must inspect every check, +including failures alongside pending checks. + +## Validation and remaining proof + +The flat invented suite writes genuine small H5 files with the maintained writer. +Its explicit test-only weighted-series double delegates sums to typed `Frame` +accounting. It exercises wrapper I/O and aggregation semantics, not actual +PolicyEngine formulas or a genuine release. It includes two SPM units in one +household, a multi-person SPM unit, missing/duplicate/mismatched targets, +malformed geography, stale diagnostics, missing and conflicting tolerances, +unweighted engine output, changed SPM order, and mutation during calculation or +teardown. + +Before release qualification, run this wrapper on the exact final exported H5 +with its authenticated release owner, exact hash, real consumer export contract, +model year, corresponding registry and diagnostics, and reviewed tolerance +decisions. Use the pinned PolicyEngine runtime to recompute all 102 state +estimates. Resolve every failed/pending required check and preserve the report +with the artifact and runtime identities. Separately retain the full release's +source, graph, calibration and publication checks. No actual population, model +execution or release approval was performed by this implementation slice. + +This adapts the final-artifact intent of +[Microcosm PR #413](https://github.com/PolicyEngine/microcosm/pull/413), reviewed +at `c538c81d46661fe2d20876391ea1a4371d14cd8d`. Its old source/feed pins, +10% defaults, caller-supplied totals, manifest-summary approval and old-build +failure lists are not part of this interface. diff --git a/docs/survey-social-security-source.md b/docs/survey-social-security-source.md new file mode 100644 index 000000000..276f6e6f0 --- /dev/null +++ b/docs/survey-social-security-source.md @@ -0,0 +1,169 @@ +# Survey Social Security source semantics + +The source-report Social Security total precedes PUF enrichment. The current +qualifier verifies the original observations and preserves their limits; it +does not fit a completion model, assign individual beneficiaries, or certify a +population. The explicit `PUF55_SURVEY_SS` profile has 55 outputs and uses an +upstream-qualified tax-unit total as its ninth conditioning input. Historical +full65 and PUF59 behavior is unchanged. The new profile's native source and +whole-population attachment integration remain pending. + +## Source measurements + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +describes `SS_YN` and `SS_VAL` on PDF page 49. The question covers people aged +15 or older and permits combined family payments on a person's report. A +positive report is therefore not necessarily one individual beneficiary's +payment. The qualifier preserves the source report's grain. + +The retained ASEC original member supplies `PERIDNUM`, household/line/age +coordinates, `SS_VAL`, `SS_YN`, `RESNSS1`, `RESNSS2`, `RESNSSA`, `I_SSVAL` +and `I_SSYN`. Source keys, the complete current cohort, raw member size/hash, +live source issuance and exact current money values must all agree. Source +year and income year are 2024; interview year is 2025. Carried reason fields +are not a substitute for these original literals. + +The [2024 ACS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf) +defines combined `SSP`, its age-15 question universe, and allocation flag +`FSSP`. The qualifier binds the retained 2024 ACS issuance and verifies +`SSP * (ADJINC / 1000000)` against the prepared amount. This remains a rolling +income window expressed in the survey's price basis. `FSSP` is not in the +current native projection, so publisher allocation provenance stays explicitly +unresolved; no observed-value claim is inferred from a missing flag. + +## Mapping judgments and unknownness + +The reason mapping is an explicit judgment at source-report grain. Retirement +and disability map to their corresponding component. Widowed and surviving +child reasons map to survivors; spouse and dependent child reasons map to +dependents. A single resolved component receives the report total. Repeated +reasons in the same component do not duplicate the amount. + +Distinct component reasons, missing reasons, code 7's combined child categories +and code 8's residual category leave the component amounts unknown. The +published reasons restrict possible components where they can; there is no +reason priority, age fallback or equal-share default. Separately observed +component amounts and individual beneficiary assignments are not claimed. + +An ASEC under-15 `SS_VAL=0` is a not-in-universe sentinel. Both ASEC and ACS +under-15 beneficiary totals remain unknown in the projection, with the raw +ASEC zero retained in the separate literal table. An in-universe zero remains +a known report total of zero. Contradictory universe/recipiency combinations +and invalid nonempty codes refuse; blank literals remain distinguishable +from invalid codes. + +An eventual component model may normalize nonnegative scores within allowed +components, preserving the source total and already resolved source basis. +It must supply positive allowed mass. The numerical helper does not authenticate +such a model; the graph must bind real fit/apply artifacts. A reporting-unit +convention, allocation to individual beneficiaries where necessary, explicit +income periods, design-weighted training, native-to-clone inheritance and +held-out checks remain separate required work. + +## Full original ASEC report basis + +`qualify_full_current_social_security(preparation)` adds a full original ASEC +basis without changing `qualify_current_social_security(preparation)`. It +requires the same live authenticated preparation and recodes the same captured +member once. Its `selected` result preserves the existing selected ASEC/ACS +description, including its evidence. + +The returned `asec_frame` retains every original current ASEC person and the +original household DESIGN weights before receiving-sample selection. The +`asec_basis` table uses those native person IDs and the same row order. It +contains report totals, resolved components, reason-allowed masks, reporting +universe, basis origin, allocation codes and labels, every captured source +literal, and current-money status, validity and zero-origin bytes. Every +selected ASEC row must agree exactly with its full-basis row. + +Receiving selection can therefore omit a resolved positive ASEC report while +the full basis retains it for a later donor-selection decision. This API does +not select eligible donors, fit component probabilities, assign beneficiaries, +or attach canonical component values to a receiving population. Under-15 NIU +zeros and ambiguous report components retain their existing unknownness. + +These returned objects contain detached data. `full_social_security_seal` +describes their exact contents; it cannot authenticate a copied object or +replace the live preparation. A future graph host must retain that preparation, +requalify its source after relevant I/O, and bind the fitted artifacts and +native-to-clone attachment separately. The qualifier checks source, callable, +configuration and detached-result identity after its final owner I/O. + +## Conditioning measurement judgment, 2026-09-09 + +The recipient measurement for predictor nine remains undecided. Passing the +numerical `PUF55_SURVEY_SS` mechanism does not establish donor/recipient +comparability or authenticate a beneficiary total. + +[IRS Publication 915 (2015), pages 3 and 5–6](https://www.irs.gov/pub/irs-prior/p915--2015.pdf#page=3) +combines spouses' benefits on a joint return and assigns benefits to the person +legally entitled to receive them. A child's payment remains the child's even +when the check names the parent. Form 1040 line 20a reports net benefits after +repayments, including equivalent tier-1 railroad benefits; Form 1040EZ does not +report benefits. Thus a sum over all modeled tax-unit members is a different +measurement from the filer/joint-spouse return amount. + +Restricting survey records to the filer and joint-return spouse would still +produce a **report-sum proxy**, not an observed beneficiary total. The +[ASEC dictionary, PDF pages 48–49](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=48) +permits combined family payments and includes reasons for payments on behalf +of children. The [ACS 2024 questionnaire, question 43, page 18](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +also permits jointly received income to appear entirely on one person's +record, with no receipt recorded for the other. Its income category includes +Social Security or Railroad Retirement. A parent may therefore report child +payments, or another household member may report an included person's income. +Multiple tax units within one household make reporter membership insufficient +to establish benefit ownership; household totals must not be duplicated across +returns or redistributed without a separate supported model. + +If adopted, a measurement such as +`social_security_filer_joint_spouse_report_sum_proxy` would need an explicit +upstream contract before supplying `puf_conditioning_social_security_total`. +That contract would bind actual return roles, source reporting records, +knownness and income periods; the coarsened PUF filing-class predictor alone +cannot establish joint-return membership. ASEC calendar-year income, ACS's +rolling window, transported donor money, repayment treatment and railroad +coverage remain comparability questions. Component completion is separate. + +Excluding a dependent from a parent's return feature does not assign that +dependent zero benefits. Both surveys' under-15 amounts remain unknown. If an +included filer has no in-universe report, an empty or missing report set cannot +silently become a known zero. Family-payment ambiguity also survives an +age-universe restriction. + +The exact 2015 `E02400` definition and publisher editing/omission conventions +remain unauthenticated by this lookup. The [live IRS PUF page](https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files), +checked on 2026-09-09, says the 2012–2015 files are unavailable. The verified +Form 1040 rules do not independently establish how every PUF record encodes +them. This record adopts no proxy, allocates no beneficiaries and grants no +dataset or release acceptance. + +## Verification scope + +The 52-case control run executes 50 numerical/literal tests and two complete +maintained source issuances from invented original CSV/checkpoint fixtures. +It verifies source-identity joins despite reordered rows, repeat qualification, +defensive returned data, exact source totals, unknown components and unchanged +input frames. Source/control files, installed model source and the 12 admitted +code resources were unchanged before, after and in the external postcheck. + +The run completed in 67.6 seconds with 428.9 MB peak RSS. Receipt SHA256: +`2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. +These controls are not native-source execution, a fitted component model, +native PUF55 integration, policy-engine validation or release acceptance. + +The additive full-original API has nine further passing invented-source tests. +They retain a resolved positive ASEC report omitted by receiving selection and +its original DESIGN weight, compare the selected projection exactly, preserve +under-15 and ambiguous unknowns, and reject configuration, source-literal and +late detached-result mutations. The existing PUF Social Security measurement +module also passes all 31 tests, including retained-source qualification and +mutation callbacks through the unchanged selected API. + +These two separate runs used 63.2 and 63.9 total supervisor-and-child CPU seconds, +with peak combined resident memory of 455.1 and 538.8 MB. Neither run loaded a +country model, fitted a model, executed a graph, accessed actual microdata, or +changed its pinned source/runtime files. Both restored the no-fit profiler, +reported zero guard denials and reaped their child. They establish this source +interface and its existing measurement consumer; a fitted component estimator +and full graph attachment remain separate work. diff --git a/docs/uk-atomic-household-lineage.md b/docs/uk-atomic-household-lineage.md new file mode 100644 index 000000000..067353f48 --- /dev/null +++ b/docs/uk-atomic-household-lineage.md @@ -0,0 +1,28 @@ +# UK household identity after cloning + +The shared atomic-area operators need a stable household key after initial cloning. `uk_runtime.atomic_household_lineage` projects supplied original-source identities through explicit selection and expansion records, using the existing `household_draw_key` encoding. It adds no assignment algorithm, data reader, issuance registry or full-build host. + +A key includes the declared source, source vintage, original household ID and ordered clone path. It excludes final offset IDs, row order, sample size, calibration weights and financial values. Exact int64 IDs above `2**53` remain distinct. Original households keep their previous path; a new child appends its explicitly supplied branch ordinal. An original branch does not gain an inferred zero ordinal. + +## Contract + +`project_atomic_household_keys(roots, *, steps, final_ids)` returns a detached two-column table: int64 `household_id` and nullable-string `geography_household_key`, in the requested final order. Roots have exactly `household_id`, `source`, `source_vintage` and `source_household_id`. An empty selection is a valid description, not evidence that an empty dataset is useful or releasable. + +Steps are immutable descriptions: + +- `HouseholdExpansion(branch, before_ids, after_ids, parent_pairs, child_ordinals)` carries exact household axes, the household part of the shared executor's `receipt["expand"]`, and explicit child ordinals. The existing ordered branches are SPI support, CGT incidence, CGT band donors and geographic support. Each arriving household needs one known preceding parent and one ordinal. Two children of the same parent cannot reuse an ordinal in the same branch. +- `HouseholdSelection(before_ids, after_ids)` explicitly removes households while preserving their keys. This supports the source-family sample between the completed source spine and geographic expansion. It does not check that the sample respected the host's family-sampling rule. + +Each before-axis must have exactly the previous step's household IDs. Expansion cannot remove incumbents; selection cannot add households. Unknown or null parents, repeated or reordered branches, incomplete parent/ordinal coverage, duplicate keys and a mismatched final axis refuse. Axis ordering may change, because ordering is not identity. The function never guesses ancestry from ID arithmetic, support flags or model values. + +## Host responsibilities + +These checks establish internal consistency of supplied descriptions. A dataclass, DataFrame or serialized receipt does not prove its source. The receiving host must retain and check the actual original source owner, complete before/after populations and source/EXPAND/FILTER evidence. It must authenticate the complete structural stage roster, the explicit branch-ordinal convention, and the exact final receiving household axis before and after relevant I/O. A coherent invented history can pass this pure helper; it is not an admission gate. + +The host can extract the household pairs from its verified shared EXPAND receipt without decoding household ID offsets. The helper returns the key column for the existing `uk_atomic_assignment_definition` and `atomic_geography_nodes` APIs. Support-byte provenance, observed-region constraints and versioned mapping relations remain separate responsibilities. Larger geographies continue to derive from the assigned atomic area. + +No production UK host is changed here. María's full-build graph currently uses its existing ladder draw after geographic expansion. Adopting this key and shared assignment must be coordinated in that host. The existing source stage order also interleaves enrichment before later SPI/CGT cloning; moving atomic assignment before every enrichment stage requires a separate staged migration. This helper does not claim that migration, native FRS qualification or a new UK release. + +## Verification scope + +Invented tests cover the full four-branch path, explicit selection, exact large IDs, source and ordinal changes, stable keys under row reordering and pool growth, unknown/colliding ancestry refusal and detached output. A real shared CREATE/EXPAND/FILTER graph supplies the household lineage receipt and replays from cache. Its projected keys then feed the existing seven-node atomic graph, including three system imports, assignment, derivation and its gate. That graph also completes required replay and preserves the supplied entity tables, weights, strata and metadata. No native source, private archive, country engine or external resource is used. diff --git a/docs/uk-national-calibration-runbook-623.md b/docs/uk-national-calibration-runbook-623.md index d7907f1c7..42e644e97 100644 --- a/docs/uk-national-calibration-runbook-623.md +++ b/docs/uk-national-calibration-runbook-623.md @@ -17,6 +17,15 @@ is the release-cut producer's job, not calibration's. ## Command shape +The current operational path requires this local workspace and its Git identity. +The calibration seam derives that identity from its module's checkout location; +standalone installed-wheel execution does not yet have a supported provenance +contract. Wheel orchestration tests use an explicit invented code pin and do not +certify installed-wheel production provenance. Supporting that mode requires a +separate identity design that binds the actual loaded implementation, rather +than substituting an unrelated checkout's Git HEAD. An unresolved code pin +continues to refuse before input or output processing. + Calibration runs through `tools/calibrate_uk_national_dataset.py`. That driver is the only one that builds the measure resolver and applies the committed measure-exclusion register, and 187 of the activated references bind model diff --git a/docs/us-asec-source-domain-contract.md b/docs/us-asec-source-domain-contract.md new file mode 100644 index 000000000..597ec5a7b --- /dev/null +++ b/docs/us-asec-source-domain-contract.md @@ -0,0 +1,43 @@ +# ASEC amount-domain contracts + +The current income-routing, interest and child-support qualifiers read monetary +definitions from the pinned `asec_current_money_domains_v1.json` artifact. +Their public amount-entry functions return detached mappings. Routing and +interest cache immutable tuples privately; child support caches immutable JSON +bytes and reconstructs its nested evidence on each call. Replacing an entry, +clearing a returned mapping or editing nested child-support metadata cannot +change a later caller's definitions. The public mapping and evidence shapes +remain unchanged. + +Each relevant field must occur exactly once and contain exactly one vintage for +the current income year. Its dictionary spelling, URL and digest must match the +qualifier's source pin. Missing or duplicate entries refuse qualification. + +The interest total and child-support parsers accept unsigned integer literals. +They explicitly require the corresponding artifact domains to describe +nonnegative integer dollars, with the supported zero and NIU meanings, no +additional missing-value codes or excluded dollar values, and consistent +encoded, dollar and printed ranges. Bounds remain artifact-derived. Re-pinning +an artifact with an unsupported domain requires a parser change and its own +review; it cannot silently change the interpretation of those literals. + +Before capturing the original source member, each qualifier compares its +relevant amount definitions with the actual retained current-money owner's +`MoneyDomain` entries. Names must be unique; entity, grain, native column, +bounds and zero semantics must agree. Interest checks the retained `INT_VAL` +total. Its additional ordinary and retirement-account components retain their +separate published definitions. Child support checks both `CSP_VAL` and +`CHSP_VAL`; routing checks its existing nine fields. + +These checks preserve the existing source meanings. In particular, a +`CHSP_YN` answer describes an obligation, and neither a no-obligation answer nor +an NIU paid amount establishes zero voluntary payments. Receipt, reporting, +allocation and top-code classifications are unchanged. The changes add no +model, canonical engine input, source issuer or native-data admission. + +Validation used invented mutations of the public domain artifact and actual +invented source-owner fixtures: 73 new domain/cache controls, 88 routing tests, +40 interest tests, 10 allocation tests and 40 child-support tests passed under +the existing bounded source guard. The new controls first reproduced 65 +failures against the prior implementation. These results certify the tested +source contracts, not a native dataset or release. diff --git a/docs/us-atomic-property-financial.md b/docs/us-atomic-property-financial.md new file mode 100644 index 000000000..4380c0f91 --- /dev/null +++ b/docs/us-atomic-property-financial.md @@ -0,0 +1,117 @@ +# Current survey property-income host + +`run_atomic_survey_financial` can append the current property-income graph to +the existing financial composition. Pass an explicit `PropertyIncomeOptions` +through `property_income`; omitting it retains the nineteen-node path. + +```python +from microcosm.build.us_runtime.graph_atomic_survey_financial import ( + run_atomic_survey_financial, +) +from microcosm.build.us_runtime.graph_current_survey_property import ( + PropertyIncomeOptions, +) + +options = PropertyIncomeOptions( + scales=(1.0, 1.0, 1.0, 1.0), + atol=1e-10, + rtol=1e-12, + n_estimators=2, +) +run = run_atomic_survey_financial( + **source_arguments, + geography_config=atomic_geography, + property_income=options, + n_estimators=2, + return_values=True, +) +checked = run.checked_view() +``` + +The small tree counts above are development settings. The caller supplies the +existing authenticated source arguments and atomic geography configuration. +They are not replaced by a downloaded graph, a detached Frame or a receipt. + +The thirty-five-node composition consists of the nine-node survey, clone and +atomic-geography prefix; ten existing financial nodes; and sixteen property +nodes. The new branches select eligible original ASEC donors and original ACS +recipients before allocation. Four ordered, DESIGN-weighted conditional models +produce raw component draws, followed by reconciliation to a known adult ACS +property-income anchor. An explicit attachment carries each original result to +both initial clones. Original ASEC components remain separately qualified even +when that person is excluded from fitting. See the +[fragment contract](us-current-survey-property-graph.md). + +The output adds 24 component, raw-draw, diagnostic and knownness columns. It +preserves the complete preceding financial population: other columns, entity +membership, geography, schema, metadata, weights, strata, ownership and mass +history. Missing source anchors and unresolved components retain missing +values and explicit knownness. This mode does not infer property-income zeros +for children outside the survey question's universe. + +The runner retains both the preceding and extended populations. Before issuance, +execution and required cache replay verify property model training against the +original donor and replay four applications against actual recipient features. +Later issued-run checks requalify the existing source owners, validate the legacy +financial output, reconstruct the complete extended population and check the +verified model artifacts' byte identities. They do not refit or reapply the +models. Copied public result dataclasses have no run authority. See the +[model receipt contract](us-property-model-receipts.md). + +The thirty-five-node option leaves the eight existing financial leaves unchanged. +Its run document therefore records `tax_split_rebased: false` and the legacy +capital-gain model's conditioning on its earlier interest and dividend draws. +It does not itself replace tax inputs. Optional tax rebasing is a separate +three-node extension, described below. + +The PUF host derives its upstream node count and checks the actual final +financial writer. Its recipient nodes read all columns of that population, +including the new property fields, so the dependency is present in the graph. +This interface does not establish that a native PUF run, tax rebase, calibration +or release has passed. Candidate execution evidence belongs in `experiments/`. + +## Optional tax rebase + +Pass `rebase_property_taxes=True` together with `property_income=options` to +append the deterministic receiving version, tax split and numerical gate. The +default is `False`; requesting a rebase without property options refuses before +source I/O. This path has 38 nodes. It retains the complete legacy financial +population and the complete 35-node property population separately from the +final `survey_property.tax_receiving` version. + +The rebase uses ordinary interest and dividends with the maintained fractions +and subtraction complements. Separate missing O/D inputs produce missing tax +leaves, replacing earlier predictions; retirement-account interest remains +auxiliary. A new FILTER conservation record is intentional, while existing +Frame mass history and all non-rebased cells remain unchanged. See the +[numerical fragment contract](us-property-tax-leaves.md). + +The host constructs the declarations from the checked clone Frame and the +existing financial/property output descriptors. It never supplies fake values +for future columns. After the real 35-node output is available, it independently +re-declares and executes the three deterministic operations, checks stored +artifact bytes and complete node receipts, and compares every expected full +population with actual cold or cached observations. The issued run retains +the source, declaration, model, artifact, intermediate population and final +population seals. Later checks reconstruct the same tax operations before the +final source requalification and pure lifetime fence. + +An incomplete numerical gate is valid development evidence. The run document +reports `tax_split_rebased: true`, three extra nodes and the actual +`tax_leaf_complete` value; it remains `release_eligible: false`. PUF recipient +qualification explicitly requires a complete independently checked numerical +gate before it reads its additional source inputs. Both PUF recipient graph +nodes carry a typed edge to that gate on the actual receiving version. The +19- and 35-node paths do not add that edge or claim a tax rebase. + +No child zero completion or missing-adult model is introduced. Consequently the +invented source fixture with under-15 and missing-anchor cases must remain a +development run and refuse full PUF qualification. Capital gains retain their +earlier conditioning; rebasing interest and dividends does not update that model. + +The bounded invented-source acceptance covers 11 host controls, including actual +cold and required-cache 38-node runs, issued-view parity, intermediate/final +population preservation, mutation refusals and incomplete PUF qualification. +The numerical fragment has a separate 40-control acceptance, including a +complete-input positive gate and descriptor-only declaration checks. These +checks do not use native survey data, run a country engine or execute full PUF. diff --git a/docs/us-current-acs-immigration-source-projection.md b/docs/us-current-acs-immigration-source-projection.md new file mode 100644 index 000000000..c1f6e149b --- /dev/null +++ b/docs/us-current-acs-immigration-source-projection.md @@ -0,0 +1,73 @@ +# Selected ACS immigration source projection + +`borrow_current_acs_immigration_projection(preparation)` in +`microcosm.build.us_runtime.current_acs_immigration_source_projection` returns +selected original ACS source inputs from a live authenticated survey preparation. +It internally invokes the existing immigration literal reader and verifies the +retained ACS native owner. Call `validate()` after the consumer's final relevant +I/O; exclusive access to mutable source and output tables remains required. + +The result contains three narrow tables and a descriptive receipt: + +- `raw` preserves the six original string fields SERIALNO, SPORDER, CIT, POBP, + YOEP and AGEP, indexed by the selected receiving person identifier. +- `person` binds typed CIT/POBP/YOEP/AGEP, observed SEX and `is_female`, original + household state, receiving and native identifiers, source namespace, source + year 2024 and observation year 2024. Nullable YOEP, NIU, public-code precision + and lower/upper bounds retain missingness and censoring without a midpoint. +- `households` binds receiving/native household identifiers, original SERIALNO, + original state token ST and typed state FIPS. + +There is no Frame, weight vector, status assignment, fit, stock calibration, +graph attachment, model input admission or release approval. In particular, +selected ACS people are not a complete-country stock denominator. The existing +shared literal reader also verifies ASEC source files, but this result projects +only ACS fields and consumes no prior-income values. It is opt-in and changes no +build default. + +## Source meanings + +The [2024 one-year PUMS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.txt) +defines the domains. The [2024 PUMS code list](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/code_lists/ACSPUMS2024CodeLists.xls) +defines detailed countries grouped under public POBP codes. The person projection +retains the ACS code namespace; it does not claim that the 222-code domain equals +ASEC PENATVTY. + +| Input | Admitted evidence | +|---|---| +| CIT | Codes 1–5: US-born, specified US territories, US parents abroad, naturalized, noncitizen. No unknown code is filled. | +| POBP | Published 2024 public categories, including state births and grouped residuals. 451 means Sudan only. 464 groups Tunisia, Western Sahara and South Sudan; exact South Sudan origin is unavailable. | +| YOEP | CIT 1 requires blank NIU; a nonblank value contradicts the questionnaire universe and is refused. CIT 2–5 require a published code. 1938 means 1938 or earlier; 1939 means 1939–1944; 1945–2024 are individual calendar years. | +| AGEP | Published 0–99 code, directly equal to the native observed age. Source top-coding is retained, without an exact-age claim. | +| SEX | Native source code 1 or 2 and exact agreement with the native boolean. No unallocated-response claim. | +| State | Original household ST through exact original membership; 50 states and DC. This view does not admit Puerto Rico Community Survey semantics. | + +The [2024 subject definitions, Year of Entry, printed page 146](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf) +include CIT 2 and 3 in the entry-question universe although Census calls them +native-born. Entry refers to the latest occasion of coming to live in the US, +with documented reporting ambiguity for repeat entrants. It is not proof of +first entry or continuous residence. Birthplace is not proof of nationality or +legal immigration status. Other public POBP categories can also be grouped or +unspecified: `published_category` deliberately makes no exact-country claim. + +## Identity and verification + +Each row joins the exact selected source namespace/native person key to original +SERIALNO/SPORDER, original household membership, observed age and sex, and the +same household's state. IDs never pass through a float conversion. The pure +projection supports arbitrary integer coordinates; authentic issuance retains +the existing upstream preparation's coordinate contract. + +The live preparation and native issuer remain the source authorities. The view +retains independent immutable seals for the qualified literals and output tables; +altering a literal owner's callback closure does not move those seals. Copies, +detached replacement tables, a copied receipt and a different preparation cannot +stand in for the retained view. Final pure identity and table checks run after +source and implementation-file I/O. These checks detect changes at validation +boundaries; they do not make concurrent pandas mutation atomic. + +Invented archive tests exercise the actual source catalogue, native issuance and +literal capture path, including household and group-quarters observations. Pure +domain/coordinate controls and mutation, copied-owner, late-I/O and editable- +closure controls complement them. This evidence does not qualify real sources, +immigration rules or a release artifact. diff --git a/docs/us-current-asec-child-support-source.md b/docs/us-current-asec-child-support-source.md new file mode 100644 index 000000000..c55593637 --- /dev/null +++ b/docs/us-current-asec-child-support-source.md @@ -0,0 +1,79 @@ +# Current ASEC child-support observations + +`qualify_current_asec_child_support(preparation)` in +`microcosm.build.us_runtime.current_asec_child_support_source` qualifies the +original received- and paid-support observations from the retained 2025 ASEC +person source. The dollar amounts refer to the current 2024 income cohort. + +The source is the [2025 ASEC public-use dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA-256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The qualifier reuses the exact packaged monetary domains and temporal +authority for `CSP_VAL` and `CHSP_VAL`. It adds source questions and flags from +the same dictionary, with their positions, widths and printed universes. + +| Fields | Source meaning | One-based PDF page | +| --- | --- | --- | +| `CSP_VAL`, `CSP_YN` | Amount received and actual receipt answer; receipt universe age 15+ | 52 | +| `CHSP_VAL`, `CHSP_YN` | Annual amount paid and whether payment is required; the latter is an obligation question | 52 | +| `CHELSEW_YN` | Whether a child lives outside the household; age 15+ | 52 | +| `I_CHELSEWYN`, `I_CHSPVAL`, `I_CHSPYN`, `I_CSPVAL`, `I_CSPYN` | Published allocation flags, using the shared I_ANNVAL code meanings | 54 | +| `TCHSP_VAL`, `TCSP_VAL` | Published top-code indicators for positive amounts | 59 | + +## Received amounts + +The printed zero for `CSP_VAL` means none or NIU. An in-universe receipt-no +answer plus zero establishes known nonreceipt. A receipt-yes answer plus zero +remains ambiguous; the qualifier does not complete it with a zero-dollar +observation. Positive receipt amounts are retained, including valid allocated +observations. Under-15 NIU, missing answers, malformed literals and +contradictory receipt/amount pairs remain distinguishable and unknown. + +## Payments and obligations + +`CHSP_YN` asks whether the person is required to pay child support. A no +answer does not rule out voluntary payment. `CHSP_VAL` describes actual annual +payments and prints zero as NIU, so every paid zero remains a published code +with an unknown canonical payment amount. No obligation answer turns it into +an observed zero. + +The paid amount's universe is `CHSP_YN = 1`. The obligation question itself +prints only the bare field `CHELSEW_YN`, while the allocation flag `I_CHSPYN` +prints `CHELSEW_YN = 1`. That discrepancy remains in the evidence. The +qualifier conservatively accepts positive payments on the observed +intersection age 15+, child-elsewhere yes and required-to-pay yes. Other +routes keep the published positive amount and an explicit unresolved status. +This intersection is a qualification choice, not a correction of the printed +universe or proof that voluntary payments do not occur. + +The obligation and child-elsewhere entries do not independently state a +reference year. Their source context is retained; the qualifier does not +issue a current-at-interview or eligibility input from those answers. + +## Provenance and consumption + +The result has two complete tables: selected ASEC people on the common +preparation's person identities, and the original current-cohort literals on +native identities. Source amount validity, statuses and zero origin are +retained alongside the descriptive projection. Allocation/top-code literal +values and parse statuses remain separate from amount knownness. All-zero +allocation flags describe the published codes; they are not a blanket claim +of unallocated or respondent-reported data. + +The qualifier takes the actual authenticated preparation, captures the +pinned original member once through the shared bounded literal reader, +checks its bytes and complete native-key roster, verifies household/line/raw +age coordinates, and compares both source amounts to their retained +current-money values with exact validity and float64 bits. Source row order +does not define person identity. + +`CurrentAsecChildSupportValues` is descriptive transport, not an issuer. +`child_support_values_seal` covers both complete tables, axes, nullable masks +and backing storage, float bits and detached evidence. The actual preparation +is requalified after source I/O, followed by final owner and physical-value +checks. A consumer must retain the actual owner, requalify immediately before +consumption and after its own last relevant I/O, and compare complete value +seals before returning or exporting a successor. Copied data or a matching +digest cannot grant source authority. + +This module assigns no engine leaf, changes no clone, fits no ACS model and +issues no new source admission or release eligibility. diff --git a/docs/us-current-asec-dividend-source.md b/docs/us-current-asec-dividend-source.md new file mode 100644 index 000000000..de977cd18 --- /dev/null +++ b/docs/us-current-asec-dividend-source.md @@ -0,0 +1,114 @@ +# Current ASEC dividend and survivor routes + +`current_asec_dividend_source.qualify_current_asec_dividend(preparation)` +qualifies current ASEC dividend observations and the recorded survivor-income +source routes that could overlap with a broad property-income concept. It +reads the original 2025 person member, whose annual money amounts refer to +2024. It returns source observations and provenance, without modeling ACS +values, assigning tax treatment or authorizing a dataset release. + +The [2025 ASEC public-use dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +is pinned to SHA-256 +`5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The dividend domain also comes from the existing pinned +`asec_current_money_domains_v1.json`. Page numbers below are one-based PDF +pages; positions and widths refer to the printed ASCII layout. + +| Field | Position / width | Page | Printed universe | +| --- | --- | --- | --- | +| `DIV_VAL` | 478 / 6 | 45 | `DIV_YN = 1` | +| `DIV_YN` | 484 / 1 | 45 | Age 15+ | +| `SUR_SC1`, `SUR_SC2` | 648 / 2, 650 / 2 | 50 | `SUR_YN = 1` | +| `SUR_YN` | 664 / 1 | 50 | Age 15+ | +| `I_DIVVAL` | 819 / 1 | 54 | `DIV_YN = 1` | +| `I_DIVYN` | 820 / 1 | 54 | Age 15+ | +| `TDIV_VAL` | 905 / 1 | 59 | `DIV_VAL > 0` | +| `TRNT_VAL` | 917 / 1 | 60 | `RNT_VAL > 0` | + +## Dividend values and source flags + +The projection preserves the exact `DIV_VAL_literal`, parsed +`DIV_VAL_published_amount`, literal and reporting statuses, and a separate +`DIV_VAL_amount_known` mask. `DIV_VAL_amount` contains only the values whose +reporting status is resolved. An in-universe receipt-yes and positive amount +is known; receipt-no and zero establish known nonreceipt. Receipt-yes and +zero remain ambiguous because the dictionary combines none and NIU in its +zero code. Missing literals, NIU and contradictory combinations retain +their source evidence and an unknown canonical amount. No analytical zero +is inferred solely from age. + +Source flags never change this knownness or select donors. A known published +cell can still have publisher allocation or topcoding metadata. It is not a +claim about an uncensored latent amount or a respondent-only report. +`I_DIVVAL` refers to the allocation meanings under `I_ANNVAL` on page 53. +`TDIV_VAL` and `TRNT_VAL` retain codes 0 (not topcoded) and 1 (topcoded), +including missing or malformed flags. `TRNT_VAL` is retained only for future +diagnostics: this qualifier does not read `RNT_VAL` or evaluate that flag's +positive-rent universe. + +The dictionary is internally inconsistent for `I_DIVYN`: its header prints +0–1, while its Values section refers to the `I_ANNVAL` codes 0–9. The result +preserves the literal and code, physical syntax status, separate +`I_DIVYN_header_range_status` and `I_DIVYN_referenced_values_status`, and an +explicit `I_DIVYN_codebook_status`. Codes 2–9 therefore retain the conflict; +they are not silently dropped or admitted under a preferred interpretation. + +## Survivor source routes + +`SUR_YN`, `SUR_SC1` and `SUR_SC2` each retain raw literals, parsed codes, +literal status and source labels. Code 8 identifies regular payments from +estates or trusts. It signals a possible property-income overlap, without +establishing an amount or tax classification. Code 10 means other or don't +know: its literal is valid, but its property route remains unspecified. + +`survivor_property_route_clear` is nullable. It is true only for an +in-universe known receipt-no with both source slots NIU, or receipt-yes with +every slot readable, at least one active slot, and neither code 8 nor code +10. Receipt-yes with an explicit code 8 sets it false. Missing or unreadable +receipts, unresolved slots, contradictory nonreceipt routes, NIU and code 10 +leave it unknown. Under-15 rows remain unknown; incomplete under-15 literals +are labeled unresolved rather than contradictory. Independent booleans +retain the presence of codes 8 and 10 even when the receipt is unresolved. + +This clearance describes only the two recorded source slots. The dictionary's +`SRVS_VAL` entry (PDF page 49, printed page 6C-28) also includes unedited third +and fourth sources. Their types are not qualified here, so two clear visible +slots cannot establish complete absence of survivor estate/trust income. +The property donor bridge conservatively excludes positive survivor receipts +until that additional scope is qualified. This clearance does not establish +that the person has no property income. No survivor amount is captured, no +donor exclusion is applied, and a missing slot is never an observed zero +amount. A later measurement bridge must explicitly choose and validate how +these routes affect donor eligibility. + +## Retained source boundaries + +The qualifier borrows the original authenticated survey preparation and its +retained native ASEC/current-money owner. Before capture it checks the live +money domain against the pinned dividend contract. The domain reader +requires a unique field and current vintage, refuses unsupported negative, +missing-code or zero semantics, and caches only immutable values. Public +entry mappings are detached on every call. + +The exact member's archive and member digests, row count and byte count +must agree with the retained owner. A single bounded capture is checked +again after reading. All current-year native keys, original household and +person-line coordinates, and source ages are joined before selection. +`DIV_VAL` validity and every valid float64 bit must match the actual money +owner, including people omitted from the selected survey. Original +`DIV_VAL_parent_statuses`, `DIV_VAL_parent_validity` and +`DIV_VAL_parent_zero_origin` remain in the projection. + +`CurrentAsecDividendValues(person, asec_literals, evidence)` is descriptive +transport. `person` is indexed by `person_id` and includes `native_person_id` +and `source_age`; `asec_literals` retains the complete current-year literal +table indexed by native person ID. `dividend_values_seal(values)` covers +both tables, their axes, exact float bits, nullable masks and backing +storage, and detached evidence. The qualifier rechecks the original +preparation after source I/O and verifies the complete seal before return. + +Consumers must retain and requalify the actual preparation before +consumption and after their last relevant I/O, and compare the complete +values seal. Matching JSON, a copied dataclass or a digest does not grant +source authority. No host, graph attachment, tax leaf, source issuer, +original/PUF clone change or release gate is added here. diff --git a/docs/us-current-asec-income-routing-source.md b/docs/us-current-asec-income-routing-source.md new file mode 100644 index 000000000..22968e633 --- /dev/null +++ b/docs/us-current-asec-income-routing-source.md @@ -0,0 +1,272 @@ +# Current ASEC income reporting and routing source + +`microcosm.build.us_runtime.current_asec_income_routing_source` qualifies the +exact retained 2025 ASEC person member against the authenticated current-money +owner and projects five original-channel income families the current survey +graph does not yet supply: + +| Family | Printed amounts | Printed receipt / routing | +| --- | --- | --- | +| `pension_annuity` | `PNSN_VAL`, `ANN_VAL` | `PEN_YN`, `ANN_YN` | +| `retirement_distribution` | `DST_VAL1`, `DST_VAL1_YNG`, `DST_VAL2`, `DST_VAL2_YNG` | `DST_YN`, `DST_YN_YNG`, `DST_SC1`, `DST_SC1_YNG`, `DST_SC2`, `DST_SC2_YNG` | +| `net_property` | `RNT_VAL` | `RNT_YN` | +| `farm` | `FRSE_VAL` | `FRSE_YN`, `ERN_YN`, `FRMOTR` | +| `other_income` | `OI_VAL` | `OI_YN`, `OI_OFF` | + +Thirteen published allocation flags are read alongside them. + +Both returned frames carry a digest: `evidence["projection_sha256"]` over +`person` and `evidence["literals_sha256"]` over `asec_literals`. Both are +re-checked at the end of the qualifier, so a host comparing the documented +digests detects a corrupted transport on either frame. + +`qualify_current_asec_income_routing(preparation)` takes a live +`AuthenticatedSurveyPopulationPreparation`, requalifies it, captures the pinned +2024 member once, joins it to the money parent by exact native keys, and returns +`CurrentAsecIncomeRoutingValues(person, asec_literals, evidence)`. The returned +values are descriptive transport. Constructing or mutating them issues no source +authority: a consuming host re-runs this qualifier and compares the result +around its own I/O, exactly as the accepted unemployment qualifier requires. + +## What the qualifier establishes + +- **The exact source member.** The 2024 pin is matched against the retained + native coverage receipt (member name, archive digest, member digest, row count + and byte size) before the capture, and the captured file's stat identity and + SHA-256 are re-checked after it. +- **Row-for-row parent correspondence.** `PH_SEQ`, `A_LINENO` and `A_AGE` from + the member must equal the money parent's `source_household_id`, `A_LINENO` and + `A_AGE` at the 2024 scope positions. The join is by native key, so member row + order cannot change the projection. +- **Literal/amount identity.** Every projected amount is the money owner's own + value, checked against the member literal, with the parent's `statuses`, + `validity` and `zero_origin` carried through as `amount_status_*`, + `amount_validity_*` and `zero_origin_*` columns. +- **A single bounded pass.** One capture, one read, one projection. There is no + per-family re-capture, no nested run issuer, and no PUF borrow. + +## What it deliberately does not establish + +- `PNSN_VAL` is printed as the *total combined amount of pension income received + from all pension sources* — `PEN_SC1`/`PEN_SC2` enumerate company, union, + federal, state, local, military and railroad pensions. It is not an observed + private or taxable amount. The archived `TAXABLE_PENSION_FRACTION = 0.590` + split is a modelled assumption and is not applied or referenced here; + `pension_annuity_private_share_applied` and + `pension_annuity_taxable_amount_known` are constant `False`. +- Account code 4 is `Regular IRA`. That identity does not observe any taxable + fraction, so `retirement_distribution_taxable_amount_known` is constant + `False` and no taxable component is derived. +- `RNT_YN` asks about land, property rented to others, royalties, roomers or + boarders, and estates or trusts, while `RNT_VAL` asks only about *income from + rent after expenses*. Both scopes are carried on every row and + `net_property_component_split_known` is constant `False`, so the total is not + independently labelled rental and no subdivision is separately modelled. +- `FRSE_VAL` is farm self-employment, not nonfarm `SEMP_VAL`; + `farm_is_nonfarm_self_employment` is constant `False`. Its printed label + includes the composite clause naming `ERN_VAL` (when `ERN_SRCE=3`) and + `FRM_VAL`, so the total already contains those components. +- `retirement_distribution_regular_ira_amount` is published only when every + applicable slot is fully resolved on both axes. An unreadable account code + could itself be a regular IRA, and a declared account whose amount is a + "none or niu" zero does not observe a zero dollar distribution, so both leave + the regular-IRA share unknown rather than at zero. +- `OI_OFF` code 20 is the reported alimony category. Nothing maps any other + code — including 19, `anything else` — onto alimony or onto a miscellaneous + residual; `other_income_residual_rule_applied` is constant `False`. + `other_income_routing_status` separates `reported_category` from + `receipt_without_category`, `category_without_receipt`, + `unresolved_receipt_routing`, `missing_category_literal`, + `unrecognized_category_literal` and `niu_category`, and + `other_income_is_reported_alimony` is set only on a `reported_category` row, + so an unreadable receipt literal never yields a reported alimony receipt. +- ACS-channel people are not projected at all. `acs_components_modeled` is + `False` in the evidence and the projection covers only ASEC-channel rows. + +## Reporting status vocabulary + +`receipt_status` is the single classifier. Only `known_receipt`, +`known_recipient_zero` and `known_nonreceipt` establish a dollar reading +(`KNOWN_AMOUNT_STATUSES`); every other status leaves the canonical amount +unknown rather than completing it with zero. + +| Status | Meaning | +| --- | --- | +| `known_receipt` | in universe, receipt yes, amount non-zero (a signed loss included) | +| `known_recipient_zero` | in universe, receipt yes, amount zero on an entry whose printed zero is valid dollars — `ANN_VAL` alone, whose NIU is the separate `-1` code | +| `receipt_with_net_zero` | in universe, receipt yes, amount zero on a signed net measure (`RNT_VAL`, `FRSE_VAL`). Distinct from a gross entry's recipient zero, but **not** a known amount: both print `0 = none or niu` | +| `ambiguous_recipient_zero` | in universe, receipt yes, amount zero on a gross entry whose printed zero reads "none or niu" | +| `known_nonreceipt` | in universe, receipt no, amount zero | +| `niu` | in universe, receipt 0, amount zero or a declared non-money code | +| `missing_amount` / `missing_receipt_literal` | one side of the pair is absent | +| `unrecognized_receipt_literal` | the receipt literal is malformed or outside the printed range | +| `contradictory_no_nonzero`, `contradictory_niu_nonzero`, `contradictory_declared_niu_amount`, `contradictory_outside_reporting_universe` | retained source contradictions, excluded from every canonical amount | +| `contradictory_offroute_evidence` | distributions only: the route that does not apply carries dollars or an answered recipiency | +| `unresolved_slot_composition` | distributions only: an applicable slot declares an account whose amount is a "none or niu" zero | +| `outside_reporting_universe` | the printed universe excludes the row and the literals agree | +| `unresolved_reporting_universe` | the printed universe cannot be resolved from the retained literals | + +Which recipient zeros resolve is read from the pinned domains artifact, not +decided here: `_zero_is_dollars` compares each entry's `zero_semantics` against +`valid_zero_dollars`. Eight of the nine entries record +`none_or_niu_not_distinguishable_from_amount_alone`, including both signed net +measures, so a signed zero is separated by label but never completed. + +The receipt code labels are per entry, not shared: `receipt_codes(field)` reads +the printed zero label from `RECEIPT_ENTRIES`, so `OI_YN` reports `none or niu` +where `PEN_YN` reports `niu` and `FRSE_YN` reports `Niu` exactly as printed. + +Other-income routing (`other_income_routing_status`) follows the same universe: +`OI_OFF` is printed for `OI_YN = 1` and `OI_YN` for persons aged 15+, so a row +outside or unresolved on that universe reports +`outside_reporting_universe_routing` or `unresolved_reporting_universe_routing` +and never a reported category. + +Universes are per family and never reduced to age alone. `PEN_YN`, `ANN_YN`, +`RNT_YN` and `OI_YN` print `All Persons aged 15+`. `FRSE_YN` prints +`ERN_YN=1 or FRMOTR=1`, so the farm universe is `True` when either literal is +`1`, `False` when both are known and neither is, and unresolved otherwise — +a nonfiler is never treated as a non-recipient. + +## Source-level questions preserved as evidence + +These are printed-dictionary facts, kept distinguishable from modelling +judgments. The first four are recorded under +`evidence["dictionary"]["printed_universe_questions"]`; the rest under +`evidence["dictionary"]["printed_scope_and_code_questions"]`: + +- `DST_VAL1`'s printed universe is `DST_SC1 = 1` although its label names the + source-1 distribution amount; taken literally that would restrict it to 401k + accounts. Retained verbatim, not silently corrected. +- `DST_SC2_YNG`'s printed universe names `DST_VAL_YNG`, a field with no + dictionary entry. +- `I_DSTVAL1COMP`'s printed `Universe:` line is empty. +- `DST_YN` and `DST_YN_YNG` print only the age-58 split. They print no 15+ floor + as the four age-universe families (`PEN_YN`, `ANN_YN`, `RNT_YN`, `OI_YN`) do, + and they are not gated on other literals as the farm family is — `FRSE_YN` + prints `ERN_YN=1 or FRMOTR=1` and so carries no age floor either. Coverage + below age 15 is therefore recorded as unresolved rather than resolved either + way. +- `OI_YN`'s printed zero label is `none or niu` where `PEN_YN`, `ANN_YN`, + `DST_YN` and `RNT_YN` print `niu`, so a zero receipt literal is not + interchangeable across families. +- `DST_SC1` is gated on `DST_VAL1 > 0 and a_age ≥ 58` while `DST_SC1_YNG` is + gated on `DST_YN_YNG = 1 and a_age < 58` — the two routes are not symmetric. + +Both printed recipiency literals are retained per row +(`retirement_distribution_receipt_58_*` and `..._receipt_young_*`) alongside all +four slots. The age route selects which pair applies; +`retirement_distribution_offroute_receipt` and +`retirement_distribution_offroute_nonzero` flag an answered recipiency or a +non-zero amount on the route that does not apply, so the complete slot/receipt +pattern stays inspectable rather than being reduced to the applicable half. + +## Amount semantics from the money owner + +The authenticated money owner normalizes `ANN_VAL`'s printed `-1` to a stored +zero and records `CodebookStatus.DECLARED_NIU`. `amount_state` therefore reads +the dollar meaning of a cell from the parent's status axis, never from the +stored number: re-deriving NIU from the number would read that cell as a zero +dollar annuity. `evidence["declared_niu_normalized_to_zero"]` (a top-level key) lists the +affected entries. + +The money owner restates non-2024 cohorts to the pinned price basis, so the +literal-identity join is taken only on `person_years == 2024`; widening it to +the pooled 2022/2023 cohorts would require handling that restatement first. +`evidence["restatement_note"]` records this. + +## Allocation provenance + +`*_allocation_origin` is derived from published flags only: + +- `publisher_allocated` — some applicable flag is non-zero. +- `published_flags_all_zero` — every applicable flag is populated and reads + zero, and every field in the family carries a published flag. This is **not** + an assertion of non-allocation: each flag prints a conditional universe (for + example `I_RNTVAL` is printed for `RNT_VAL > 0`) and those universes are not + evaluated here, so a non-recipient row sits outside its own flag's universe. +- `published_flags_all_zero_with_unflagged_fields` — the same, but the family + also holds a field the dictionary does not flag. +- `allocation_flag_not_populated` — a flag literal is absent. Every flag here + prints a conditional universe, so this is not a defect and not "no + allocation". +- `unresolved_allocation_provenance` — a flag literal is malformed or outside + its printed range. + +`PUBLISHED_ALLOCATION_FLAG_BY_FIELD` names which flag covers which field. +`UNFLAGGED_FIELDS` records the ten fields for which the 2025 dictionary +publishes no flag: `PNSN_VAL`, `FRSE_VAL`, `FRSE_YN`, `OI_OFF`, `OI_YN`, +`DST_VAL1_YNG`, `DST_VAL2_YNG`, `DST_YN_YNG`, `DST_SC1_YNG`, `DST_SC2_YNG`. +`AMBIGUOUS_FLAG_COVERAGE` records three printed ambiguities: `I_DSTSCCOMP`'s +label names `DST_SC(2)` while its universe names both routes, so its coverage of +the under-58 source codes is unresolved; `I_DSTSC`'s label uses the same +`DST_SC(2)` notation, so mapping it to both `DST_SC1` and `DST_SC2` reads the +parenthesis as a slot count that the dictionary does not state; and `I_FRMYN` +prints an empty `Values:` block, so only its `(0:9)` range header is published +and no meaning is claimed for its codes. A nonzero `I_FRMYN` alone yields +`allocation_code_meaning_unpublished`; a documented nonzero allocation flag +such as `I_ERNYN` can still establish `publisher_allocated`. All-zero readings +remain descriptions of the published codes, not assertions of nonallocation. +The original flag values and parse statuses remain unchanged. No row is ever labelled a raw +respondent value. + +## Constants + +The nine printed money entries are **not** retyped here: they are read on demand +from the packaged `asec_current_money_domains_v1.json` under its pinned digest +(`money.RESOURCE_PINS[0]`), which already attests printed position, length, +page, range, universe and values per vintage. The routing code systems +(`RECEIPT_CODES`, `ACCOUNT_CODES`, `OTHER_INCOME_CATEGORIES`, +`ALLOCATION_ENTRIES`) are not in that artifact and are defined here from the +pinned dictionary. + +The printed entry tables (`RECEIPT_ENTRIES`, `ACCOUNT_ENTRIES`, +`ALLOCATION_ENTRIES`, `OTHER_INCOME_CATEGORY_ENTRY`) are `NamedTuple`s, so every +use site and the emitted receipt reach their fields by name rather than by +position. + +Two adjacent definitions exist in tree and are deliberately **not** imported, so +this qualifier keeps the accepted source-only import neighbourhood +(`asec_coverage_authentication`, `asec_current_money`, `source_csv_builtin`, +`survey_population_preparation`, `support_provenance`) and stays free of the +modelled stage machinery: + +- `retirement_distributions.US_RETIREMENT_DISTRIBUTION_REQUIRED_SOURCE_COLUMNS` + — the same eight `DST_SC*`/`DST_VAL*` column names. +- `alimony._ASEC_ALIMONY_OTHER_INCOME_CODE` and + `alimony._ASEC_STRIKE_BENEFITS_OTHER_INCOME_CODE` — the same category codes 20 + and 12. +- `retirement_distributions._VALID_ACCOUNT_CODES` — the same 0-7 account code + domain that `ACCOUNT_CODES` gives printed labels for. + +`test_routing_code_systems_agree_with_the_existing_domain_constants` asserts +both agreements, per `docs/shared-constants.md` rule 5. If a later change makes +the modelled modules importable from a source qualifier, import those +definitions directly and drop the local copies. + +`RINT_SC1`/`RINT_SC2` reuse the identical 0:7 retirement-account enumeration for +retirement *interest*. They are out of this slice; a future consumer mapping +account codes to outputs must decide whether they share this code system. + +## Remaining work + +This lane delivers source qualification only. Still open, and explicitly not +decided here: + +1. **Canonical attachment.** Which projected columns become graph leaves, on + which clone, and under what name. These families overlap PUF-owned values, so + the unemployment/health both-clone policy cannot simply be reused. Root's + active amount owner owns `graph_us_survey_enrichment`. +2. **ACS clone0 components.** ACS publishes aggregate anchors + (`acs_retirement_income`, `acs_interest_dividend_rental_income`) rather than + these components. A conditional model plus a documented reconciliation rule + is needed, and the existing interest/dividend draws and ACS `INTP` overlap + mean rent cannot be added independently without checking that aggregate. +3. **Tax composition.** The pension private/taxable split and the distribution + taxable fraction remain unobserved. Any split must be approved explicitly + against the source, not inherited from the archived assumptions. +4. **Net property decomposition.** The receipt/amount scope mismatch must be + resolved before rent, royalties, roomers/boarders and estate/trust income are + modelled as separate components. +5. **Other-income residual.** Categories other than 20 are preserved but + unrouted; a residual-to-miscellaneous rule is a separate, reviewed decision. diff --git a/docs/us-current-asec-interest-source.md b/docs/us-current-asec-interest-source.md new file mode 100644 index 000000000..d4cbfbfd8 --- /dev/null +++ b/docs/us-current-asec-interest-source.md @@ -0,0 +1,91 @@ +# Current ASEC interest observations + +`current_asec_interest_source.qualify_current_asec_interest(preparation)` +qualifies ordinary and retirement-account interest from the original 2025 ASEC +person member, referring to income in 2024. It preserves the already qualified +`INT_VAL` combined total and exposes the published components and their +reporting, account, allocation and disclosure metadata. + +The source is the [2025 ASEC public-use data dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA-256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The exact total-income domain is reused from the pinned +`asec_current_money_domains_v1.json`; the additional fields use this same +dictionary. Pages below are one-based PDF pages. + +| Fields | Meaning and printed universe | Page | +| --- | --- | --- | +| `INT_VAL`, `INT_YN` | Combined interest; amount universe `INT_YN = 1`, receipt universe age 15+ | 46 | +| `TRDINT_VAL` | Interest excluding retirement-account interest; `INT_YN = 1` | 50 | +| `RINT_YN` | Retirement-account interest receipt; age 15+ | 49 | +| `RINT_SC1`, `RINT_SC2` | Account identities, including 401k, 403b, Roth IRA, Regular IRA, KEOGH, SEP and other; `RINT_YN = 1` | 49 | +| `RINT_VAL1`, `RINT_VAL2` | Interest in the two reported account slots; corresponding account code greater than zero | 49 | +| `I_INTVAL`, `I_INTYN` | Combined-interest allocation flags, with distinct composite code systems | 56 | +| `I_RINTSC`, `I_RINTVAL1`, `I_RINTVAL2`, `I_RINTYN` | Retirement-account allocation flags; `I_RINTSC` names slot 1 only | 57 | +| `TRINT_VAL1`, `TRINT_VAL2`, `TTRDINT_VAL` | Published top-code indicators for the respective positive component amounts | 60 | + +The account, receipt and applicable allocation code systems reuse the income +routing qualifier. `I_INTVAL` accepts the printed codes 0 and 11–15; its header +interval does not make codes 1–10 valid. There is no individually published +allocation flag for `TRDINT_VAL` or `RINT_SC2` in this dictionary. Allocation +labels describe publisher processing, not observation validity or tax status. + +## Amounts and knownness + +The result keeps raw source literals, parsed published amounts, parse status, +reporting status and canonical amount knownness separately. Missing and +malformed components remain unknown. Interest amounts must be nonnegative +integers within each field's published encoding; these disclosure ranges are +not latent-income bounds. + +`TRDINT_VAL` describes its range as a dollar value, so a published zero with +combined-interest receipt `yes` is an observed zero ordinary component. +`INT_VAL` and `RINT_VAL1/2` describe zero as none or NIU, so zero with receipt +`yes` remains ambiguous. A retirement-interest recipient with account code 0 +and slot amount 0 has an `unreported_account_slot`, not an imputed account +amount. Receipt `no` plus account code 0 and amount 0 establishes known +nonreceipt. Under-15 NIU values never become analytical zeros. + +The diagnostic `combined_minus_published_components` subtracts the three +parsed published component cells from `INT_VAL`. Its knownness means that all +four published numbers are readable; it does not establish that all three +components are reported observations. `all_component_amounts_observed` +separately records that stronger condition. The diagnostic may be negative; +the qualifier never balances, clips or replaces any published number. + +## Source lifetime and composition + +Qualification borrows the actual `AuthenticatedSurveyPopulationPreparation` +and its retained native ASEC/current-money owners. It matches the original +member's archive digest, member digest, byte count and row count, captures that +member once, and verifies the captured file after reading. Current-cohort +`PERIDNUM` keys must form an exact roster; household coordinates, person line +numbers and raw ages must match the retained parent. Source row order is +irrelevant. The original `INT_VAL` validity and float64 bits must agree with +the current-money owner before selected ASEC people are projected. + +`CurrentAsecInterestValues(person, asec_literals, evidence)` is descriptive +transport. Its seal covers both complete tables, axes, nullable masks and +backing storage, exact float bits and evidence. The qualifier rechecks the +actual preparation after its last source I/O and checks the complete value +seal before returning. A consumer must retain the actual preparation, +requalify it immediately before consumption and after its own last relevant +I/O, and compare the full value seal before returning or exporting a +successor. A copied dataclass, evidence dictionary or matching digest cannot +grant source authority. + +The only existing helper change is an optional explicit column/amount-grammar +roster on the private income-routing literal reader. Its original defaults +remain in effect for the income-routing qualifier. This module adds no host, +run issuer, graph execution or model fitting. + +## Bridge work still required + +[ACS response guidance](https://www.census.gov/programs-surveys/acs/respond/get-help.html) +includes interest credited to retirement accounts in its broad +interest/dividend/property item. The live guide currently refers to the 2025 +questionnaire; it is not an archived 2024 instruction artifact. This +ASEC qualification therefore keeps ordinary interest and retirement-account +interest separate for a later, reviewed measurement bridge. It does not +declare either component taxable, tax-exempt, regularly withdrawn, or equal +to an ACS component. It does not alter tax leaves, original or PUF clones, +financial imputation, calibration or release eligibility. diff --git a/docs/us-current-asec-property-basis.md b/docs/us-current-asec-property-basis.md new file mode 100644 index 000000000..40a4be5ea --- /dev/null +++ b/docs/us-current-asec-property-basis.md @@ -0,0 +1,132 @@ +# ASEC property donor basis + +`build_asec_property_basis` is a pure operation over the selected, qualified ASEC +person descriptions. It does not qualify a source, issue a handle, read files, +fit a model or assign tax leaves. The owning country host retains the actual +preparation and source owners, checks them before consumption and requalifies +and checks their complete seals after its last relevant I/O before return or +export. Reconstructing the descriptive input or output dataclasses grants no +source authority. + +## Inputs and outputs + +The keyword-only inputs are the `person` tables from the current interest, +income-routing and dividend/survivor qualifiers, named `interest`, +`income_routing` and `dividend`. Every table must have the same exact ordered +`person_id` index, matching unique `native_person_id` values and matching source +ages. Both ID axes are int64. Source ages may use different numeric storage +but must be finite, integer-valued ages 0–99 and equal without tolerance. No +sorting or inner join repairs a mismatch. A common row permutation is valid; +permuting one source alone refuses. + +`original_household_membership` is an int64 Series on that exact person index. +`original_household_design_weights` is a finite, nonnegative Series indexed by +unique int64 `household_id`, with exactly the membership's household set. +Extra, missing or duplicated household keys refuse. These inputs must be the +original household design mapping, before importance allocation or cloning; +this pure operation cannot authenticate that origin. Zero-weight persons are +retained. No engine or microsimulation aggregation occurs. + +The descriptive `AsecPropertyBasis` contains: + +- `person`: the component values, reported total, discrepancies, named retirement + derivations, original household mapping and the two eligibility masks. +- `provenance`: detached copies of all three supplied qualified tables, with + `interest.`, `income_routing.` and `dividend.` prefixes. Published amounts, + unknownness, allocation and disclosure indicators are preserved independently. +- `exclusions`: independent reasons for exclusion on the complete person axis. + A person can have multiple reasons; reason totals must not be summed as a + disjoint partition. +- `summary`: counts and original design-weight mass for the whole cohort, each + eligibility mask, the joint-fit complement and each exclusion reason. + +The shared names live in `property_income_constants.py`. `PROPERTY_COMPONENTS` +is, in order, `property_ordinary_interest`, `property_retirement_interest`, +`property_dividends`, `property_broad_receipts`. `PROPERTY_REPORTED_TOTAL` names +`property_reported_total`. They are donor observations or named routing +contributions, never already-reconciled model draws. + +## Measurement bridge + +Ordinary interest uses qualified `TRDINT_VAL`; dividends use qualified +`DIV_VAL`; broad signed property receipts use qualified `RNT_VAL`. The latter +is not a pure rental-income tax leaf. Dividends need a qualified yes/positive +or no/zero receipt. A receipt-yes zero for dividends or signed property stays +unknown under the published none-or-NIU coding. + +Retirement-account interest uses the two qualified account slots. Known no +with both slots NIU and zero supplies the already-qualified zero contributions. +Known yes requires at least one declared account with a known positive amount. +Every other slot must be another known positive account or an explicitly unused +slot: readable account code 0, published amount 0, in-range amount literal and +`unreported_account_slot` source status. The latter contributes a **derived** +zero and gets its own boolean, slot count and derivation label. Its original +canonical amount stays unknown in provenance. An amount zero by itself, +a declared account with unknown/zero amount, yes without any active account, +a missing code, or contradictory receipt/account evidence does not resolve the +retirement total. The derivation applies only when the complete route resolves. + +The reported donor total is separately retained as `INT_VAL + DIV_VAL + RNT_VAL`. +The component sum uses ordinary plus retirement-account interest, dividends and +broad property receipts. Both `interest_component_discrepancy` and +`reported_minus_component_total` remain visible, with signed magnitudes. No +balancing rewrites an ASEC observation. Allocation never filters the donor fit. +Disclosure flags do not prove the cause of a discrepancy. + +Other-income clearance requires known nonreceipt with a NIU category, or a +readable reported category outside property codes 5–8 and unspecified code 19. +A reported outside category resolves its route even if its own dollar amount +is ambiguous; no other-income amount is included in this basis. Missing, +unreadable or contradictory routing stays unresolved. The dividend qualifier's +nullable survivor clearance is checked against its readable codes and retained +as `survivor_visible_routes_clear`. It covers only `SUR_SC1` and `SUR_SC2`. +The 2025 dictionary describes `SRVS_VAL` as including edited sources 1/2 plus +unedited sources 3/4 (PDF page 49, printed page 6C-28); those additional source +types are not qualified here. A positive survivor receipt therefore cannot +establish complete absence of estate/trust income even when both visible +slots are outside codes 8 and 10. + +The first bridge accepts only known survivor nonreceipt with readable NIU +slots, exposed as `survivor_full_scope_clear`. Positive receipts have a separate +`survivor_additional_sources_unresolved` exclusion and weighted coverage row. +Possible visible property overlaps and unknown visible routes retain their own +exclusion reasons. Overlapping exclusion rows must not be added together. +Neither other income nor survivor amounts enter the component or reported sum. + +`reported_total_eligible` requires age 15+, a known finite reported total, and +clear other-income routes and complete survivor scope. It is a diagnostic sample, not a second +aggregate model. `joint_component_fit_eligible` additionally requires all four +components and an exactly zero interest discrepancy. Missing or under-15 values +are never turned into analytic zeros. Negative and zero net totals remain valid: +`(100, 20, 0, -120)` has total zero and keeps both positive interest components. + +The summary reports `design_weighted_person_mass` (one mapped original household +design weight per selected person) separately from `union_household_design_mass` +(each household with at least one selected person counted once). Overlapping +selection groups can share household mass. Stable finite summation avoids +row-order-dependent loss of small weights. These are coverage diagnostics, not +calibrated population claims. Conditional donor support, bias from exclusions +and held-out model quality still need the model owner's diagnostics. + +## Source and acceptance boundaries + +The source contracts use the pinned +[2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +(SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`). +The bridge targets the broad property concept in the +[2024 ACS questionnaire](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18). +The live ACS guide follows the 2025 form; its retirement-interest wording and +the historical 2016 Census methods discussion corroborate the concept without +establishing vintage or respondent identity. ASEC calendar-year and ACS +rolling-year measurement remain different. Broad property routing and +retirement-account earnings remain distinct from taxability. + +The bounded tests use invented tables and a composition test through the real +pure source projectors using invented literals. They cover both identity axes, +permutations, missing/contradictory routes, unknown account slots, signed and +zero-net examples, separate masks, allocation preservation, weight mapping, +empty/zero-weight records and refusal of malformed/nonfinite descriptions. +They do not run source qualification, native data, the country model, fitting, +cloning or replay. The later graph owns joint draws and signed reconciliation; +the later host must preserve original ASEC observations and share survey draws +across original/PUF clone pairs before the PUF alternative takes ownership. diff --git a/docs/us-current-asec-retirement-basis.md b/docs/us-current-asec-retirement-basis.md new file mode 100644 index 000000000..890b15df8 --- /dev/null +++ b/docs/us-current-asec-retirement-basis.md @@ -0,0 +1,164 @@ +# ASEC retirement candidate ledger + +`current_asec_retirement_basis.py` describes potential retirement-bridge support +from the accepted retirement-detail and income-routing projections. It preserves +source observations, route uncertainty and published accounting differences. +It produces no fiscal inputs, fitted model, graph host or source authority. + +## API + +`build_asec_retirement_basis` accepts keyword arguments `retirement_detail`, +`income_routing`, `original_household_membership`, +`original_household_design_weights`, and `assumptions`. The two DataFrames must +have identical ordered int64 `person_id` indices, exact native person IDs, and +original interview ages. Shared amount observations and pension receipt codes +must agree. A joint permutation is accepted; a one-sided permutation refuses. +The current contract requires at least one selected person. + +Membership must cover exactly those selected person IDs in the same order. +The weight Series must have a unique int64 `household_id` index containing +exactly the households represented by that membership, with finite nonnegative +values. Extra unselected household keys refuse, including keys from a larger +original survey. The caller must select the exact original household weights +without replacing them with clone, importance, or calibrated weights. Original +weight custody remains external; a Series does not authenticate DESIGN status. + +It returns `AsecRetirementBasis` with six descriptive fields: + +- `person`: source identifiers, observed routed subtotals, candidate family + intervals, accounting differences and candidate-support flags. +- `slots`: ten records per person, retaining six pension/disability/survivor + slots and four applicable/off-route distribution slots, source codes, + published amounts, knownness, source statuses and bridge routes. +- `provenance`: detached, prefixed copies of both complete supplied projections, + including allocation and disclosure flags. +- `exclusions`: uncertainty and route diagnostic masks, including cases that + remain numerically usable. These masks are not all applied as donor filters. +- `summary`: counts and supplied original DESIGN-weight support for each + candidate tier and diagnostic mask. +- `assumptions_payload`: canonical immutable bytes identifying the choices. + +The source projection dataclasses and these outputs contain mutable descriptive +tables. Neither their type nor a digest grants source admission. A future host +must retain and requalify the actual original source owners, complete person +scope, table seals and original DESIGN weights around its relevant I/O. This +pure function does not write to either input, calibrate weights or attach clones. + +## Required choices + +Every `RetirementCandidateAssumptions` field must be supplied. No country caller +or development scenario is enabled by this change. + +| Field | Supported values | +| --- | --- | +| `pension_annuity_regularity` | `unresolved`, `assume_regular` | +| `disability_pension_eligibility` | `unresolved`, `assume_qualifying` | +| `survivor_annuity_overlap` | `unresolved` | +| `withdrawal_regularity_netting` | `unresolved` | +| `aggregate_accounting` | `exact_visible_balance_only` | + +`assume_regular` is a measurement scenario about pension/annuity regularity. +`assume_qualifying` explicitly assumes both the disability-pension scope and +regularity needed for the candidate bridge. Neither is inferred from an annual +amount, age, `DIS_HP`, or `DIS_CS`. The latter source answers remain provenance. +There are no private-pension, regular-withdrawal or taxable fractions. Changing +either supported scenario changes `to_bytes()`; unsupported choices refuse. + +## Routes and candidate amounts + +The accepted [retirement-detail source](us-current-asec-retirement-detail-source.md) +and [income-routing source](us-current-asec-income-routing-source.md) retain the +published codebooks. This ledger imports those definitions and applies a separate +bridge route classification: + +| Family | Candidate routes | Separate or unresolved routes | +| --- | --- | --- | +| Pension | Codes 1–6 | 7 Railroad; 8 unresolved | +| Disability | Codes 2–5, subject to the explicit scenario | 6 Railroad; 1/7/8/9 other compensation; 10 unresolved | +| Survivor | Codes 1–4 are visible candidate subtotals | 5 Railroad; 8 property; 9 annuity overlap; 6/7/10 unresolved | +| Distribution | Applicable account codes 1–7 retained individually | Regularity/netting remain unknown; account 4 is a type of IRA | +| Other income | 2/13 indicate additional retirement candidates | 1 Social Security overlap; 8 property overlap; 19 unspecified | + +The retirement ledger never consumes account-earnings amounts or adds Railroad +amounts to Social Security. Other-income amounts are retained separately and +never added to the candidate sum. Categories 2/13/19 leave combined candidate +scope unresolved; known other categories can remain outside that sum. These +routes do not establish any person's tax treatment. + +A source-known amount and a published literal remain separate. A declared +positive slot contributes to its observed routed subtotal. A readable unused +NIU slot can contribute an explicitly labeled structural zero to a numerical +comparison, but its source-known amount remains missing. The subtotal is only +a sum of the readable known slots; it never asserts that unknown slots are zero. +Yes-receipt dollar zeros stay ambiguous where the source domain says none/NIU. +Annuity preserves its distinct valid dollar zero and published `-1` NIU encoding. + +The four published total-minus-slot comparisons are recomputed and checked +against R1. `DBTN_VAL` compares the main distribution slots only; the applicable +known distribution total is supplied independently by the routing owner. No +comparison is allocated to another source or attributed causally to disclosure. +For age 58 and over, the main slots apply and this exact comparison must be +finite and zero before a distribution candidate interval is available. A +positive residual leaves additional scope unresolved; a negative residual is +contradictory accounting. Both retain the published DBTN total, applicable slot +total and signed difference without a cap, clipping or redistribution. The main +slot comparison does not constrain the separate applicable young slots below 58. + +For pension/disability candidate bounds, all relevant slots must be readable and +receipt-consistent, and their exact integer-dollar sum must equal the aggregate. +Positive discrepancy leaves additional scope unresolved; negative discrepancy +is contradictory accounting. Neither is clipped, used as a cap, or silently +allocated. Readable outside routes are subtracted only under this exact balance. +With unresolved regularity/scope the lower bound is zero; under the named +scenario it includes only known candidate-route amounts. Unresolved source +routes can widen the candidate interval without becoming identified labels. + +Positive survivor receipt keeps full survivor bounds unknown, even when its +published comparison balances. Visible slots do not prove the scope of sources +3/4. Only qualified no receipt with two NIU slots and zero total supplies a +zero survivor family amount. Source 9 alongside positive annuity is explicitly +flagged; no overlap priority or max/sum assumption is applied. + +A fully resolved applicable distribution composition produces a zero-to-total +candidate interval only when the applicable aggregate check also passes. +Missing/unreadable account types, ambiguous active zeros or off-route +answers/dollars prevent that interval. Off-route receipt and account literals +must be readable NIU codes, and off-route amounts must be readable zeros; +unreadable literals are uncertainty, not evidence of absence. The routing owner's +known total is still preserved when the stricter candidate test fails. + +Routed subtotals and distribution account subtotals are evidence only, not +admitted candidate amounts. In particular, account subtotals can be finite when +all account types/amounts are readable but the applicable receipt is unreadable +or the DBTN comparison fails. `distribution_account_composition_known` describes +that account evidence; it does not establish a valid receipt, aggregate balance, +regularity, netting, or an available interval. Consumers must use the candidate +interval availability and family statuses rather than infer admission from a +finite subtotal. Raw observations remain in the detached provenance regardless. + +Under-15 people retain their source evidence with outside, unresolved-outside or +contradictory-outside status and no candidate zeros. Combined bounds are available +only when every family and other-income scope passes. `candidate_basis_eligible` +means this conservative numerical support exists. `candidate_point_under_assumptions` +means the candidate bounds coincide; `point_identified` and +`fiscal_outputs_produced` remain false. No training label is created merely by +calling a quantity an interval endpoint. + +## Source and validation boundaries + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +provides the source categories and domains. The [2024 ACS subject definitions, +page 95](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf#page=95) +provide the target's regularity and disability scope. The ledger's conservative +accounting and scenario choices are separate decisions. It does not equate the +surveys' reference periods, allocate joint income, or derive source regularity +or tax treatment from those documents. + +Tests use the actual pure R1/routing projectors on invented records. They check +source routes, exact accounting, unknowns, assumption identities, interview-age +routes, native IDs above `2**53`, one-sided mutations, permutations, detached +provenance and descriptive design-weight support. Person mass uses the supplied +original household weight for each included person; household mass counts each +included household once. The Series has no independent weight-kind authority. +Allocation/disclosure flags do not filter candidates. No native data, source +capture, engine, fitting, calibration, PUF or release acceptance is involved. diff --git a/docs/us-current-asec-retirement-detail-source.md b/docs/us-current-asec-retirement-detail-source.md new file mode 100644 index 000000000..e125201e4 --- /dev/null +++ b/docs/us-current-asec-retirement-detail-source.md @@ -0,0 +1,108 @@ +# Current ASEC retirement details + +`current_asec_retirement_detail_source` qualifies pension, disability and +survivor source details from the current ASEC member owned by an authenticated +survey preparation. This is source evidence for a later ACS retirement bridge. +It produces no model, tax leaf, payment-frequency classification or new issuer. + +## Source and field ownership + +The source is the [2025 ASEC public dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`, +for calendar-year 2024 income. Page references below are physical PDF pages; +the corresponding printed pages are 6C-23 through 6C-39. + +| Amount fields | Qualification | +| --- | --- | +| `PNSN_VAL`, `ANN_VAL`, `DST_VAL1/2`, `DST_VAL1_YNG/2_YNG` | Reuse immutable printed domains from the existing income-routing owner. Compare exact retained current-money values and validity. They remain reference observations here; their canonical receipt/account interpretation stays with that owner. | +| `DIS_VAL1/2` | Explicit dictionary domains on page 44, also compared against actual retained current-money values and validity. | +| `PEN_VAL1/2`, `SUR_VAL1/2`, `DSAB_VAL`, `DBTN_VAL`, `SRVS_VAL` | Independently qualify literals from the exact authenticated source member using published width, range, universe and zero semantics. These seven fields are absent from the current retained-money roster; the qualifier never fabricates a `ready.field()` for them. | + +The live money domains must agree with the eight existing fields. A future +money roster containing one of the seven independent fields fails until the +corresponding retained comparison is reviewed. The amount-entry cache contains +immutable entries; each public call returns a detached mapping. + +The complete source join verifies native person keys, household IDs, person +line numbers and source ages. Selected original ASEC persons retain their +actual composed person IDs. No clone or PUF population is modified. The source +literal table covers the complete current source roster; the person table +covers selected ASEC persons. + +## Knownness and comparisons + +`PEN_YN`, `DIS_YN` and `SUR_YN` are separate from their two source codes and +amounts (pages 44–45, 47 and 49–50). A positive readable amount is canonical +only with a valid positive source code and yes receipt. A no receipt, NIU source +slot and zero amount yields explicitly derived known nonreceipt. A yes receipt +with a declared source and zero amount stays ambiguous. A yes receipt with a +zero source slot and amount stays unreported, not an observed analytical zero. +Missing, malformed, out-of-range and contradictory values stay distinct. + +People under 15 never acquire analytical zeros. Unreadable outside-universe +answers stay unresolved; readable nonzero outside-universe evidence is flagged +as contradictory. `DIS_CS` and `DIS_HP` are preserved source answers. The latter +includes limited work as well as prevented work and does not establish the +ACS disability-pension criterion. + +Four raw numerical comparisons retain total-minus-slot differences: + +- `PNSN_VAL - PEN_VAL1 - PEN_VAL2`; +- `DSAB_VAL - DIS_VAL1 - DIS_VAL2`; +- `SRVS_VAL - SUR_VAL1 - SUR_VAL2`; +- `DBTN_VAL - DST_VAL1 - DST_VAL2`. + +These comparisons include readable published none/NIU zeros; they are not +proof of analytical completeness. Missing or malformed components produce an +unknown comparison. No difference is allocated to a pension, taxable income +or missing source. `DBTN_VAL` is retirement distributions, not disability, and +its printed formula uses the main slots rather than young-person slots. + +`SRVS_VAL` includes edited sources 1/2 and unedited sources 3/4 (page 49). +Its visible pair is therefore never declared exhaustive. Source labels retain +Railroad Retirement, estate/trust, annuity and unspecified categories without +assigning them to an ACS aggregate. In particular, positive survivor receipt +with two visible non-property routes does not prove the absence of additional +estate/trust income. + +Allocation and top-code flags preserve their own published domains. Source +allocation flags with values 0/1/9 do not acquire all intermediate header-range +values. `I_SURVL2` retains its printed `SURV_VAL2` universe spelling. Allocation +does not invalidate an otherwise observed amount; a disclosure flag does not +explain a particular component discrepancy by itself. + +## Why the ACS bridge remains separate + +The [2024 ACS questionnaire, question 43g](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +asks about regular account withdrawals, while [ASEC Q98Ar](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf#page=205) +asks about any withdrawal/distribution. “Regular IRA” is an account label, not +payment frequency. Account type alone does not establish taxability. The +qualifier assigns neither regularity nor tax treatment and makes no ACS +retirement-component allocation. + +## API and validation boundary + +`project_retirement_detail_literals(DataFrame)` is a descriptive pure +projection. `qualify_current_asec_retirement_detail(preparation)` requires the +actual preparation and original source ownership, captures the pinned member, +checks source identity and retained money, and requalifies the parent after +the last source I/O. Its last checks include actual owner identity and a full +physical result seal. + +The retained-money comparison follows the existing owner's annuity encoding: +published `ANN_VAL=-1` matches normalized positive zero with `DECLARED_NIU`, +while the descriptive literal and published amount remain `-1`. A dollar-zero +literal cannot substitute for that NIU/status pair. Other valid dollar amounts +retain exact float64 comparison. This distinction is tested through actual +invented source preparation, as well as direct mismatched-status controls. + +`retirement_detail_values_seal(values)` covers table axes, dtypes, exact values, +nullable Float64 backing storage and masks, literals and detached evidence. +The returned frozen dataclass contains mutable descriptive tables and grants no +source admission. A consuming host retains/revalidates the actual parent and +the result at its own boundaries; a copied dataclass is not an issuer. + +Tests use only invented records and the real source-preparation path. They +cover source joins, retained versus independent fields, member-byte drift, +late parent mutation and result mutation beneath null masks. No native source +or country engine is needed, and passing tests do not certify a release. diff --git a/docs/us-current-property-income-sources.md b/docs/us-current-property-income-sources.md new file mode 100644 index 000000000..b722a1218 --- /dev/null +++ b/docs/us-current-property-income-sources.md @@ -0,0 +1,109 @@ +# Original property-income source composition + +`qualify_current_property_income_sources` composes the source branches for the +property-income model. It borrows the existing shared-predictor preparation, +qualifies the original ACS anchors and ASEC interest, income-routing and +dividend/survivor observations, and applies the reviewed pure ASEC donor basis. +It fits no model, supplies no tax split and performs no clone attachment. + +The arguments match the existing shared predictor boundary: the actual +preparation, allocated population and clone population, with optional shared +demographic conditioning and geography configuration. The allocated/clone +objects satisfy that existing ownership contract. Their weights never supply +the donor weights. All donor membership and DESIGN weights come from the +original source Frame selected by the shared qualifier. + +The result is descriptive `QualifiedPropertyIncomeSources`. Its source values, +source Frame and full origin document are retained for inspection and later +source/clone joins. The object has no issuer registry and cannot replace the +actual retained preparation. The complete descriptive seal is a mutation +check, not a credential. Downstream owners must qualify before consumption +and after their last relevant I/O before returning or exporting a successor. + +## Selected branches + +`donor_frame` is the original ASEC DESIGN Frame restricted to jointly eligible +persons, with referenced groups pruned by the existing Frame selector. +`donor_columns` contains the qualified shared features, reported property total +feature and four original components. It uses the same exact ordered person +IDs as that branch. The four component columns and reported total import their +canonical names from `property_income_constants.py`. + +`recipient_frame` selects original ACS persons with a finite, known adult INTP +anchor. `recipient_columns` contains the same shared features plus the qualified +ADJINC-adjusted anchor under the reported-total feature name. The existing +recipient matrix codec preserves their ordered identities and float64 bits. +`recipient_matrix` is the encoded matrix. Source AGEP and canonical shared +feature age remain distinguishable; no raw age is overwritten by the feature. +Shared earnings features preserve the existing qualifier's declared NIU operator; +this composition adds no new zero completion for earnings or property anchors. + +Adding these model columns to a graph population is a later visible graph +operation. The selected Frames themselves keep all original source columns, +weights, memberships, strata and metadata. The source composition does not +quietly change a Frame's tax leaves or model columns. + +The complete original selected source axis is never reduced in `origins` or +`source_frame`. `origin_document` preserves all original entity origin records; +`donor_basis` contains every original ASEC row's component knownness, provenance, +exclusions and design-weight summaries. `recipient_diagnostics` preserves every +selected ACS row's raw literals and parsed anchor metadata, plus separate +under-15 and unknown-anchor exclusions and the eligible-recipient mask. + +An empty eligible branch has an empty indexed columns table and `None` for +its Frame, and no recipient matrix when no ACS recipient is eligible. Complete +diagnostics remain available. The existing Frame selector and matrix codec +require nonempty inputs; the composition does not manufacture a record or zero +to satisfy them. A later model host must refuse fitting an empty branch or make +an explicitly reviewed completion decision. + +## Exact identity and lifetime + +The complete shared source Frame, its origins and the original preparation's +person origin records must agree in order, source channel and native identity. +Each ASEC qualifier must exactly match the selected ASEC person and native-ID +axes; the pure basis also checks their source ages. ACS anchors must match the +selected original ACS axis. Donor feature and recipient matrix axes are checked +before selection. No sorting, inner join or missing-key fill hides a mismatch. + +The donor household design weights are mapped by exact original membership. +Their person mapping must match the original Frame's resolved DESIGN weights; +importance or clone weights are never substituted. This is source construction, +not engine microsimulation aggregation. + +Each initial qualifier's complete physical description is sealed as soon as it +returns, and checked while subsequent qualifiers do I/O. After pure composition, +all five original source qualifications are repeated and compared to those +seals. Every returned value, including nullable data beneath missing masks, +is checked throughout. The actual preparation is then rechecked after the last +qualifier's I/O. The final checks are pure: retained issuer identity, nested +original source Frames, allocated/clone support, the exact optional geography +configuration payload and all complete derived seals. + +The income-routing qualifier has no standalone physical-seal API, so this +composition uses the existing full table-stamp helper over its complete person +and original-literal tables plus evidence. Existing shared/ACS/interest/dividend +seal APIs are reused directly. No source-reader implementation or new authority +framework is introduced. + +## Verification scope + +The invented fixture creates closed source bytes before issuance, then executes +the actual survey source graph and source composition. It includes a negative +ACS anchor, malformed adult anchor, under-15 blank anchor, a resolved ASEC +ordinary/retirement-interest route, and excluded ASEC overlap/unknown cases. +Controls check ordered axes, weight kind, adjusted bits, all returned seal +surfaces and copied preparation refusal. A separate injected mutation isolates +the final original-source I/O fence using previously checked descriptive values; +it does not claim another complete source replay. + +These checks run without native microdata, QRF, PUF or a country engine. Model +quality, donor conditional support, country financial integration, both-clone +fanout, replay and release acceptance remain later boundaries. + +The initial composition acceptance used donor basis `2c959096eec0441e2acd975e63522aac7f1b7016`. +A subsequent primary-source review found that survivor slots 1/2 do not establish +complete absence of estate/trust overlap: the aggregate can include sources 3/4. +The country host must adopt the separate conservative donor-basis correction +before using this branch for fitting. The initial test's eligible donor reports +no survivor income, so its observed values do not rely on a yes-survivor clearance. diff --git a/docs/us-current-survey-amount-successor.md b/docs/us-current-survey-amount-successor.md new file mode 100644 index 000000000..70d0b28ae --- /dev/null +++ b/docs/us-current-survey-amount-successor.md @@ -0,0 +1,236 @@ +# Current survey amount successor + +## Optional workers compensation, 20 September 2026 + +`run_us_survey_enrichment(..., groups=("unemployment", "health_costs", +"workers_compensation"))` adds the person input `workers_compensation`. The +default remains the first two groups. The new route borrows the genuine +preparation and captures the same authenticated current ASEC member, joining +`WC_VAL`, `WC_YN`, age, and native person/household/line coordinates to the +existing current-money owner. It uses the maintained strict direct amount +mapper on qualified values. A carried column or detached projection cannot +establish source authority. + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +PDF page 51 (printed 6C-30), asks the WC receipt question of people aged 15+. +`WC_YN` is 0 for NIU, 1 for yes, and 2 for no. `WC_VAL` is asked of recipients; +its zero code means none or NIU. The question includes workers compensation +and other job-related injury or illness payments. This route preserves that +published concept, without selecting a subset by `WC_TYPE`. + +In-universe yes/positive and no/zero pairs retain their canonical amounts. +NIU, under-15 observations, yes/zero pairs, contradictions and missing or +unrecognized responses retain their raw values and unresolved canonical +amounts. No child-income zero convention is introduced. Published known +amounts are not claimed to be unallocated responses. + +The existing amount graph extends its source projection and declares WC-specific +original ASEC design-weight donor selection, fit and ACS recipient draw before +the shared clone attachment. +Only eligible ACS originals aged 15+ receive a draw; each original draw is +shared by both support clones. Both ASEC clones retain the same source amount +and reporting knownness. All prior columns, typed weights, identifiers and +memberships remain under the existing parent checks. The current source +predictors and 2025 ASEC / 2024 ACS transport choice are unchanged. + +Bounded invented verification exercises genuine source capture and refusal +after source mutation, then the actual declared QRF fit/apply nodes, required +cache replay and source-keyed clone attachment. The numerical component test +uses explicit invented producers; it does not issue a native enrichment owner. +Combined host acceptance with the other new fragments, actual-source coverage, +held-out fit quality, calibration and release qualification remain separate. + +Implementation plan and source inventory, 12 September 2026. This is a local +successor to the retained survey PUF55 run, not an amendment to the frozen d35 +native pilot. The branch starts at reviewed integration 99543c3. + +## Immediate scope + +| Native ASEC field | Canonical person input | Reporting evidence | +| --- | --- | --- | +| UC_VAL | unemployment_compensation | UC_YN receipt code; amount question universe UC_YN = 1 | +| PHIP_VAL | health_insurance_premiums_without_medicare_part_b | All persons | +| PMED_VAL | other_medical_expenses | All persons | +| POTC_VAL | over_the_counter_health_expenses | All persons | + +These four names are absent from both PUF tax-detail ownership rosters. They +already have qualified current-money domains and direct amount mappings in +`cps_carried_current.py`. The complete legacy carry helper is not reusable: +it also writes existing PUF leaves and a legacy Social Security decomposition. +The modern `survey_social_security` source owner remains the authority for +Social Security; no age-62 zero fallback or prior wages is introduced here. + +The current ASEC source is the authenticated original 2025 person member, +income year and price basis 2024. ACS observations use the already qualified +2024 source predictors, whose earnings refer to the rolling prior 12 months. +That temporal harmonization is a documented model choice, not equivalent +observation windows. No new donor dataset is needed. + +## Amount and reporting status are different + +The [2025 CPS ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +PDF page 50, defines UC_YN codes 0 (NIU), 1 (yes), and 2 (no), with the receipt +question asked of persons aged 15 or older. UC_VAL is asked of recipients; +its zero code conflates none with NIU. The maintained domain metadata pins +this dictionary to SHA256 +`5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The three health amounts have all-persons universes (PDF page 83). + +A small bounded UC_YN projection reads the same exact authenticated original +member. Native person and household/line coordinates must join the original +current-money owner exactly; carried columns and detached JSON do not confer +source authority. The projection retains the original amount, amount validity, +source classification, zero_origin, literal receipt code and reporting status. +Today's upstream preparation requires complete qualified current money before +issuing its handle. Literal missing-amount controls test the join's unknown +semantics; they do not claim that a missing amount passes this unchanged parent +prerequisite. Reporting unknownness can still occur with numerically valid amounts. + +UC attachment and donor policy: + +- A valid positive amount with an in-universe yes code is known receipt. +- A zero amount with an in-universe no code is known nonreceipt. +- Yes with zero is an ambiguous amount, not known nonreceipt. It remains in + the source projection but is excluded from fit and canonical attachment. +- NIU, blank/unrecognized codes, missing amounts, and contradictory amount/ + receipt pairs remain unresolved and are excluded from fit and attachment. +- Out-of-universe age does not infer a zero. ACS persons outside the receipt + question universe also retain unknown UC rather than receiving an adult draw. + +The health fields retain qualified valid all-persons amounts, including genuine +zero-dollar codes. No noanswer cell is converted to zero. Allocation and source +status remain evidence; known source values are not claimed to be unallocated. +PHIP_VAL is deliberately retained as the declared premium source, rather than +silently substituting PHIP_VAL2, whose treatment of inconsistent reported zero +premiums differs. The [Census health insurance user note](https://www.census.gov/programs-surveys/cps/technical-documentation/user-notes/health-insurance-user-notes/User-Notes-Selected-CPS-ASEC.html) +describes that distinction. Person-level conditional fitting preserves modeled +cross-component dependence; it does not certify household or insurance-unit +correlation, which still needs empirical validation. + +## One family, explicit graph operations + +The family has one parameterized source projection and attachment mechanism. +Each selected field declares its source transformation, donor eligibility, +recipient matrix, QRF fit and draw as graph nodes/artifacts. UC has its own +reporting mask. The health group uses an ordered PHIP → PMED → POTC conditional +chain, retaining observed health-component dependence through prior-target +conditioning instead of fitting the three amounts independently. This avoids +filling unknown UC to make a joint matrix finite. Fits use the existing regime-gated QRF implementation and +current source predictor qualification; donor weights remain original design +weights before allocation or cloning. No PUF value becomes an ASEC donor. + +Draws are keyed to the original ACS source person, then deliberately shared by +its original and PUF support clones. This preserves source observations and +paired support interpretation; clone labels are checked during attachment. +Source amount values and reporting status are copied to both ASEC clones. +Every existing parent column, owner, axis, membership, atomic geography, +weight, design anchor and mass ledger must remain identical. + +The parent is the exact `SurveyPuf55Run` checked by +`check_survey_puf55_run`. The original handle is retained and checked before +consumption and after source/store/export I/O. A checked JSON payload or copied +dataclass cannot issue authority. The extension copies kernel/source-codec +registries privately, preserves all prefix declarations/keys/artifacts, and +verifies actual observed populations on cold and required replay. Failed checks +revoke the original handle. The checked-output branch supplies this additive +API; the frozen native packet is left untouched. + +## Verification + +First run cheap invented controls: all UC code/amount/universe combinations, +health zero and unknown behavior, literal bounds and source-coordinate joins, +clone pairing, missing/duplicate source IDs, invalid source/clone channels, +ownership collisions and raw/status/zero-origin mutation. + +Then an actual invented authenticated-source fixture runs the family through +real graph cold and required replay, QRF train/apply, coverage, complete Frame +store/export/readback, parent retention and unchanged prefix seals. Rejection +controls mutate source projection, parent handle/population, declared graph, +artifact bytes, fit/draw lineage and output attachment independently. Acceptance +requires closed receipts and independent review. No native run, engine outcome, +held-out quality, calibration or release certification is claimed by these tests. + +## Remaining input inventory + +The accepted invented PUF baseline has 161 required names, of which 100 are +absent. Its 61 present names are not necessarily complete: some PUF leaves are +populated only on PUF clones. Counts are inventory for that fixture, not a native +coverage measurement. The next stage reduces absent names while reporting +unresolved cells separately. + +| Input family | Existing source producer or route | Integration requirement | +| --- | --- | --- | +| UC and three health amounts | authenticated current-money owner; direct `cps_carried_current` maps | This stage | +| Social Security components | `current_social_security_source`, `survey_social_security` | Preserve source reporting basis and unknown components; never legacy age split | +| Health coverage | Authenticated `current_survey_health_source` and explicit `graph_current_survey_health` fragment | This stage; seven narrow ACS concepts remain unknown where the survey cannot distinguish them | +| Disability, student status, children, veteran status | `eligibility_inputs` | Qualify literals and reporting universes before reuse | +| Parent relationships | `eligibility_inputs._own_children_in_household` resolves PEPAR1/PEPAR2 then drops links | Issue 884: retain parent_1_id/parent_2_id with household/line and clone joins; unresolved and absent differ; consumer issue 9404 is separate | +| Marital roles | `relationship_inputs` | Current source observations and clone preservation | +| Hours, work history, occupation, wage/tips | `hours_worked`, `org_wages`, `weeks_unemployed`, `sipp_tips` | Source/QRF qualification and explicit ownership; prior wages excluded | +| Child support, disability income, workers compensation | respective maintained runtime modules | Current source amount/universe qualification, then existing fit contracts | +| Housing, energy, care, education | `housing`, `energy_subsidy`, `adultcare`, education producers | Household/SPM source and clone-keyed joins; reconcile PUF tuition ownership | +| Financial wealth, vehicles, loans | SCF/SIPP producer modules | Actual donor authentication, existing fit dependencies and held-out quality | +| Retirement distributions/contributions | maintained distribution/contribution modules | Reconcile already PUF-owned taxable IRA/pension and self-employed pension leaves | +| Pregnancy, SSI disability, voluntary filing | maintained producer modules | Resolve source authority and distinct behavioral assumptions | +| Take-up | `take_up_contract` | Unsourced rates remain unresolved; no convenience constant fills | +| Mortgage details, marketplace benchmark ratio, remaining retirement details | Not yet mapped in this bounded inventory | Identify actual producer/owner or make an explicit source/model decision | + +The inventory used tracked files and `rg --no-ignore` within explicit source +directories because ordinary searches can skip the tracked build package. +Existing function names establish implementation leads; they do not prove that +those producers are integrated into the current graph or current native file. + + +## Fixed country composition + +`run_us_survey_enrichment` retains the original checked `SurveyPuf55Run` and +assembles the amount family and independently source-qualified health fragment. +Both attach to `survey_puf55.receiving`; the health CREATE and final attachment +explicitly consume `survey_amounts.attach/attachment`. Health adds nine coverage +inputs and their literal/status/knownness evidence after all four monetary leaves. +The host preserves the complete parent Frame, owners, design anchors and mass ledger, +compares every unchanged prefix version against the original parent manifest, and +requalifies both original-source projections before issuing its output. There is +one private issued-handle registry and one public checked country output, not a +family-level authority or executor. The health fragment's descriptive values do +not grant access to native sources. + +The combined invented fixture includes the exact source ACS HINS/flags and ASEC +NOW/flags before authenticating source owners. It runs the original financial and +PUF stages, both enrichment fragments, a required replay, input-coverage diagnosis +and complete Frame store/readback. No native build, model engine execution, +calibration, or release approval is implied by these checks. + + +## Invented acceptance, 12 September 2026 + +The final production source passed the 271-node combined graph: 245 validated +prefix hits followed by 26 newly executed enrichment nodes, then a required +replay with all 271 hits. Complete parent columns, owners, design anchors, +geography and the mass ledger were preserved. The full resulting Frame was +stored and read back with exact physical comparison. The fixture has 18 people; +required-name coverage improved from 61 to 74 of the 161-input profile, leaving +87 missing names. Presence does not imply complete values or statistical quality. + +All 18 people have the three health-cost values and an ESI observation. UC has +10 known and eight unresolved values. The narrow Medicaid-at-interview input has +eight known and ten unresolved values, preserving the ACS semantic gap. These +are counts from invented records, not estimates or native-data diagnostics. + +Acceptance combines five passing ordered tests from the third full attempt and +one corrected, focused final parent-revocation test. The third attempt retained +its failed result because that last test named a nonexistent generic income +column; it stopped before mutating anything. The final test uses the actual +`employment_income_before_lsr` input and verifies permanent parent and child +revocation after mutation, including refusal after restoring the original value. +It rebuilds a real parent and one child, without repeating the already-passed +required replay or readback. The focused attempt passed with all 857 Python +source pins stable, 624.872 seconds wall time and 885,669,888 bytes peak RSS. + +The separate 32-case cheap suite covers UC reporting classifications, literal +missingness, source/clone identity, ownership collisions and source-origin +serialization when `person_id` names both an index and a column. The serialization +binds the original int64 axis and its name separately from ordered column values. +Independent review covers the frozen UC source, health fragment, combined host +and this serialization correction. Preserved failed attempts remain part of the +local acceptance record; no native execution or release certificate is implied. diff --git a/docs/us-current-survey-household-roles.md b/docs/us-current-survey-household-roles.md new file mode 100644 index 000000000..a5bb2bf10 --- /dev/null +++ b/docs/us-current-survey-household-roles.md @@ -0,0 +1,47 @@ +# Current survey household reference-person roles + +The canonical `is_household_head` leaf means the observed reference person of a surveyed household. `current_survey_household_roles` qualifies that role from the retained ACS/ASEC preparation and original source owners. `graph_current_survey_household_roles` supplies a private source projection and a binding node on the existing population after cloning. This fragment is not yet wired into the country host or validated on native inputs. + +## Source meanings and unknownness + +| Source evidence | Qualified household role | +| --- | --- | +| ASEC `A_EXPRRP` code 1 or 2, every relationship token named, exactly one reference person in the published household | Reference person is true; other members are false | +| ASEC missing/unnamed relationship, no reference person or multiple reference persons | Whole affected household remains unbound, with the owner's reason | +| ACS housing-unit roster with exactly one `RELSHIPP` code 20 and all relationship tokens admitted | Reference person is true; other members are false | +| ACS missing/invalid/mixed housing-unit roster or incorrect reference-person count | Household role remains unbound | +| ACS group-quarters codes 37 or 38 | Published code remains known; housing-unit reference-person role remains unbound | + +The ASEC interpretation reuses `asec_demographic_source` and its pinned dictionary contract: `A_EXPRRP` has a 1–14 printed range, but code 6 has no named meaning. A value inside the range is therefore insufficient. The recorded field locators identify the 2023/2024/2025 dictionary pages 21/22/23 and the all-persons universe. Its owner proves the equivalence of published and prepared household memberships before classifying a role. `P_SEQ`, roster line number, family relationship, age and tax-unit position are not substitutes. + +The ACS interpretation reuses `asec_demographic_source.acs_household_reference_states`, reading original `RELSHIPP` tokens from the retained catalogue's pinned person archive. It reads whole selected household rosters so a selected subset cannot conceal another reference person or an invalid relationship. A group-quarters person is not an observed non-reference member of a housing unit; the named group-quarters code is retained separately from the unresolved canonical leaf. Synthetic household construction and the harmonized `A_EXPRRP` recode cannot supply reference-person evidence. + +These are the requirements already recorded by `graph_composed_asec_binding` for its unbound `is_household_head` column. Their exact immutable declarations live in the import-free `survey_demographic_contract` module; the historical composed stage re-exports the same names and still refuses its own canonical writes. The later role qualifier imports only the shared declarations, not the composed runtime. Its source identity and first/final live checks bind both complete shared tuples, while the old composed stage includes the helper in its existing implementation inventory. This fragment pins that requirement and refuses changes. It does not relabel the composed stage as having supplied this observation. + +Neither arm supplies relationship-allocation evidence through this fragment. The current ASEC readset has no allocation flag for `A_EXPRRP`; ACS `FRELSHIPP` is outside this fragment's readset. The public receipt records allocation provenance as unresolved and makes no respondent-reported or equivalent-evidence claim. + +## Binding and inspection + +The private projection retains exact original person identifiers, source relationship codes, role states, reasons, household verdicts and knownness. The binding fans each original onto its two initial clones using shared provenance columns with exact int64 identity. It writes only `is_household_head`; an existing leaf is an explicit rewrite on a population version below its CREATE node. The expected-output verifier reconstructs the full population through the maintained patch operation and compares all other cells, entities, relationships, weights, geography, provenance, owners and mass history through shared replay equality. + +A known incumbent must have a qualified binding and agree with it. Contradictions and unsupported known incumbents both refuse without changing the original value. This includes an artificial group-quarters head: it cannot become source-confirmed headship. `household_role_reconciliation` reports aggregate disagreement and unsupported-incumbent counts for separate adjudication. A null incumbent is filled only when the source role is qualified. Source-unbound nulls stay null. + +The qualifier accepts only the actual retained preparation type and registry entry. It checks source bytes, live functions, code objects, defaults, closures and contract constants around the first and final preparation/ACS owner callbacks. After the last callback and source identity read it checks retained buffers and independently reconstructs the projection. The graph kernels similarly bind the host callback, qualified inputs and detached output seals. These are bounded in-process integrity checks, not a general execution sandbox. Public artifacts contain only the declared aggregate receipt; source identities and rowwise evidence remain private. + +## Independent-minor audit + +This lane does not create a generic `independent_minor` or emancipation variable. Existing `spm_role_source.py` defines the distinct `is_spm_independent_minor_role`, reconstructed from `SPM_HEAD`, `A_FAMTYP` and `A_FAMREL`, before an age gate. Its documented consumer reconciliation uses SPM adult/child/person counts; it is not household reference-person status, tax dependency, filing status or statutory emancipation. + +`tools/build_us_spm_role_enrichment.py` calls `derive_spm_role_source` for the separate legacy BuildP source-enrichment contract described in [the SPM runbook](us-native-spm-role-source-enrichment.md). Those retained source/release checks do not establish authority for the new combined graph or its ACS arm. A future SPM lane needs an explicit graph owner, current complete-unit source reconstruction, unit-count reconciliation, clone mapping, and verified consumer input semantics. This source-only audit did not import a country engine or inspect installed model definitions, so it makes no new consumer-compatibility claim. No age-only, head-only or tax-unit approximation is introduced. + +## Validation scope + +The proposed tests use invented observations through the real classifier, clone binder, graph compiler/executor, ContentStore and cold/required replay. They cover GQ unknownness, unsupported/conflicting incumbents, exact large integer IDs, complete population preservation, public receipt boundaries and mutations at first/final callbacks. A separate file uses the maintained tiny source fixture and real preparation, ASEC demographic and ACS catalogue owners, including both group-quarters types. + +At this source checkpoint the tests are written but unrun. Source AST, lint and CI inventory checks establish neither execution success nor data certification. The runtime proposal requires separate review of source/resource maps and guard limits before either invented phase. Native qualification, country-host integration, calibration and release verification remain separate gates. + +The finite roster has 71 pure cases, 58 graph cases and 23 actual-owner cases. The import-boundary successor adds exact historical assignment-AST parity, compatibility re-export identity, actual module-body import interception, shared-file source sealing and shared-constant mutations across source and graph owner callbacks. The earlier 113-case phase A attempt stopped during collection because the old eager composed-runtime import reached an unadmitted schema probe; no substantive tests ran. That failed packet and held phase B remain preserved. The successor tests are authored but unrun; no schema/resource permission is added. Additional controls verify non-nullable Boolean storage, invalid incumbent dtypes, and detached coordinate/roster mismatches against actual retained ASEC and ACS owners. An unsupported known Boolean incumbent refuses before the final unresolved-Boolean defense; tests do not fabricate nullable cells inside non-nullable storage to force that later branch. + +The **64 MiB serialized private-artifact cap is an operative capacity limit**, independent of the 2,097,152-original-person ceiling. Per-row JSON size varies with identifiers and values, so the person ceiling is not a supported national capacity claim. The maintained ASEC demographic owner also caps its own full input at 600,000 people. The first applicable source, row or byte limit governs, and none has been validated here against a full national universe. Larger builds need a reviewed storage and resource plan rather than silently increasing limits or assuming the row ceiling can be reached. + +Roster lines supply coordinates only. The current 2025 ASEC dictionary prints `A_LINENO` as 01–16, while ACS `SPORDER` uses 1–20. The fragment's common 20-coordinate ceiling therefore does not establish the suggested failure for a legitimate current ASEC line above 20; this review found no such documented value and did not widen production bounds. The exact ASEC owner and coordinate comparison remain required. [ASEC 2025 dictionary, physical page 22](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf). diff --git a/docs/us-current-survey-housing.md b/docs/us-current-survey-housing.md new file mode 100644 index 000000000..cfa623939 --- /dev/null +++ b/docs/us-current-survey-housing.md @@ -0,0 +1,80 @@ +# Housing participation in the common US survey graph + +The common ACS–ASEC graph connects independent survey housing observations to +two country-model inputs: `receives_housing_assistance` and +`takes_up_housing_assistance_if_eligible`. Both inputs attach to the modeled +assisted family's SPM unit. The country model calculates program awards, +allocates those awards across SPM units and applies the SPM resource cap. + +`graph_us_survey_enrichment.run_us_survey_enrichment` includes this connection +after the current survey amount and health fragments. It uses the retained +source preparation and original common frame; it does not construct a separate +ASEC–PUF population or infer participation from `SPM_CAPHOUSESUB`. + +## Observations and modeled inputs + +The qualifier reads the authenticated original ASEC household and person +members and ACS household and person members already owned by source +preparation. The existing ASEC housing classifier distinguishes known receipt, +known nonreceipt, unknown, not-in-universe and conflicting responses. The +attached household evidence preserves literal fields, status, allocation +quality, knownness, source year and survey year. + +`housing_observed_receipt` remains nullable. `housing_receipt` is the completed +model input, and `housing_receipt__origin` records whether it came from an +observation, a modeled exclusion or imputation. A modeled value never changes +the source observation's knownness. + +The projection records these assumptions: + +| ID | Assumption | +| --- | --- | +| A1 | The household reference person's SPM unit represents the assisted family. | +| A2 | Interview-time participation carries through the preceding income year. | +| A3 | Public-housing and reduced-rent reports represent the modeled HUD assistance family. | +| A4 | ACS group quarters are excluded from this modeled household-assistance family. This is a modeled exclusion, not observed nonreceipt. | +| A5 | Independently observed owner households are excluded from this modeled public/lower-rent family. Positive or conflicting source answers cannot be overridden by this assumption. | + +ASEC `A_EXPRRP` and ACS `RELSHIPP` identify the reference person. Row order and +ASEC `P_SEQ` do not establish that role. A household with an ambiguous reference +role refuses attachment. Explicitly classified group quarters may lack a +reference person only when their completed modeled receipt is false. + +An owner household with positive or conflicting public/lower-rent answers +stops qualification with `OWNER_SOURCE_CONFLICT`. The owner exclusion cannot +silently override those source answers. + +## Household imputation and attachment + +Known ASEC households supply one binary label per household to the existing +regime-gated QRF operator. The donor frame retains complete households and +their original household design weights before geographic allocation and +cloning. Predictors are reference-person age, household size and household +employment and self-employment income. When the parent qualifies demographic +conditioning, the corresponding reference-person predictors also participate. + +The model fills only unresolved household participation cells. One draw per +original household attaches to both of its clones through explicit source and +clone identities. If every household already has a completed input, the graph +omits fitting and application. The connection preserves the receiving frame's +existing columns, entity memberships, weights and mass ledger. + +Only the reference person's SPM unit receives the two true participation +flags. Other SPM units sharing that household receive false. No program-dollar +or SPM-dollar variable is produced by this fragment. + +## Verification and release scope + +The host binds source projections, donor values and weights, recipient matrix, +fitted-model history and attachment outputs. It reconstructs each population +change during readback and checks live implementation and configuration around +owner callbacks. Mutating a retained observation, source mapping or output +invalidates the affected result. + +The focused tests use invented source files and Frames. They exercise source +knownness, household donor grain, reference-person routing, clone attachment, +cap independence and graph replay. These code checks do not establish +imputation quality on national data or a revised 2025 poverty rate. The +projection and attachment therefore carry `release_eligible: false` until the +separate native population, country-model, calibration and release checks are +complete. diff --git a/docs/us-current-survey-property-graph.md b/docs/us-current-survey-property-graph.md new file mode 100644 index 000000000..61c0c4a45 --- /dev/null +++ b/docs/us-current-survey-property-graph.md @@ -0,0 +1,55 @@ +# Current survey property-income graph + +The opt-in `graph_current_survey_property.py` fragment adds 16 operations to the existing US financial graph. It puts qualified ASEC components and modeled ACS components on the complete initial clone pair. The country financial host retains the original source owners, verifies every graph observation and artifact, and issues the result. The fragment creates no separate issuance registry. + +`PropertyIncomeOptions(scales=..., atol=..., rtol=..., n_estimators=...)` requires all four settings. The options object is frozen and has a canonical `to_bytes()` representation. No numeric modeling default is supplied. The first three components are nonnegative; broad property receipts can be signed. A negative or zero anchor can coexist with positive components and an offsetting negative broad-property component. + +## Visible operations + +| Operation | Meaning | +| --- | --- | +| `survey_property.source_projection` | Bind the actual qualified property composition, the existing financial predictor projection/matrix, original host artifacts, and geography validation when present. Emit a projection and exact donor/recipient matrices. | +| `survey_property.asec_eligible_donor` | Filter the original CREATE by the exact eligible ASEC person IDs. Preserve original household DESIGN weights, including their person mapping. | +| `survey_property.asec_donor_columns` | Materialize shared predictor columns, reported total, and four qualified components. | +| `survey_property.acs_eligible_recipient` | Filter original CREATE to ACS persons whose source anchor is known and in the adult reporting universe. | +| `survey_property.acs_recipient_columns` | Materialize shared predictors and adjusted INTP as the reported-total feature. | +| `survey_property.fit.000`–`.003` | Fit the existing DESIGN-weighted chained QRF over the four components. | +| `survey_property.apply.000`–`.003` | Draw once per eligible original ACS person, preserving ordered prior-component conditioning. | +| `survey_property.draws` | Retain raw draws and their complete model/application history. | +| `survey_property.reconcile` | Apply the shared signed-total reconciliation rule with the explicit scales and tolerances. | +| `survey_property.attach` | Join original source identities to both initial clones and retain all existing rows, entities, weights, geography, metadata, and legacy financial leaves. | + +Both model branches leave CREATE before importance allocation. Their FILTER operations return person keep masks, preserving actual raw source columns. Shared predictor materialization, including the existing analytical treatment of ACS earnings NIU, happens through declared column operations. The FILTER does not substitute the separately prepared feature frame for CREATE. + +The projection binds complete descriptive physical seals, including unknown masks and excluded rows, not just the finite fitting matrices. It carries the donor exclusion summary and recipient exclusion counts. No excluded person silently becomes a zero-valued donor or recipient. Empty eligible donor or recipient branches are currently refused before fitting; an empty-branch execution policy is not implemented. + +## Attachment and knownness + +The attachment owns the four `property_*` components, their four `draw_property_*` columns, `property_reported_total`, reconciliation adjustments, nullable active-bound diagnostics, residual and objective, two source-basis discrepancy columns, and two knownness flags. `owned_columns()` is the authoritative roster. + +ASEC persons retain each individually qualified component and reported total, including NaN for an unresolved value. Donor exclusion does not erase otherwise known amounts. The source-basis discrepancies remain available, and the complete source qualification/exclusion descriptions stay bound to the projection. ASEC raw-draw and reconciliation diagnostics are absent because no draw or reconciliation is performed on these observations. + +For example, a qualified survivor estate/trust route can exclude an ASEC person from the joint four-component fit while that person's ordinary interest and dividends remain known. The attachment copies those independent amounts to both initial clones. The separate tax-leaf successor splits each known amount independently; the donor exclusion does not turn it into an unknown tax input. A focused regression exercises this path with authenticated invented survey files and the real attachment and split functions, without fitting a QRF or rerunning the complete financial host. + +Eligible ACS persons receive one ordered joint draw and its reconciled components. Their original `person_id` and pandas row index are checked separately, then the original support ID, native source ID, source label, and exact clone roles establish the two-row fan-out. Excluded ACS persons receive NaN for model components and the attached anchor, with false `property_anchor_known` and `property_components_known`. Their original source anchor status remains in the qualified source projection; under-15 values are not silently converted into a new analytical zero. + +`property_anchor_known` describes the attached anchor's finite value. `property_components_known` describes all four attached components being finite; it is distinct from eligibility for donor fitting. Reconciliation bound flags use nullable booleans so an unperformed reconciliation does not become a false observation. + +## Tax and period boundaries + +The eight existing financial tax leaves remain unchanged. Attachment reads all eight and explicitly follows the legacy financial attachment, checks them against its retained draw history, then adds property columns. The legacy capital-gains chain still conditions on legacy INT and DIV draws. The receipt records `tax_split_rebased: false` and the capital-gains limitation. `property_legacy_interest_draw_minus_reconciled_interest` measures the legacy ACS interest draw minus the reconciled ordinary plus retirement interest; it is a diagnostic, not a new tax assignment. + +Broad property receipts are not identified with rental income or a tax-law leaf. Retirement-account earnings are not retirement withdrawals. Tax-leaf rebasing and a CAP model conditioned on reconciled components require a separate declared successor. + +Periods remain those of the qualified sources: ASEC 2025 interview and calendar-2024 income; ACS 2024 rolling prior twelve months, adjusted by ADJINC to the declared price basis. No equivalence of the reporting windows is asserted. The person-level conditional chain is not a certification of household-level dependence or held-out fit quality. + +## Host integration and verification + +- `current_survey_property_nodes(qualified, clone_frame, host_pins=..., options=...)` declares the fragment. +- `register_property_kernels(...)` registers the country and shared numerical kernels over the retained source owners. +- `reconstruct_property_results(..., artifacts=..., legacy_matrix_producer_key=...)` returns eight expected non-fit results: projection, two filters, two model-column operations, draws, reconciliation, and attachment. The host patches these at their declared population versions and checks actual observations. The separate model-receipt verifier binds the eight fit/apply receipts to actual donor/recipient populations and stored model bytes. +- `verify_materialized_property_income(..., legacy_population=..., population=...)` requalifies actual source owners and compares the full final population against attachment applied to the independently checked legacy population. + +Reconstruction validates the complete ordered raw-draw history, training-model references, recipient index, person IDs and summaries. It invokes the existing deterministic reconciliation rule to derive the expected reconciliation columns; it never fits or draws another model. Store artifact type, producer-key and key validation remains a prerequisite owned by the host. The host must retain its existing final source, implementation, cache, full-population and issuance checks after relevant I/O. + +Changing only scales affects the reconciliation and attachment declarations, leaving source selection and QRF fitting declarations unchanged. The absence of property options in a host call preserves the existing strict legacy path. Graph replay validates cached output observations as well as labels; a cache hit is not source admission or a release verdict. diff --git a/docs/us-fact-to-target.md b/docs/us-fact-to-target.md index 9b5d14ec9..9ba3071f5 100644 --- a/docs/us-fact-to-target.md +++ b/docs/us-fact-to-target.md @@ -111,6 +111,21 @@ exclusion register instead. wiring is safe and normal (the keogh ALD facts rode the feed unmapped for weeks). +### Table 1.1 size-of-AGI shares + +Cross-period national Table 1.1 return counts and AGI from the $100,000 lower +edge can bind through `_rebase_stale_soi_agi_size_distributions`. Each class +uses its own vintage's national denominator and the latest eligible national +control of the same measure. Missing denominators or controls drop the class. +The amount ager completes only the interval after the control year; counts do +not receive a CBO dollar factor. Other tables and cross-period lower-income, +filing-status and state slices remain excluded by this rescue. + +The [native integration note](us-native-soi-agi-targets.md) documents exact +source lineage, the distinction between source and alignment years, preserved +same-period behavior, and the separate support and release qualifications. +No tail support or held-out improvement follows from compiling these rows. + ## 4. Keep the exclusion register honest `US_FISCAL_TARGET_SUPPORT_EXCLUSIONS` is keyed by `source_record_id` and its diff --git a/docs/us-full-asec-immigration-donor.md b/docs/us-full-asec-immigration-donor.md new file mode 100644 index 000000000..34223a1b0 --- /dev/null +++ b/docs/us-full-asec-immigration-donor.md @@ -0,0 +1,94 @@ +# Full original ASEC immigration donor + +`borrow_full_asec_immigration_donor(preparation)` returns a checked retained source +view for a future immigration consumer. It assigns no status, admits no national +stock proxy, and does not attach anything to the graph. The preparation must be +an authentic retained native preparation; a selected Frame or copied receipt is +insufficient. + +The view reuses the native owner's complete current-cohort roster with +`selected=None` and its descendant constructor. Original `HSUP_WGT / 100` +fractions supply household DESIGN weights. Selection fractions, source shares, +clones and the legacy parent's pooled weights do not rescale this donor. +The accompanying household projection retains the exact literal weight, +numerator/denominator, reduced fraction and original household key. No person +weight, MARSUPWT authority, stock normalization or prior-wage input is created. + +The Frame contains only structural IDs, original person keys, the reviewed +23-field immigration evidence, `age`, `is_female`, and household `state_fips`. +Raw literals remain separately available without loss. Sex comes from the +maintained authenticated A_SEX/AXSEX source reader; unresolved sex binding +refuses. State comes from original household GESTFIPS through the maintained +state reader, with a valid state/DC code required. Carried parent predictors do +not fill missing source evidence. The existing observed-age normalization keeps +common age equal to original A_AGE, including top codes. ASEC observation year +is 2025 and income year is 2024. Neither state residence nor legal status is +asserted as an income-year observation. + +## Owning-source token admission + +Every one of the 23 required fields must contain a nonblank numeric literal. +Raw surrounding whitespace and leading zeroes remain in the raw view; the +numeric projection admits the corresponding integer code. Blanks, malformed +text, infinities, unexplained codes and unreviewed response values refuse before +any future rules coercion. No blank becomes zero. Legitimate printed NIU codes +remain distinct integers, including PEAFEVER = -1 and the appropriate zero codes. +This validates source code meanings, not a legal-status classification. + +The [2025 ASEC person dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +provides the following admitted values: + +| Fields | Admitted codes | +| --- | --- | +| PRCITSHP | 1–5; unexplained negative header values refuse | +| PEINUSYR | 0–28; the enumerated 27/28 values override the stale 0:26 header | +| A_AGE | 0–80 and 85; 80 and 85 remain the printed top codes | +| A_MARITL / A_SPOUSE / A_HSCOL | 1–7 / 0–16 / 0–2 | +| A_LFSR | 0, 1, 2, 3, 4, 7; no prior-wage inference | +| MCARE, CAID, IHSFLG, CHAMPVA, MIL, SS_YN, SSI_YN | 0–2, preserving each field's own zero/NIU meaning | +| PEN_SC1/2, RESNSS1/2 | 0–8 | +| PEIO1COW / A_MJOCC | Enumerated 0–8 / 0–11; unexplained header-only values refuse | +| PEAFEVER | -1, 1, 2 | +| SPM_CAPHOUSESUB | Exact whole dollars 0–99,999; trailing decimal zeroes accepted, fractional dollars unresolved | + +PENATVTY uses the 162 printed country/area codes in Appendix J of the +[2025 ASEC technical documentation](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf). +The person dictionary's Appendix H cross-reference does not identify the actual +2025 country appendix. Header-only missing/response codes are not assigned a +nationality. The monetary field agrees with the maintained +`asec_current_money_domains_v1.json` contract (2025 dictionary physical page 61, +printed 6C-40). If actual source tokens require additional meanings, keep the +literal owner intact and review that domain explicitly; do not default them. + +## Retention and validation + +The caller must retain the view and call `validate()` after its last relevant +I/O. The view retains the authentic preparation, native source owner, original +literal qualifier and sex source owner. Independent immutable seals live in an +external issued-state tuple, not solely in an editable validation closure. +Validation checks source owners and implementation bytes, then rechecks pure +source/output/issuance seals. Detached copies, mutated projections, changed +source anchors, altered implementation, or changes during final foreign I/O +refuse. A receipt and a Frame alone confer no authority. The projected Frame's +metadata and mass history remain neutral; the retained receipt explicitly keeps +status assignment and national stock alignment false. + +The tests use tiny invented files through the genuine source issuers: a selected +preparation omits a household/person retained by the full donor, original row +orders differ, and exact HSUP_WGT fractions survive. Pure token controls cover +all 23 fields. Malformed owning evidence reaches the donor after genuine source +preparation and refuses with a field-specific code. Independent demographic +coordinates and ordered household/state alignment have pure mismatch controls. +Genuine preparations also reach donor refusal for unresolved sex and blank, +unnamed or out-of-area state codes on a donor omitted by support selection. +The positive source case asserts the complete ordered person-column surface. +Original-anchor changes, detached copies, output mutation and late I/O mutation +are also tested. Omission/duplicate/weight validation belongs to the retained +preparation, rather than a donor-level refusal claim. No actual population, +country model, fit, calibration, full-51 proof or legal-status distribution was run. + +Later work still requires reviewed #779 rule adoption with per-row observation +years, explicit admission of its mixed-date stock proxies, source-stable draws, +ACS transfer/reconciliation and original allocation weights, clone attachment, +graph replay, and final exported-artifact/model qualification. This source slice +settles none of those downstream decisions. diff --git a/docs/us-full-original-amount-donors.md b/docs/us-full-original-amount-donors.md new file mode 100644 index 000000000..ebed211dd --- /dev/null +++ b/docs/us-full-original-amount-donors.md @@ -0,0 +1,53 @@ +# Optional full-original ASEC amount donors + +`run_us_survey_enrichment(..., full_original_amount_donors=True)` fits the +selected amount groups using complete original current Annual Social and Economic +Supplement (ASEC) support from the Current Population Survey, independently of +which households were selected for the receiving survey spine. The default is +`False`, preserving the existing selected-donor path. The supported groups include +unemployment compensation (UC), health costs and the separately opt-in workers' +compensation (WC) group. The separately opt-in [child-support group](us-native-child-support.md) +requires this option and fits received support only; paid support stays observed-only. +Selecting full-original donors does not itself add an amount group. + +The retained authenticated preparation supplies the full original 2024-income, +2025-survey ASEC roster and current-money source. A separate graph `CREATE` node +exposes this donor population with its original household DESIGN weights. +Group-specific `FILTER` nodes retain rows with jointly known targets and finite +required predictors. The existing fitted artifacts and apply nodes then draw for +the selected original American Community Survey (ACS) recipient axis. The graph +copies each original draw to its two existing clones. It introduces no new +receiving rows, allocation, geography assignment or clone draws. + +Predictors are current age, employment income and self-employment income. When +the financial parent uses demographic conditioning, observed source sex and state +are also required. The source projection keeps missing predictors and targets +unknown; their exclusion from donor fitting does not turn them into zero. UC and +WC retain the existing literal receipt/universe qualification. A reported zero, +unknown response and outside-universe source observation remain distinct. +Recipient age applicability is separate from donor source qualification and is +unchanged. Prior-year income fields are not included in the new donor frame. + +The selected receiving originals, observed source amounts, memberships, IDs and +IMPORTANCE weights are unchanged. Full donor support is not a claim that every +original is eligible for fitting: each amount group still applies its own +knownness filter. It also does not qualify a release or establish scientific +equivalence with a published dataset. Donor exclusions, rare positive coverage +and sensitivity to the expanded fitting support still need actual-data review. + +The new projection borrows the live source owners; it does not issue a replacement +source authority. Descriptive receipts and cached frames cannot recreate that +authority. The host binds the option, full source frame and exact donor/model +identities, reconstructs materialized outputs on replay, and requalifies sources +at its final I/O boundary. The additional source module participates in the +enrichment implementation hash. These implementation changes invalidate affected +enrichment cache identities even with the option off; earlier source/runtime +snapshots remain separate immutable studies. + +Invented tests cover a genuine source preparation whose selected receiving spine +omits a positive WC donor, that donor's inclusion in the maintained numerical +graph, cold execution and required replay, source observation preservation and +identical draws across the two existing clones. The receiving frame in that +numerical test is an explicit component fixture. Full combined financial and +enrichment owner acceptance and an actual-data fit remain separate verification +steps. diff --git a/docs/us-household-role-namespace.md b/docs/us-household-role-namespace.md new file mode 100644 index 000000000..2d9568272 --- /dev/null +++ b/docs/us-household-role-namespace.md @@ -0,0 +1,36 @@ +# Separate carried headship from source-qualified roles + +Native preparation preserves a carried `is_household_head` column as +`legacy_prepared_is_household_head` on its detached source copy. Every value, +missing bit and physical dtype is preserved. The original source Frame, +structural IDs, membership, weights, strata and mass history remain unchanged. +The preparation receipt records the versioned namespace rule and which sources +supplied the carried column. A preexisting reserved diagnostic name refuses. + +This separates two meanings that previously occupied the same column. Legacy +ASEC preparation can derive headship from a within-household sequence. ACS +preparation supplies a dense relationship-derived boolean, including false for +group quarters. Neither establishes a source-qualified housing reference-person +role for every row. In particular, the source role is unresolved for ACS group +quarters, rather than an observed false answer. + +The existing role operation now owns a fresh nullable canonical +`is_household_head`, using the original ASEC `A_EXPRRP` and ACS `RELSHIPP` +observations and their existing universe rules. It still refuses conflicting or +unsupported known canonical incumbents. It does not use the renamed diagnostic +to fill an unknown. `household_role_reconciliation` separately reports legacy +agreement, disagreement and known legacy values without qualified counterparts, +including counts by survey. Relocating a column must not hide these differences. + +The rename happens at the existing authenticated preparation boundary, before +spine assembly and support cloning. The graph's same-dtype rewrite constraint +is unchanged. All preparation and descendant keys change with the source code +and receipt; an old checkpoint cannot confer a new live source qualification. + +Tests preserve both dense and nullable legacy storage, reject collisions and +modified copies, retain disagreements in diagnostics, and pass the maintained +genuine source issuers over invented housing and group-quarters records. Existing +canonical-conflict refusals remain tested separately. These controls do not +certify actual-source counts, a full graph run or a published file. The final +model/export boundary must still handle any out-of-universe nullable role under +an explicit, checked applicability convention. diff --git a/docs/us-immigration-observation-years.md b/docs/us-immigration-observation-years.md new file mode 100644 index 000000000..582c183e6 --- /dev/null +++ b/docs/us-immigration-observation-years.md @@ -0,0 +1,91 @@ +# Immigration rules and source observation years + +The immigration rules and controls are adopted from Microcosm #779 at +`7ee36ac1bea125218912996b1030b77a208db963`. The port replaces the earlier +wage proxy with the source `A_LFSR` labor-force predicate and carries the paired +status codec, evidence-constrained humanitarian reconciliation, and existing +composition bands together. Only the immigration stage is adopted from that +revision; other source stages and transfer families keep their current contracts. + +`observation_year_column` is an optional keyword on assignment, evidence, +candidate-mask, reconciliation, composition-gate, and ACS-transfer APIs. It names +a numeric person column containing a finite integral calendar year on every row. +Missing columns, missing rows after a join, strings, booleans, fractional years, +and non-calendar values refuse. It does not authenticate a source or establish +which year belongs to it. A native caller must bind the values to retained source +evidence: 2025 for current ASEC demographics and 2024 for current ACS. ASEC income +year 2024 remains distinct from its demographic observation year. + +Both ASEC arrival helpers and mixed-source profile arithmetic use the explicit +row year. This matters at cohort boundaries: age 33 and entry midpoint 2007 imply +age at entry 15 in 2025 and 16 in 2024. Assignment, immutable-row reconciliation, +candidate selection/replay, and final composition validation must use the same +convention. The composition gate accepts scalar `time_period` (default 2024), +matching assignment. Original ages and the PEINUSYR 27/28 approximation are +unchanged. Omitting the keyword retains the adopted rules' scalar convention; +finding a column with a suggestive name does not enable it. + +For paired ACS transfer, the column must exist on both frames and must survive +the donor projection. The three evidence predictors remain citizenship, +raw origin code, and approximate entry year, alongside the existing age/sex/state requirements. The +observation year is not added as a QRF predictor and does not change pattern +seeds. Rows without arrival evidence use a constant zero in the feature frame, +so the ASEC/ACS observation-year difference cannot leak through NIU entry values. +The rule profile's cohort arithmetic is unchanged. The execution contract version +4 binds the feature version, NIU value and complete ASEC approximation table; +paired checkpoint patterns independently bind the same contract so even a caller +without an outer bank identity cannot resume stale feature semantics. +The year column name and ordered typed person-ID/year pairs on both +frames enter the immigration checkpoint pattern identity. Existing bank +validation refuses stale patterns and rebuilds them. A caller-written outer bank +identity cannot bypass that check. Non-immigration patterns are unchanged. + +Reconciliation receipts also record ordered resolved-weight and mutable-mask +digests. `us_immigration_humanitarian_transfer_selection_masks` optionally accepts +`reconciliation_receipt` and refuses changed weights, mask or seed. This checks +the numerical selection inputs; the receipt does not authenticate source evidence +or controls. Explicit-row-year receipts use `time_period: null` and retain their +separate observation-year declaration rather than claiming scalar 2024. + +The published arrival documentation has a material inconsistency. The standalone +[2025 ASEC dictionary, printed 6C-6](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=27) +lists PEINUSYR 28 as 2022–2024; the +[2025 technical documentation, printed 6C-6](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf) +lists 2022–2025. Both enumerate codes 1–28, with no 29, despite a stale 0:26 +header. The retained value 2023 is an explicit midpoint/floor-midpoint +approximation compatible with either interval; it does not resolve their endpoint +disagreement or prove an exact entry year or statutory cohort. Raw code 28 and +this unresolved interval must remain visible at any later source-admission step. + +The technical documentation's Appendix J identifies 459 as Zaire (DR Congo), +and 461 as Zimbabwe. The +[2024 ACS Place of Birth list](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/code_lists/ACSPUMS2024CodeLists.xls) +identifies 451 as Sudan only; South Sudan is inseparable in public code 464 with +Tunisia and Western Sahara. The rules do not add that entire residual group. +The complete ACS and ASEC code domains are not interchangeable; matching active +named-country selectors does not create a general lossless crosswalk. + +ACS YOEP is also not universally an exact year: the +[2024 PUMS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.txt) +uses 1938 for 1938 or earlier and 1939 for 1939–1944. The reported year concerns +the most recent move to live in the US, not verified first entry or continuous +residence. Complete YOEP/CIT domain and source qualification remain separate from +this legacy rule helper; its permissive numeric input checks do not admit native +source literals. + +These are rule interfaces, not a native status owner or a release certificate. +The separate full-original-ASEC source view must remain available without +assigning statuses. Later native assignment must reject unauthenticated existing +status pairs, qualify owning tokens before legacy coercion helpers, use original +ASEC household design weights once, and preserve original identities. Later ACS +reconciliation requires an authentic complete-source selection and the receiving +population's allocated pre-clone weights. Candidate exhaustion remains an error. + +The imported control values are unchanged mixed-date imputation assumptions, +including broad unauthorized estimates, cumulative admissions, trailing flows, +TPS registrations, and an explicit-zero withholding assumption. They are not +measured legal-status stocks for a common calendar year. This port does not admit +those assumptions for a native release, update targets or tolerances, enable +native assignment or graph nodes, or qualify an actual export. Source-owner +admission, full-source reconciliation, no-worse comparisons, and final engine +and export checks remain separate work. diff --git a/docs/us-input-coverage-diagnostic.md b/docs/us-input-coverage-diagnostic.md new file mode 100644 index 000000000..b7aa85fa0 --- /dev/null +++ b/docs/us-input-coverage-diagnostic.md @@ -0,0 +1,16 @@ +# US population input-coverage diagnostic + +Select `USInputProfile.NATIVE_NATIONAL_CD` explicitly for the approved native +scope. Its 159 names are the existing 161-name national/CD profile minus +`self_employment_income_last_year` and `previous_year_income_available`. +`employment_income_last_year` was already absent. The report's +`scope_excluded_inputs` records all three canonical prior-year income names, +whether present or absent in the supplied population; it does not remove +physical columns or classify missing data as non-applicable. Other required +names, assigned-block diagnostics and unresolved applicability stay unchanged. +The historical 163-name profile and default 161-name diagnostic retain their +existing behavior and report no native scope exclusions. This explicit name +profile does not resolve the separate native scientific/source gates or alter +historical release requirements. + +[`diagnose_us_input_coverage`](../packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py) takes a `Population`, its `CompiledGraph` and attached `RunManifest`, and returns a typed `PopulationInputCoverage` report. Its default `USInputProfile.NATIONAL_CD` describes 161 national/congressional-district input names; [`required_us_inputs()`](../packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py) keeps the historical 163-name default. The national/CD profile omits only engine `block_geoid` and `tract_geoid`, while separately reporting assigned `census_block_geoid`; neither profile adds `employment_income_last_year`. The diagnostic checks complete declared current ownership and replayed storage, then reports actual entity grains, source-channel/clone groups, null and invalid values, typed weights, and masked-writer versus carried rows. Missing grain and applicability remain unresolved. It verifies consistency of supplied graph objects; source ancestry and source requalification remain the host's responsibility. [Twenty-five frozen controls](../experiments/us-input-coverage25-20260910.json) passed, including actual cold/required replay and late-mutation refusals, with 515 physical source/owned checks and zero model/resource inputs. This is scoped evidence, not maintained-CI validation: four eager calibration imports differ from the frozen run and retain their newer maintained versions. The report establishes neither complete input coverage, statistical signal, applicability, native build acceptance nor release eligibility. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md new file mode 100644 index 000000000..8914d6bea --- /dev/null +++ b/docs/us-launch-review.md @@ -0,0 +1,454 @@ +# US graph build: review guide + +This branch consolidates the US graph build work for review. The intended file +supports national and 119th congressional district analysis from one population. +It is still a development build: complete enrichment, fiscal calibration and +release verification have not passed together. + +## Architecture and review order + +The intended construction order is ACS + CPS ASEC → complete survey multispine +including its initial support/PUF clones → household Census block assignment → +derived larger geographies → donor enrichment → rules evaluation and calibration +→ verified full and pruned exports. Each resulting household retains its assigned +location through the later stages. The PUF +operator acts on the combined survey frame. A separate ASEC–PUF base is not the +intended architecture. + +National and local analysis share one fully constructed, enriched Frame. The +analysis branch begins after harmonization, cloning, atomic geography assignment +and imputation. Calibration changes weights; L0 controls sparsity. An export can +select a geographic scope and remove zero-weight households, but must preserve +every retained record's input values, knownness, entity relationships and assigned +geography. There is no separate national or local enrichment operation. A source +or mapping correction produces a new common parent Frame for both views. + +Release verification must bind each full or pruned export to that parent's +identity and its calibration target/weight specification, compare retained +inputs by stable entity ID, and reject changed inputs, dangling entity links or +regenerated geography. Weight and scope changes are explicit exceptions; record +values are not. This is the required contract, not a claim that all of these +checks have already passed on the release candidate. The separate legacy +ACS-local hours omission in [#765](https://github.com/PolicyEngine/microcosm/issues/765) +illustrates the drift this common construction path removes. + +Pre-calibration source quality and post-calibration export integrity are distinct +checks. Joint calibration may change an origin's contribution or set its weights +to zero. A pruned export must not be required to make every survey origin +independently resemble a national population, or to rerun imputation to satisfy +such a requirement. + +The supplied-parent export comparison now passes 36 focused checks, including +actual checkpoint write/readback, complete household selection, unchanged input +values and missingness, signed and unsigned stable IDs, and full/pruned/local +weight and scope handling. A household with zero original weight can receive +positive calibrated weight without mutating its parent or other entity weights. +The comparison binds the full ordered weight vector and specification, including +rows outside the selected scope. It deliberately reports calibration ancestry +and release eligibility as unverified: the owning build must establish those +and seal its inputs across export I/O. See the +[scoped export acceptance](../experiments/us-common-frame-export36-20260910.json). + +The earlier ten-node survey prefix assigns a block before cloning, and its +thirteen-node age-calibration extension passes cold execution and required +replay on invented originals. A twenty-node extension now carries atomic +geography through current financial imputation and required replay. Native +Delaware and complete national block support pass source normalization; +national/CD acceptance of the enriched population remains pending. The earlier joint tract/district +operator is a separate path. See +[Geography assignment](geography-assignment.md) for the country contracts, +implementation and source boundaries. + +The 10 September sequencing correction now passes 70 inventoried controls on +invented inputs, across seven separately guarded runs, plus one separate default +compatibility case. The nine-node prefix completes its initial clones before +block assignment; the nineteen-node financial extension and twelve-node age +extension retain that geography through cold execution and required replay. +Assignment uses the qualified original-source key and stable clone discriminator, +so changing row order or numeric household coordinates does not redraw the same +clone. Observed state/PUMA constraints, input knownness, entity links, complete +population ownership and weights remain checked. Different clones may draw +different blocks; an occasional shared block is valid. + +The financial donor FILTER requires the exact typed geography-validation artifact +and retains original ASEC DESIGN weights. The shared gate emits this artifact +only when requested, preserving its default API for UK consumers. Two test-only +corrections repair the CREATE source declaration and isolate the obsolete-order +budget counterexample from unrelated serialized storage differences; production +matches the independently reviewed candidate. See the +[scoped postclone acceptance](../experiments/us-postclone-geography-70-controls-20260910.json). + +The earlier ten/twenty-node native runs retain their original order. Native +postclone execution, the new-order PUF host, held-out model quality, national/local +fiscal calibration and full release acceptance remain pending. The age control +uses invented targets and does not establish that complete calibration path. + +| Area | Main source entry points | What to review | +| --- | --- | --- | +| Graph execution and storage | `packages/microcosm-graph/src/microcosm/graph/{decl,executor,manifest,store,codecs,schema}.py` | Typed inputs and outputs, ownership, artifact ancestry, cache identity and nullable round trips | +| Survey source preparation | `packages/microcosm-build/src/microcosm/build/us_runtime/{acs_population_catalogue,asec_population_catalogue,survey_population_preparation}.py` | Source identity, entity links, source universes and current income transformations | +| Combined survey and clone | `us_runtime/{graph_composed_population,graph_survey_population,graph_combined_clone}.py` | ACS and ASEC composition before the clone; native versus detail channels | +| Enrichment | `us_runtime/{full_puf_enrichment,graph_full_puf_enrichment,graph_current_survey_puf_transfer}.py` | Target ordering, conditioning, observed-value preservation and complete replay | +| Conditional models | `packages/microcosm-fit/src/microcosm/fit/{qrf_target,graph_legacy_train,graph_legacy_apply_matrix}.py` | Reusable model artifacts, deterministic draws and target regimes | +| Geography | `atomic_geography.py`; `us_runtime/{atomic_block_support,atomic_block_api_sources,survey_atomic_geography,graph_atomic_survey_population}.py` | Atomic block assignment after complete initial cloning, observed-source constraints, versioned mappings, stable clone identities and the optional typed validation prerequisite | +| Survey mass and calibration | `us_runtime/{survey_origin_budget,graph_survey_budget,graph_survey_calibration}.py`; `packages/microcosm-calibrate/src/microcosm/calibrate/{group_bounds,solve}.py` | Original survey mass, grouped bounds, fixed support and existing ungrouped solver behavior | +| Compatibility | `packages/microcosm-build/src/microcosm/build/{frame_checkpoint,us_runtime/__init__}.py` | Current-main APIs, checkpoint metadata and existing country consumers | + +Paths beginning `us_runtime/` are relative to +`packages/microcosm-build/src/microcosm/build/`. +The branch contains a substantial consolidation; review each area and its tests +before deciding how to land it. Shared model work overlaps with +[#873](https://github.com/PolicyEngine/microcosm/pull/873). Anthony considers his +runtime proposal [#885](https://github.com/PolicyEngine/microcosm/pull/885) +superseded by this direction; it is reference material, not a prerequisite to +retain or merge separately. This does not imply that its GitHub PR is closed or +that this draft has received his approval. + +## Parallel review and dependencies + +Completion of this integration draft is not a prerequisite for all UK work. +Depend on identified shared changes where necessary, with independently reviewed +PRs and compatible interfaces. Keep the US native build, UK candidate repair, +source/target reconciliation and shared runtime work moving in parallel. + +| Reviewer | Review focus in this draft | Work that can continue independently | +| --- | --- | --- | +| Anthony | Shared graph/Frame and calibration compatibility; target hierarchy and metadata flow into diagnostics/dashboard, including #855 | Target/dashboard changes follow #900; UK staging #896 proceeds independently of full UK graph conversion and native US PUF integration | +| Maria | Register each existing UK pipeline stage as a graph node using this branch's shared machinery; verify UK behavior and coverage | #900 lands before graph conversion; UK imputation, support/fit diagnosis and candidate validation continue; graph-node registration stacks on #893, with stage reordering in a later separate PR | +| US integration owner | Resolve overlap, maintain exact compatibility evidence, complete the US population and release checks, and extract shared changes agreed in review | US-specific source preparation, imputation, quality diagnostics and progressive local builds | + +Coordination updated 10 September: Maria is deliberately stacking her UK +graph-node registration on #893. That specific dependent work should pin the +parent revision and coordinate shared-interface changes here. Her source/target +repairs and Anthony's staging work do not have to wait for the whole draft. +PR #900 merged at 14:54 UTC and is incorporated here from main at +`6f7571e1ab8c516154289bd8ebf75cb2646f03b5`. The combination preserves both parent +histories and does not rerun or recertify the frozen US controls. +Anthony's hierarchy/diagnostics [#855](https://github.com/PolicyEngine/microcosm/pull/855) +and its dashboard consumer follow the target reconciliation in +[#900](https://github.com/PolicyEngine/microcosm/pull/900). Registering UK graph +nodes and changing their order are separate review steps. The US integration +work must not build a competing full UK graph; its shared atomic-area adapter +is an input to Maria's owned integration. + +Record shared dependencies as named changes with pinned revisions. Extract a +smaller shared prerequisite where useful, while preserving compatibility with +Maria's intentional stack. Proposed sequencing does not certify or merge any PR. + +The shared post-clone geography interface now has a tested additive validation +artifact contract. The default gate API remains unchanged; UK registration can +adopt the optional typed prerequisite independently of the US stage ordering. +Existing target/source repairs can proceed on their reviewed base. Passing this +draft's invented controls does not certify Maria's UK candidate, and the final +US release need not gate her independent source and calibration work. No +whole-PR merge is implied by this review map. + +## Verification and review scope + +The first native twenty-node survey, block-geography and financial cold pilot +now passes at 1/1000: 1,584 source households and 3,464 people become 3,168 +household records and 6,928 person records after cloning. Ten financial nodes ran +cold over the stored ten-node prefix, which retains the block assigned before +cloning. The composition retains location inheritance and attaches seven financial +fields while preserving the other population fields. It took 110.12 minutes and +11.45 GB peak memory. The final receipt and independent source/resource +postcheck pass. This is the original frozen 491-source implementation; newer +cache and budget changes are separate revisions. Required native cached replay +now also passes: all twenty final nodes and ten prefix nodes were cache hits, +the four non-manifest exports matched the cold bytes, and the portable manifest +retained the same content and stored-artifact identities. Replay took 74.17 +minutes and 11.31 GB peak memory. Independent postchecks verified the recorded +result and exact code/resource identities without reopening native inputs or +exported payloads. PUF augmentation, model quality and calibration remain pending. +See the [scoped replay result](../experiments/us-native-atomic-financial-required-replay-20260910.json) and +[scoped native cold result](../experiments/us-native-atomic-financial-cold-20260910.json). + +The previously published head `a85c9cbb79081a980e7d3afe8916c80153de5bf8` +passed 36 integration controls and 57 enrichment/replay controls on invented +inputs. The 36 cover eight solver, eleven snapshot/storage, nine geography and +eight facade cases. The 57 include cold execution, required replay, reconstruction +through a fresh Frame store, inherited tax-unit values and malformed ancestry +refusals. Source/control, model-declaration and resource pins matched before, +after and in external postchecks; the tax-benefit engine was not executed. + +Those receipts are identified by SHA-256 +`ad8ab6d8761db3d21343bc1fc60023c0dfa305e1367622630a5de6c68892d62a` +and `85a192f8c5c851648875d66b90bea6b8a4ee11e86e67d1875c3ca34a076ee014`. +These checks do not certify subsequent source additions or the entire repository. + +CI exposed test-helper collection failures and a pool/registry circular import. +This revision adds the test helper directory to pytest's path and moves the +registry import after the pool functions it validates. Two fresh-process import +regressions are assigned to the US-engine CI lane; their runtime result remains +pending. No import-order result is inferred from an AST check. + +The corrected checkpoint test selection passed all 58 invented controls against +the current integration checkpoint source, including nullable integer widths, +missing-value masks, deterministic round trips and malformed v4 refusals. It +took 5.36 seconds wall time and 348.5 MB peak RSS. All 45 source/control files, +three provider pins and two code resources matched in the external postcheck. +Receipt: `f8009ec6d07d076eff6a90ec97061d538756293547a21758ab2bc38fa5f7aab8`. + +Fable's earlier replay review closed seven findings. A separate review of the +published consolidation identified the checkpoint test gap and additional +grouped-bound, metadata-store and facade coverage work. The portable grouped-bound, +fixed-support and metadata-store selection now passes all 117 tests with no skips. +The fixes reject nonfinite stored JSON as `StoreCorrupt` and normalize the deprecated +`apg` alias before grouped-mode validation. Source/control, resource and provider +bytes match before, after and in an external postcheck. Receipt: +`3674af9d610417897539da6eae766726d0f582ae6dd3bc2c3e550fefa45bab68`. +The historical platform fixture and complete facade import coverage remain separate. +Environment-specific prepatch byte fixtures do not establish portable CI byte +parity. Existing v1 graph stores remain preserved; v2 execution uses a new store +rather than silently promoting old frames. + +Private runtime records and generated population artifacts stay outside this +source PR. Receipt identifiers are audit references, not reproducibility inputs. + +The composed 156-case test run against source commit `8157300` at documentation +head `78d84f9` passed 152 cases and failed four current-survey cases with +`PRODUCER_CHANGED`. Its execution guard also recorded unexpected test-bytecode +and installed-model cache-directory probes. Source/control, model-source and +resource bytes remained unchanged. This is a failed integration run, not a +replacement for the earlier passing component evidence. Its receipt is +`67ff10fef45c58d2d6ea15c72b4311a4daa901dd2aae263fbb863d38c13bc7c8`. +The successor now passes all 314 selected invented integration and identity +checks with no skips or unexpected refusals. This includes the original 156 +cases, five metadata-copy/mutation regressions and 153 encoder controls; seven +synthetic-issuance controls remain explicitly outside this selection. Shallow +copy, deepcopy and pickle previously added a class cache entry captured by +Python 3.14 annotation closures. An explicit immutable-metadata representation +avoids that mutation without weakening producer checks. Two stale ACS pins were +updated only after verifying AST equality with their accepted source revisions. +All 476 source/control, 5,983 model-source and 12 resource entries matched before, +after and in the external postcheck. The run took 253.5 seconds and 738.7 MB peak +RSS. Receipt: +`d940913f918af4455dab2b9e680d55f9efc7bb922f72ff3d8fe26b0530b070f7`. + +A separate 16-case current seven-node age-development run initially passed 13 +cases and failed three on exact diagnostic reconstruction. The checker omitted +two solver-option fields added by the consolidated solver. Reconstructing the +grouped solver's closing-state selection and empty selection receipt fixes that +mismatch while retaining exact whole-document comparison. All 16 tests now pass, +including actual seven-node execution and late target/successor mutation checks, +in 170.3 seconds at 541.9 MB peak RSS. All 469 source/control, 5,983 model-source +and 12 resource entries matched before/after/current checks. Receipt: +`5849dd42e914e4d8304a5d290ff451da45910e7ac9a61e76fff36468949ffe3a`. +Broader CI's missing helpers and stale schema, seed and runtime classification +expectations remain separate from these selected passing suites. + +## Component evidence and remaining work + +Separate component work has verified the national Census joint-geography source +artifact: 2,462 PUMAs, 87,841 joint cells, all 50 states plus DC and 436 districts +including DC. PUF source ingestion retained 207,692 ordinary records and excluded +four disclosure records. A corrected SCF wage-code interpretation passed source +preparation and model-mechanism checks. Those results belong to their respective +component revisions; they do not establish a calibrated result on this branch. + +The corrected canonical-donor codec v2 was rebuilt from the same accepted typed +PUF source. All 207,692 returns and 59 outputs passed selected-cohort replay and +byte/value serialization checks in 24.37 seconds, using 1.95 GB peak RSS. The 481 +source/control files and four resources matched their before/after/current pins. +Receipt: `6b334ca8df6c6ba721b60f84d28c43151a1e1b54e097616a3c0fa208daeb8cba`. +This run constructs the donor; it does not fit or place it onto survey recipients. + +The identity encoder passed two 40-case preflights and four 40-case +timing runs in both execution orders. The 36 tiny invented cases retain exact +identity bytes and mutation checks. Nine of the 72 case/trial comparisons were +slower, including the small Python-string Frame in both orders; larger numeric +fixtures improved substantially. These timings do not establish genuine-build +speed. It is now integrated and passes the current 314-case selection, with the +newer current-wage projection preserved byte for byte. That projection was absent +from the benchmark's accepted baseline. + +The following work remains open: + +The [US and UK release path](us-uk-release-path.md) puts these implementation +items in the complete population-quality, calibration and consumer-release +sequence. Passing component controls is one step in that sequence. + +1. Extend the accepted native twenty-node atomic and financial cold/replay + pilot to the complete survey scale. Its + invented cold/required replay, complete retained + fields and geography, default three-feature versus explicit five-feature + conditioning, and both final mutation refusals pass on invented inputs. + Two test-code mistakes in the initial four-case run were corrected and + rerun separately; the passing mutation cases were preserved. See the + [composition acceptance](../experiments/us-atomic-financial-composition-acceptance-20260909.json). + Native fit quality and admission of the financial result into subsequent + calibration remain unassessed. +2. Integrate the adopted Social Security conditioning measurement and qualify + recipient return roles before native PUF55 fitting. Thirty-one invented + controls pass the authenticated filer/joint-spouse report-sum proxy, source + preservation, missingness and refusal checks. An incomplete selected report + uses a separately declared eight-predictor route; a known total uses nine. + The helper uses the preparation's retained modeled roles, whose relationship + to current-money tax-unit construction still needs explicit qualification. + See the [measurement decision](puf55-survey-ss-measurement-decision.md) and + [scoped control result](../experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json). + Route-aware graph fitting and attachment remain pending. The genuine + 207,692-return canonical donor has already passed its 55-output projection; + that does not establish recipient matching. Complete survey + beneficiary/component modeling, and supply SCF loan inputs. Separate component runs passed + 128 profile/compatibility controls and 26 current-survey predictor controls; + see [the implementation status](current-survey-puf59-progress.md) and + [donor construction and growth](puf2015-canonical59-and-growth.md). + The genuine 207,692-return donor construction does not establish survey fit + quality, complete enrichment or a releasable population. +3. Verify every applicable model input on each source/clone channel. The proposed + national/CD profile retains 161 required model inputs, with state and county + but without requiring the engine to consume block and tract. Independently, + the build must retain one assigned Census block and derive its larger + geographies. Prior wages remain excluded. A source operator's existence is + not complete cell coverage. +4. Retain the earlier seven-node age-development result as historical evidence; + the twenty-node cold/replay pilot in item 1 is the newer native milestone. + The seven-node run passed at 1/1000: 1,584 source households, + 3,168 records after the support clone, final export/readback and owner/target + verification, 71.1 minutes and 8.26 GB peak RSS. Its source is frozen at + `a9895d8a5`, before the new Social Security, PUF55 and atomic-geography work. + The prior two-hour failure and stale-derived-attachment failure remain failed + historical runs. [The scoped acceptance record](../experiments/us-survey-age-development-20260909.json) + does not certify enrichment, full national/CD calibration or a release. +5. Run real model evaluation, national/CD calibration, holdout checks and complete + export/replay verification, followed by a dashboard bound to that exact file. + Check every pruned analysis file separately. + +These are active workstreams. No release, merge or deployment is implied by this +draft, and the source changes do not relax the outstanding acceptance checks. + +The two-route numerical finalization layer now passes 46 invented controls, +including 110 actual target fits, 220 applications and comparisons against the +maintained whole-cohort finalizer for both routes together and either route +alone. The tests also refuse changed donor row order, weights and late raw or +donor mutations. The v2 numerical receipt seals the complete finalized candidate +before return and refuses a change during receipt encoding. This verifies the +numerical layer; typed model ancestry and complete survey attachment remain +separate graph checks. +The first canonical donor CREATE attempt failed during kernel construction +because its live-code checker followed a circular function closure. The +[failed observation](../experiments/us-puf55-canonical-create-failed-20260910.json) +is preserved. The cycle correction passed its ten new marker controls, but +the [39-case follow-up](../experiments/us-puf55-canonical-cycle39-failed-20260910.json) +still refused before CREATE because the code-state snapshot changed during +initialization. The changing member was the serialized Python code object; +the corrected marker retains the actual code object and its immutable fields. +After correcting three obsolete test accesses to the public PopulationView API, +all 47 canonical CREATE/converter controls pass, including cold execution, +required replay and teardown. The corrected numerical and canonical sources are +adopted locally. These checks use invented donors; the native full PUF host +remains pending. See the +[46-control numerical evidence](../experiments/us-puf55-numerical-output-seal-46-controls-20260910.json) +and [47-control canonical evidence](../experiments/us-puf55-canonical-create-47-controls-20260910.json). + +All 149 selected calibration controls now pass on the combined US/UK solver, +including grouped bounds, fixed zero support, informed gates, budget search, +ordinary best-iterate behavior and strict diagnostic payloads. The run took +4.57 wall seconds at 394 MB peak memory; independent checks confirm the exact +34 source files, copied fixtures and all 149 unique test results. The preceding +run stopped before tests because two still-denied import probes exceeded their +reporting ceilings; the fresh invocation changed only those reporting ceilings +and run identities. No data access was added. This is invented-data solver +verification, separate from native national/CD calibration; see the +[calibration evidence](../experiments/us-calibration-consolidation-149-20260910.json). + +Recipient qualification now covers all 28 distinct original public controls: +27 passed in the original run and the five-case corrected detached-output run +passed, with four overlapping controls. The original run remains failed because +one NaN mutation test did not change its input bits; its correction flips a bit +in the actual float64 storage. This is a compound result across two source +snapshots, not a single passing 28-case run. Exact original-source issuers and +the 22-node cold/required graph are included. See the +[recipient evidence](../experiments/us-puf55-public-recipient-controls-20260910.json). + +The population-only Census API adapter passes thirty-five invented controls. +The native Delaware source control preserves all 15,317 populated blocks and +reconciles them to the independent Census state total of 989,948. It counts +4,881 zero-population blocks, checks CD119/PUMA joins and exactly reads back its +79,769-byte support artifact. The run took 10.94 seconds and 1.735 GB peak RSS. +The first attempt correctly refused an incomplete ZIP-member declaration; +a separate metadata probe established the six-member archive roster, and the +corrected check still decompresses only `NationalCD119.txt`. The +[accepted source control](../experiments/us-atomic-native-de-corrected-1-control-20260909.json) +preserves the original failed evidence. This is not national population or +release acceptance. Delaware-only support must not enter the national survey +graph. + +The subsequent population-only national acquisition and normalization now pass +for all fifty states and DC. The 104 exact sources produce 5,769,942 populated +blocks, independently reconciling every state to a total of 331,449,281 people. +Every block retains its identity and population, with complete CD119/PUMA joins. +The full support artifact is 28,862,508 bytes and passes exact serialization and +readback. Normalization took 82.97 seconds and 6.814 GB peak RSS. Its SHA-256 is +`5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960`; +see the [national source control](../experiments/us-atomic-native-national-1-control-20260909.json). +Postchecks reauthenticated code and compared the 104 native pins from bounded +receipts without reopening source bodies. National support normalization and +household assignment in the small native financial pilot now pass. Evaluating +geographic fit and calibrating the enriched survey remain separate acceptance +steps. + +The financial successor's unchanged positive control now passes with both +original fixture teardowns and final code/resource checks: 647.19 CPU seconds, +651.21 wall seconds and 570.9 MB peak RSS. This separately reviewed invocation +used a 1,200 CPU-second budget; the earlier 600-second attempt remains recorded +as incomplete. No memo or profiler was used. The [positive control evidence](../experiments/us-financial-successor-positive-20260910.json) +checks financial replay and a subsequent weight-only change. The [seven remaining refusal and callback controls](../experiments/us-financial-successor-remaining-controls-20260910.json) +now also pass with their original shared fixture and teardowns: 798.61 CPU seconds, +802.89 wall seconds and 563.6 MB peak RSS. All eight original admission controls +are accepted on the same source snapshot. A subsequent source review found an +additional detached-view callback gap in the budget and weight-only views. +The correction is adopted after four targeted controls passed in 543 wall +seconds, including nine callback branches and independent source/resource checks. +The original eight and corrected four are separate source-version checks; see +the [correction evidence](../experiments/us-budget-detached-view-correction-20260910.json). +The native pilots exclude this overlay, and native calibration remains separate. + +A separate, bounded profile of the unchanged invented cold/required financial +fixture passes its test, original teardown and all final code/resource checks: +303.90 CPU seconds, 308.26 wall seconds and 594.3 MB peak RSS. This measures the +fixture rather than the additional financial and weight-only admission work. +See the [profile evidence](../experiments/us-financial-fixture-profile-20260910.json). +Loaded ACS code verification accounts for 180.20 cumulative seconds inside the +298.15-second profiled fixture. Its nested function checker runs 2,009,108 times +and uses 91.05 self seconds; complete Frame identity uses 7.31 cumulative seconds. +Cumulative timings overlap and profiling adds overhead. The invocation-local +code comparison memo now passes all 18 focused invented controls, including +changed source, aliases, globals, closures, mutable constants and the historical +catalogue caller. Source and resource checks pass before, after and in the +external postcheck. See the [control evidence](../experiments/us-acs-loaded-code-controls-20260910.json). +The paired full-fixture run also passed correctness and final checks, but took +319.79 wall seconds versus 308.26 for the baseline. The memo added eligibility +work and demonstrated no speed improvement, so its implementation was not +adopted. The exact experimental patch and [paired comparison](../experiments/us-acs-code-memo-comparison-20260910.json) +remain recorded. A separate [one-call diagnostic](../experiments/us-acs-producer-profile-20260910.json) +now passes without constructing a fixture: the original producer takes 0.656 +profiled wall seconds. Its 157 compiler calls take 0.238 seconds, including +0.122 seconds inside 103 AST parses; those overlapping times must not be added. +A bounded bytecode compilation cache now passes 25 invented controls and final +code/resource checks, with fresh source reads and every loaded-function check +retained. No native-data speedup is established. The compilation cache has separately passed +the complete invented fixture and is adopted; the native financial cold and +required-replay pilots passed with their original frozen code. The earlier +startup refusal remains preserved and confers no data acceptance. + +Current CI separately reports stale source-attested spec/seed fingerprints. +The worker resource identity test now follows the actual lazy import closure +and includes an invented opened-JSON regression; its new CI result is pending. +Neither the component passes nor this profile establishes a green consolidation. + +## Related reviews + +- [Microcosm → Orrery exporter, #888](https://github.com/PolicyEngine/microcosm/pull/888): + already published separately. It preserves the supplied schema metadata and + exact large integers; graph visibility does not infer domain verdicts. +- [Orrery review index, #6](https://github.com/TheAxiomFoundation/orrery/pull/6) + and [search improvements, #4](https://github.com/TheAxiomFoundation/orrery/pull/4). +- [Dynamics graph integration, #420](https://github.com/PolicyEngine/microcosm-dynamics/pull/420): + owned by the separate retirement-model task and tested on synthetic inputs. +- [Candidate methods and paper updates, site #72](https://github.com/PolicyEngine/microcosm.institute/pull/72): + public component evidence and remaining release checks. + +The accepted local shared viewer remains a separate consumer. There is no package +upgrade or competing generic graph shell in this consolidation. diff --git a/docs/us-native-child-support.md b/docs/us-native-child-support.md new file mode 100644 index 000000000..659c64aad --- /dev/null +++ b/docs/us-native-child-support.md @@ -0,0 +1,49 @@ +# Optional native child-support amounts + +Select `groups=("child_support",)` together with +`full_original_amount_donors=True` in `run_us_survey_enrichment`. The default +amount groups and selected-donor behavior are unchanged. This is a development +transport assumption from ASEC 2025 income-year 2024 observations to ACS 2024; +scientific qualification and release validation remain outstanding. + +The sole fitted target is `child_support_received`, from qualified `CSP_VAL`. +The maintained zero-aware QRF uses all original current ASEC persons with known +receipt or known nonreceipt and finite current predictors, retaining their +original household DESIGN weights. It includes known nonreceipt zeros; ambiguous +recipient zeros and missing or contradictory source routes are not donor labels. +A selected receiving frame can omit a donor without removing it from this fit. +Eligible recipients are original ACS persons aged at least 15. One draw is shared +by both existing clones. Observed ASEC amounts remain source values, including +unknown values; under-15 ACS recipients remain unresolved. + +`child_support_expense` is an observed-only additional output. It retains only +qualified positive `CHSP_VAL` payments under the documented source route. Paid +zero/NIU is not treated as known nonpayment. There is no expense model, no inferred +absence of voluntary payments, and no expense or native-input-profile completion +claim. `CHSP_YN` describes an obligation, not observed payment incidence. + +Both outputs carry separate source-knownness, literal, route, allocation and +topcode diagnostics under `survey_child_*`. The `survey_current_*_origin` labels +distinguish ACS modeled receipts from ASEC source answers and unresolved values. +A modeled value does not alter its source observation mask. The source qualifier +returns the complete literal table separately from its selected observation view. + +If either canonical child-support column already exists, this graph node owns an +explicit rewrite from its newly qualified source/model values. The inherited +values are not accepted as evidence. Supported carried storage is NumPy +`float32` or `float64`; the new values must round-trip without loss, including +unknowns. Other storage refuses. Only these two declared columns can be rewritten; +all unrelated columns, structural IDs, weights, source records and clone mappings +remain unchanged. Earlier source projection, fitting and reporting nodes retain +the prior population version and their complete input checks. A final keep-all +`FILTER` opens a derived version after every prior host branch; only the two +canonical child amounts are written there. The version adds an explicit +conserved-mass ledger record without changing represented mass. Reconstruction +refuses any changed entity axis. The attachment evidence lists the replaced +columns and storage. + +The implementation reuses existing source qualification, full-donor CREATE and +FILTER nodes, fit/apply artifacts, model verification and final attachment. It +adds no new source issuer or release authority. Changing the maintained amount +host's implementation closure changes its cache identity, including for existing +routes; old runtime/source snapshots remain independently pinned. diff --git a/docs/us-native-geography-tenure-bindings.md b/docs/us-native-geography-tenure-bindings.md new file mode 100644 index 000000000..d0b7e748a --- /dev/null +++ b/docs/us-native-geography-tenure-bindings.md @@ -0,0 +1,31 @@ +# Native geography and tenure bindings + +The canonical state fragment converts the existing `assigned_state_fips` into household `state_fips`. It requires exact equality with `survey_observed_state`, which the native source projection qualifies from ACS `ST` and current ASEC `GESTFIPS`. Both inputs must be canonical two-digit strings in the shared state FIPS constants. There is no default-state fallback or float conversion. + +`graph_current_survey_state.state_binding_node` depends on the typed `geography.gate` artifact and the two household inputs. The existing graph assigns blocks after complete initial cloning and derives geographic layers through the shared atomic support. This fragment neither assigns blocks nor parses block identifiers to construct a second mapping. Each clone keeps its existing assignment. The gate orders the graph; its serialized bytes alone grant no source authority. + +The output is `int64` when new. If the receiving population carries `state_fips`, the declaration uses the graph's explicit full-cell rewrite and replaces it from the qualified derived input. Matching, incorrect, or missing incumbent values are not used as evidence. An existing `int64` or nullable `Int64` dtype is retained, as required by the graph rewrite contract; unsupported incumbent storage refuses. Every other column, axis, membership, weight and mass record is unchanged. The kernel does not import the country engine. + +This is an independent fragment. The enrichment host must explicitly register it, retain its genuine source and population ancestry, and reconstruct/requalify it on cold and required replay before it qualifies a complete native run. This slice does not change that host's default graph or claim release readiness. + +## Tenure questions that remain + +The consumer source declares household `tenure_type` as `RENTED`, `OWNED_OUTRIGHT`, `OWNED_WITH_MORTGAGE`, or `NONE`. SPM-unit `spm_unit_tenure_type` separately declares `OWNER_WITH_MORTGAGE`, `OWNER_WITHOUT_MORTGAGE`, and `RENTER`; its documentation identifies Census `SPM_TENMORTSTATUS` codes 1, 2 and 3 respectively. These are different entity grains and code systems. + +Current qualified housing capture (`current_survey_housing.ASEC_HOUSEHOLD_COLUMNS` and `ACS_HOUSEHOLD_COLUMNS`) retains original ASEC `H_TENURE` and ACS `TEN`. ASEC `H_TENURE=1` identifies ownership but does not establish whether a mortgage exists. The legacy `housing_inputs._HOUSEHOLD_TENURE_MAP` maps it to `OWNED_WITH_MORTGAGE`. That historical convention cannot become a claimed mortgage observation. `H_TENURE=2` is cash rent and `3` is no-cash-rent occupancy; the legacy household mapper uses `RENTED` and `NONE`, respectively. NIU or missing values must remain separately unresolved, not silently mapped to `NONE`. + +ACS `TEN=1` identifies ownership with mortgage/loan, `2` ownership free and clear, `3` cash rental, and `4` no-cash-rent occupancy. The maintained `acs_inputs._HOUSEHOLD_TENURE` maps `TEN=4` to household `NONE`, while its separate `_SPM_TENURE` maps that code to SPM `RENTER` through `_map_tenure`'s membership join. `acs_release_predictors.ACS_TEN_TO_SPM_TENMORTSTATUS` also maps `TEN=4` into the SPM non-owner consumed bin 3. The official Census ASEC 2025 dictionary (PDF page 103, printed 6C-66) describes SPM_TenMortStatus=2 as “Owner without Mortgage or rent-free.” Thus the maintained ACS TEN=4→SPM code 3 mapping requires source reconciliation, and ASEC code 2 alone cannot establish household ownership. The consumer enum label is narrower than the source definition. Group-quarters and unknown/inconsistent records need explicit applicability and missingness handling. No tenure output or scientific convention is added by the state fragment. + +A separate source-completion proposal can capture `SPM_TENMORTSTATUS` from the same authenticated original ASEC person member, retain literals and source year, and require within-SPM consistency against the existing genuine SPM membership. `current_survey_spm_source` currently captures membership and role fields, not that tenure field. A later binding must retain the actual SPM source owner and applicability. It must not derive household-owner mortgage detail from unrelated or multiple SPM units, reinterpret all owners as mortgaged, or qualify an inherited canonical tenure column merely because it exists. + +Consumer declarations were read without imports from the pinned local `2.2.1+spmuniverse.20260919` consumer used by the native159 source audit: `variables/input/geography.py`, `variables/household/expense/housing/tenure_type.py`, and `variables/household/income/spm_unit/spm_unit_tenure_type.py`. No country-engine execution or consumer update is part of this work. + +## Enrichment host option + +`run_us_survey_enrichment(..., canonical_state_input=True)` opts into the canonical state binding after the existing terminal enrichment fragment. The default stays false. A visible keep-all FILTER creates `survey_geography.canonical_state_version` before `survey_geography.bind_state_fips`; it preserves every support row and weight and adds one structural ledger record. The separate version is necessary when earlier projections read an inherited `state_fips`: rewriting that column on the same population would redirect those earlier reads to the later writer and create a dependency cycle. + +When child support is selected, state takes the child-support version as its base and its typed final attachment as the ordering dependency. The state binding is then the returned terminal population; both keep-all transitions remain in the ledger. Selecting all people must preserve every entity axis, including otherwise orphan groups; the reconstruction refuses if selection would prune any. + +The host retains the genuine parent population and authenticates its source/geography through the existing cold and required-replay entry/final fences. The version kernel checks the exact projected person IDs, memberships and clone coordinates. The binding kernel checks the exact retained household IDs and qualified derived/observed states, then calls the deterministic state operator. The host independently reconstructs both complete population transitions and compares persisted artifacts, Frames, weights and ledgers on every execution and replay. These checks do not turn a detached receipt into source authority. + +An existing canonical `state_fips` must use `int64` or nullable `Int64` storage; unsupported inherited storage remains an explicit schema prerequisite. The option never silently casts that incumbent. ASEC household mortgage status remains unresolved by H_TENURE alone; this option makes no tenure inference. diff --git a/docs/us-native-health-completion.md b/docs/us-native-health-completion.md new file mode 100644 index 000000000..c392fe511 --- /dev/null +++ b/docs/us-native-health-completion.md @@ -0,0 +1,57 @@ +# Optional native health coverage completion + +`run_us_survey_enrichment(..., health_completion=True)` enables a development +model for seven coverage categories that ACS does not separately observe. The +default is `False`; the existing literal health projection remains unchanged. +This option does not qualify a release or establish complete input coverage. + +The model uses the full authenticated original ASEC donor roster and original +household DESIGN weights, independently of which survey households were selected +for receiving support. Training excludes people with an unknown target or required +predictor. The predictors are observed age, sex and observation-time state. A +missing required ACS predictor causes refusal; there is no filled predictor or +assigned-block-state fallback. + +The seven targets come from the existing coverage field contract: + +| Coverage category | ASEC item | +| --- | --- | +| Marketplace | `NOW_MRK` | +| Non-marketplace direct purchase | `NOW_NONM` | +| Medicaid | `NOW_CAID` | +| Other means-tested coverage | `NOW_OTHMT` | +| TRICARE | `NOW_MIL` | +| CHAMPVA | `NOW_CHAMPVA` | +| VA | `NOW_VACARE` | + +Valid allocated source answers retain their values and allocation status. Blank +or unrecognized codes remain unknown. In particular, broader ACS coverage items +and ASEC `NOW_MCAID` do not become aliases for these narrower categories. + +The graph exposes the full donor projection, target encoding and donor selection, +one original-ACS predictor matrix, seven fitted QRF target artifacts and seven +ordered apply steps. Each ACS original receives one set of draws, then its two +existing clones receive the same values. The attachment preserves every incoming +column, structural ID, membership, weight and source observation. + +The original source-recode nodes retain their nullable values. On the receiving +population, `__known` and `__source_status` continue to describe source knowledge, +while a separate `__imputed` flag identifies modeled cells. A modeled true/false +value does not make an unobserved ACS answer source-known. ASEC unknown responses +and unknown ESI/IHS observations are outside this seven-category completion. + +ASEC coverage is observed at the 2025 interview and ACS at the 2024 interview. +Transporting the relationship between those years is an explicit development +assumption. Scientific qualification is pending: held-out performance, donor +exclusion and allocation patterns, rare coverage categories, broader ACS category +consistency and sensitivity to the interview-year difference still need review. +The chained model does not guarantee that every predicted seven-category tuple +appeared together in a donor record. + +Source authority stays with the live retained preparation and enrichment owner. +The private projection, graph artifacts and descriptive receipts cannot recreate +that authority. The host checks exact fitted-donor and draw/matrix identities, +reconstructs complete outputs on replay and requalifies sources at its final I/O +boundary. The full combined country-owner lifecycle and actual-data fit require +their own separately admitted verification; small invented graph tests do not +substitute for those checks. diff --git a/docs/us-native-immigration-source.md b/docs/us-native-immigration-source.md new file mode 100644 index 000000000..408f88344 --- /dev/null +++ b/docs/us-native-immigration-source.md @@ -0,0 +1,102 @@ +# Native immigration source projection + +`current_survey_immigration_source` provides the original evidence needed for +the humanitarian immigration consumer reviewed at Microcosm PR #779 head +`7ee36ac1bea125218912996b1030b77a208db963`. It assigns no status and runs no +stock draw, fit, calibration or country model. It is not attached to the graph. + +The new sibling source intentionally continues native integration +`cf04225c2c2c5c2e3eafe8ef5f49aa4c5feabd8f`; the freshly fetched main lacked the +native owner APIs. No old PR source tree, source-stage manifest, schema or seed +identity was adopted. The original humanitarian checkout was inspected +read-only. Subfleet's live-session inventory had no matching humanitarian owner; +its recent ledger contained only a completed #779 gate review. No message was +sent and no other checkout changed. + +## API and evidence + +An authorized native build can call: + +```python +from microcosm.build.us_runtime.current_survey_immigration_source import ( + borrow_cloned_immigration_evidence, + qualify_current_survey_immigration, +) + +qualified = qualify_current_survey_immigration(preparation) +columns = borrow_cloned_immigration_evidence(qualified, receiving_frame.person) +``` + +`preparation` must be the live issued `AuthenticatedSurveyPopulationPreparation`. +The qualifier authenticates the retained current ASEC CSV and ACS person ZIP, +exhausts their bounded records, verifies exact digests and counts, and rechecks +the owners after I/O. It returns private tables plus an aggregate receipt: + +- `asec_full_raw`: all income-2024 ASEC persons from the issued coverage roster, + indexed by the original parent `source_person_id`. Exact `PERIDNUM`, `PH_SEQ` + and `A_LINENO` remain strings. The full native-key roster and ages must match + the independent coverage owner. Only identity fields and the exact 23-field + #779 readset are retained. This full table survives smaller support selections + so a later national stock assignment cannot accidentally run on selected rows. +- `asec_selected_raw` and `acs_raw`: original literals for selected people, + indexed by preparation person ID, bound to the separate `origins` table. + ACS retains `CIT`, `POBP`, `YOEP`, `AGEP` and its original keys. These support + #779's source-aware profile; they do not supply an ASEC-UA indicator set or + authorize ACS status assignment. +- `person_evidence`: namespaced `immigration_source_*` literals and observation + year. Blank source tokens remain blank strings; another survey's columns are + missing. Codes, decimals, whitespace and native string IDs are not filled or + coerced. Allocation/response provenance is not inferred. + +The exact ASEC readset replaces the older consumer's `WSAL_VAL` / `SEMP_VAL` +proxies with `A_LFSR`; its order was checked against the exact PR source by AST. +`A_LFSR` is current labor-force evidence at the **2025 interview**, whereas +income refers to **2024**. ACS observation year is **2024**. The qualifier does +not silently apply the old arrival-code midpoints or stock-year controls to +these literals. No prior-income columns or person-weight column are read here. + +The clone function verifies the existing clone-0/1 ancestry, source channel and +native IDs, then returns detached column Series without changing the source or +receiver. It refuses column collisions. It performs no draws. Exact output and +receiver seals are checked after the last source validation. A graph host must +still authenticate its own receiving population; these columns and the JSON +receipt are not receiving or release authority. Keep the qualified object and +call `validate()` after subsequent relevant I/O before consuming its tables. + +The physical index/Series digest helpers are live-bound through the retained +preparation: `survey_population_preparation._modules()` includes +`asec_2024_native_population._modules()`, whose `parent_owner` is +`asec_current_money_source`. `_pure_final()` checks that full live function seal +before trusting frame digests. Regression tests replace either helper to mask +a changed projection and require refusal even with a warm verification epoch. +The source module stays under the scoped provenance scanner; only its fixed +field/owner-map dynamic selectors are reviewed, with no blanket provenance +exemption. + +## Required next integration + +The receipt's `missing_rule_admissions` lists the concrete unfinished steps: + +1. Check 2025 ASEC observation/arrival-code semantics against the exact rules. + Reconcile the rule period and stock year explicitly. +2. Bind the full ASEC native-key roster to its authoritative household design + weights and approved controls. Run the reviewed deterministic status draw + once on original sources, then select and transport its results. The full + literal table alone does not qualify the national stock mass or weight basis. +3. Admit the ACS transfer/assignment method with its own source eligibility and + cohort rules; do not populate absent ASEC fields with defaults. +4. Add the graph attachment under the retained native source and receiving + owners, preserving original status assignments across clones and replay. +5. Qualify country-model/export inputs and run the written-H5 immigration + composition and four HR1 smoke probes. A source projection cannot satisfy + their national stock bands or dollar floors. + +`status_assignment_performed` and `national_stock_alignment_qualified` describe +this projection's actual scope. They are not a permanent veto on a later +separately qualified consumer. The existing immigration implementation and +legacy release gates remain unchanged. + +Tests use real source issuers and original-member capture paths over invented +archives. They exercise full-versus-selected coverage, literal/unknown-state +preservation, exact clone transport, copied-owner rejection and mutations during +final I/O. No real Census/ACS data or country model was used in this slice. diff --git a/docs/us-native-other-disability-source.md b/docs/us-native-other-disability-source.md new file mode 100644 index 000000000..45d182730 --- /dev/null +++ b/docs/us-native-other-disability-source.md @@ -0,0 +1,149 @@ +# Native ASEC other-disability source + +`qualify_current_asec_other_disability(preparation)` reduces the two published +CPS ASEC disability income slots to one non-workers-compensation leaf. It adds +no raw reader of its own: it borrows +[`current_asec_retirement_detail_source`](../packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py), +which already qualifies `DIS_VAL1/2`, `DIS_SC1/2`, `DIS_YN` and their allocation +and topcode literals, and cross-checks the two retained `DIS_VAL` amounts — and +only those — against the authenticated money owner's own bits. The public entry point +calls that owner's qualifier, so the single member capture and its +requalification happen there and are not repeated or replaced here; the pure +`compose_other_disability` and `project_other_disability` entry points read no +member at all. This module issues no source capability, fits nothing, and +attaches no canonical consumer input. Callers retain the genuine owners and +requalify after their relevant I/O. + +## What the leaf is + +The retired eCPS pipeline summed the two annual slots wherever the slot's +source code was not 1, workers' compensation. That arithmetic lives in +`disability_benefits.derive_us_disability_benefits_from_asec`, and this adapter +reuses that function and its archived parameter dictionary rather than +restating them; a drift in either refuses with `ARCHIVED_ROSTER` or +`IMPLEMENTATION_CHANGED`. The exclusion is bound to the printed meaning of code +1, not to the bare integer, so renaming the published label refuses too. + +Social Security disability and workers' compensation are separate leaves and +are not folded in here. The `DIS_CS` and `DIS_HP` work-limitation answers the +detail owner keeps beside these slots describe leaving or limiting work; they +are not this leaf's dollars, and their allocation flags are deliberately +outside this family's provenance. + +## What stays unknown + +The archived arithmetic returned a number for every row whose four literals +it could read, and never an unknown; a row that was never asked the question +read whatever its literals happened to hold, which on a consistent +never-asked row is zero and on a contradictory one need not be. This adapter does not. Each slot is classified into one closed vocabulary +(`SLOT_KINDS`); only the first four resolve, and the rest leave the person +unknown: + +| slot kind | contributes | when | +| --- | --- | --- | +| `reported_source_slot` | the published amount | yes receipt, readable non-workers-compensation code, nonzero amount | +| `excluded_workers_compensation` | 0 | yes receipt, a readable code 1, and a readable amount, whatever that amount was | +| `unused_source_slot` | 0 | yes receipt, code 0, zero literal: no source in this slot | +| `nonreceipt_slot` | 0 | a "no" to `DIS_YN` with code 0 and a zero literal | +| `niu_not_observed_zero` | unknown | `DIS_YN = 0` with code 0 and a zero literal | +| `outside_age_universe` | unknown | under the printed 15+ reporting universe | +| `unresolved_slot_reporting` | unknown | missing, malformed, out-of-range or contradictory literals | + +A receipt answer alone does not settle a slot. A "no" or an NIU answer that +arrives beside a nonzero amount or a populated source code is a contradiction, +not a nonreceipt or an NIU: the owner labels it as such and the slot stays +unresolved. + +A workers' compensation slot whose amount cell is missing or unreadable is +`unresolved_slot_reporting`, not a known zero: a record that cannot be read +here is not evidence that its source code was read correctly. + +A person's leaf is known only when both slots resolve. A "yes" answer with no +populated source in either slot is `affirmed_receipt_without_reported_source` +and stays unknown rather than becoming a zero. A recipient zero under a +reported non-workers-compensation source stays ambiguous, because the printed +entry says `0 = none or niu`. Under-15 rows and NIU rows stay unknown +wherever the archived arithmetic would have read them; whatever it read — zero +on a consistent row, a positive number on a contradictory one, or nothing at +all when a literal will not parse — is recorded in +`other_disability_archived_arithmetic_amount` and +`other_disability_archived_arithmetic_evaluable`, and never adopted. + +That replay runs over the four fields **as the owner parsed them**, so it is +evaluable only where each read inside its own printed domain. The archived +function itself accepted any finite number, so a source code outside the +published table — `11`, say — is a row the retired pipeline would have summed +and this replay reports as unevaluable. It is a replay of the admitted +literals, not of every byte the old pipeline consumed. + +## Provenance is not knownness + +Every published allocation flag for this family (`I_DISYN`, `I_DISSC1`, +`I_DISSC2`, `I_DISVL1`, `I_DISVL2`) is evaluated against its own printed +conditional universe, read from the detail owner's entries rather than +restated. A flag outside its universe, unpopulated, or unreadable is labeled as +exactly that, in that order of precedence: a flag whose universe cannot be +resolved, or that sits outside it, is answered before its own literal is +judged, so an unpopulated or malformed literal is only reported where the flag +was in its universe to begin with. + +Two family-level answers are reported, under names that say which is which. +`other_disability_published_flag_origin` is the routing owner's reading, which +is universe-blind by that owner's charter: a nonzero flag read outside its own +printed universe can still count as a publisher allocation there — unless some +flag's literal is unresolved, which that owner answers first. +`other_disability_allocation_status` is this module's own reading, built from +the per-flag labels above, so a flag outside its universe does not allocate +this family. Its precedence is: an in-universe publisher allocation settles the +family; otherwise an unresolved universe or literal, then an unpopulated flag, +then a clear in-universe zero; and `no_flag_in_its_universe` only where every +printed universe — all of which are `X > 0` here — is false. Neither qualifies or disqualifies a receipt: +`other_disability_allocation_qualifies_receipt` is `False` on every row. +Topcode flags describe only the dollars this leaf admits, and +`topcode_corrected` stays `False`. + +## Clone transport + +`attach_other_disability_columns(values, receiving)` is the thin attachable +interface. It joins on the pre-clone source id, checks that each receiving row's +spine source identity matches the qualified row's `native_person_id`, requires a +uniform clone index set per original, and copies one qualified row to every +clone of that person. No draw is taken and no value is redrawn; the returned +receipt records `redraw_issued: False` and `draws_consumed: 0`. Rows on an arm +this source never observes — ACS today — carry a null amount, `known` false and +the reason `acs_source_unobserved`. They are never zero. + +The attachment returns the canonical leaf under `disability_benefits` and every +other column under the `survey_other_disability_` report prefix. It returns +columns to the host and mutates no receiving cell. + +## Boundaries + +The pure `project_other_disability` function accepts a supplied detail basis so +its semantics can be inspected and tested; neither it nor +`other_disability_values_seal` confers authority. `compose_other_disability` +seals the borrowed detail owner before and after projection and rechecks its +own active implementation dependencies. That fence binds every module-level +constant of this file, detached rather than aliased; every function defined in +it; the file's bytes as read at the end of its own import, so an edit after +that point refuses instead of being reported — the window between the loader's +read and that one is not attested — and an **enumerated** set of borrowed callables — the detail +owner's qualifier, literal projection, slot status, amount pair, member capture +and amount comparison and seal; the routing owner's receipt, literal, +code-frame, capture, digest and allocation helpers; and the archived arithmetic +with its own input guard and parameter dictionary. That list is deliberate, not +a transitive closure: it covers the callables this module's own path depends on +for the literals it reads, and it does not attest anything those callables in +turn call. `qualify_current_asec_other_disability` additionally rechecks the +retained preparation and native issuance. + +The detail owner cross-checks the retained `DIS_VAL1/2` amounts against the +authenticated money owner's bits, but `DIS_YN`, `DIS_SC1/2` and the allocation +and topcode literals reach this leaf from the member capture alone. That is why +the capture path is inside the fence, and it is a boundary a consumer should +know about rather than a property this module can prove on its own. + +ACS completion, SSI eligibility, tax treatment, topcode correction, host +composition, calibration and release remain separate work owned elsewhere. The +tests use invented literals; passing them establishes these contracts, not +actual-data quality, coverage or release readiness. diff --git a/docs/us-native-row-ceilings.md b/docs/us-native-row-ceilings.md new file mode 100644 index 000000000..8d394e012 --- /dev/null +++ b/docs/us-native-row-ceilings.md @@ -0,0 +1,379 @@ +# US native build: the row-count ceilings, and the rule their new values follow + +The [transport lane](us-native-scale-transport.md) lifted the 64 MiB +whole-roster receipt ceilings and, in its §5, recorded a second family it did +not touch: row-count bounds that a full-source native US build meets. This note +is that family's argument. + +It is one rule, not five decisions. Max decided (2026-09-17) that these are +lifted as one change with one argument rather than one build at a time, and a +rule is the only thing that makes the sixth case decide itself. + +## 1. The rule + +> **A bound moves only if a full-source native build meets it. A bound that moves +> becomes four times the measured full-source count of exactly what it counts, +> rounded up to the next whole million.** + +Three clauses, each load-bearing. + +**"only if a full-source build meets it."** Most of the bounds censused in §3 do +not bind, and most of those never will, because what they count is an upstream +file's own size rather than a roster this build chooses. A rule that raised +everything uniformly would weaken real checks for no gain. This one leaves them +alone by construction, and §4 says why each must stay. + +**"of exactly what it counts."** The commonest way to get one of these wrong is +to bound the wrong roster. The 19-node path carries at least five distinct +quantities that differ by more than an order of magnitude — the upstream ACS +file's rows, the selected ACS roster, the selected ASEC roster, the stacked +roster, and the combined clone — and a ceiling raised against the wrong one is +either dead or still binding. §2 measures each, and every moved constant carries +a comment naming the one it counts. + +**"four times … rounded up to the next whole million."** The multiple is the +transport lane's: `MAX_ROSTER_BYTES` is `64 × MAX_SEGMENT_BYTES` = 4 GiB, which +that lane measured at 3.9× a full-source preparation receipt. Adopting the same +headroom keeps one law across both families. The rounding makes the constants +legible and the rule checkable at a glance: divide any moved constant by the +count its comment names and the answer is between 4 and 4.6. + +The rule is not only written here. `test_us_native_row_ceilings.py` holds the +measured counts and asserts that each moved ceiling is exactly what the rule +produces from its count, so a later edit that drifts fails a test rather than a +build. + +### 1a. What the rule is not for + +**Byte transports take the transport lane's argument, not this one.** That +lane's answer to a payload that outgrows its cap is a segmented stream under one +explicit total, not a larger single cap, and it proved the segmented bytes +identical to the unsegmented ones. Raising such a cap here would contradict the +neighbouring argument rather than extend it. §5 names the byte ceilings that +bind and leaves them to it — including one that binds harder than anything this +lane moved. + +## 2. The counts, measured rather than extrapolated + +Every number below comes from the recovered 1/1000 pilot preparation artifact +(`_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json`, +sha256 `34b362d85d2f06acd255390976d76343aff45114958ba382edb10ffa3789a8a0`), and +specifically from its **catalogues** rather than its rosters. + +That distinction is the point. A roster at 1/1000 has to be scaled to say +anything about full source, and scaling a stratified sample introduces error. +The catalogues do not: they are counts of the whole upstream ACS and ASEC files, +and they are identical in a 1/1000 artifact and a full-source one. What makes +them answer the full-source question is this identity, which +`experiments/native-row-ceilings/roster_census.py` asserts before reporting +anything: + +``` +ACS occupied_hu 1,348,408 + + institutional_gq 84,422 + + noninstitutional_gq 98,784 + + ASEC households 55,762 + = 1,587,376 == selection.supplied_households +``` + +"The whole catalogue" and "what a full-source selection supplies" are therefore +the same set, and the catalogue's counts *are* the full-source counts: + +| quantity | full source | at 1/10 | +|---|---:|---:| +| ACS households (selectable; excludes 100,355 vacancies) | 1,531,614 | 153,161 | +| ACS persons | 3,422,888 | 342,288 | +| ASEC households | 55,762 | 5,576 | +| ASEC persons | 142,125 | 14,212 | +| **stacked households** | **1,587,376** | 158,737 | +| **stacked persons** | **3,565,013** | 356,501 | +| combined-clone households | 3,174,752 | 317,475 | +| combined-clone persons | 7,130,026 | 713,002 | + +The transport lane's §5 quoted ~3,471,000 stacked and ~6,943,000 cloned persons. +Those were the 1/1000 artifact's 2.186869 persons per household extrapolated, +and the catalogue figures supersede them — about 2.7% higher, which changes no +verdict but is the number each new constant is derived from. + +One further split matters, because two of the moved bounds are met by one +channel rather than the total: **`_normalized_source_copy` normalizes the ACS +and ASEC native frames separately**, and the ACS channel carries 96.0% of the +persons. A bound met per channel is met by 3,422,888, not by 3,565,013. + +## 3. The moved ceilings + +| constant | was | now | counts | full source | × | +|---|---:|---:|---|---:|---:| +| `acs_pums.MAX_EXACT_HOUSEHOLDS` | 1,000,000 | **7,000,000** | exact ACS household keys in one snapshot | 1,531,614 | 4.57 | +| `acs_pums.MAX_EXACT_PERSON_ROWS` | 1,000,000 | **14,000,000** | `NP` summed over the selected ACS households | 3,422,888 | 4.09 | +| `acs_person_coverage_columns.MAX_SELECTED_ROWS` | 1,000,000 | **14,000,000** | requested ACS person keys | 3,422,888 | 4.09 | +| `survey_observed_age.MAX_ROWS` | 2,000,000 | **14,000,000** | one channel's person rows | 3,422,888 | 4.09 | +| `survey_origin_budget.MAX_GROUPS` | 1,000,000 | **7,000,000** | allocation instructions = selected households | 1,587,376 | 4.41 | +| `current_survey_geography.MAX_HOUSEHOLDS` | 524,288 | **7,000,000** | selected households | 1,587,376 | 4.41 | +| `asec_demographic_source._MAX_PERSONS` | 600,000 | **14,000,000** | retained ACS persons (the larger of its two rosters) | 3,422,888 | 4.09 | + +Each refusal keeps its code, its exception type and its expression; only the +number moves. + +**One consequence, named so a reader does not have to find it.** +`MAX_SELECTED_ROWS` at 14,000,000 now sits *above* `MAX_ROWS` at 6,000,000 in the +same module. That is not an inconsistency: `MAX_SELECTED_ROWS` bounds +*caller-supplied* person keys, which can exceed what an archive holds, while +`MAX_ROWS` bounds the archive's own rows. But it does mean the effective ceiling +on that path is 6,000,000, by containment. Setting `MAX_SELECTED_ROWS = MAX_ROWS` +would be tighter *and* provable rather than chosen — and it would make the rule +two rules, which is why this note keeps one. + +| constant | refusal | raises | +|---|---|---| +| `MAX_EXACT_HOUSEHOLDS` | `"ACS exact selection requires bounded unique raw native keys."` | `ValueError` | +| `MAX_EXACT_PERSON_ROWS` | `"ACS selected complete roster exceeds native person budget."` | `ValueError` | +| `MAX_SELECTED_ROWS` | `"ACS coverage selected person count is outside the bound"` | `ValueError` | +| `survey_observed_age.MAX_ROWS` | `"SURVEY_OBSERVED_AGE_ROW_BOUND"` | `ValueError` | +| `MAX_GROUPS` | `"GROUP_COUNT_BOUND"` | `SurveyOriginBudgetError` | +| `current_survey_geography.MAX_HOUSEHOLDS` | `"CURRENT_SURVEY_GEOGRAPHY_HOUSEHOLD_COUNT"`, `…_PROJECTION_STORAGE` | `ValueError` | +| `asec_demographic_source._MAX_PERSONS` | `"ACS_ROWS"`, `"MEMBERSHIP_ROWS"`, `"CLASSIFY_ROWS"`, `"DEMOGRAPHIC_ROWS"` | `ValueError` | + +### 3a. `survey_origin_budget.MAX_GROUPS`, which the transport lane left open + +Its §5 recorded this one as "not established — the group count was not derived +here". It is established now, and it binds. `_initial` builds its groups with + +```python +instructions = graph.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] +) +_require(0 < len(instructions) <= MAX_GROUPS, "GROUP_COUNT_BOUND") +``` + +and `allocation_instructions` opens with +`_require(len(household_origins) == len(plan.selected), "ORIGIN_COUNT")`. One +instruction per selected household, so the group count *is* the selected +household count: 1,587,376 at full source against a 1,000,000 bound. + +Where it runs, stated precisely rather than implied. **No node in the +nineteen-node graph executes `freeze_survey_origin_budget`**, which is where +`GROUP_COUNT_BOUND` is checked — the recovered run's `graph.json` lists all +nineteen and none is a budget node, and the completion host has none either. +`graph_atomic_survey_financial` does call into the module, but only for +`_config_payload` and the `_live()` producer seal, neither of which reaches +`_initial`. `survey_age_calibration` and `graph_survey_budget` are what execute +it, over the same full-source selection, so the count and the verdict stand. + +## 4. Why a bound stays a bound, and why some must not move at all + +**Deleting a bound is not the cheap version of raising it.** Each of these +stands between a malformed or runaway input and an unbounded allocation, and the +refusal is what makes the failure legible and early rather than an OOM kill deep +in a multi-hour build with no statement of which roster was wrong. A ceiling at +4× the real thing still refuses a 10× runaway, which is the shape a real mistake +takes: a fraction argument off by a decimal place, a selection that forgot to +filter, a key list built from the wrong catalogue. + +Three classes must not move at all, and this note names them so a later reader +does not mistake a low number for an oversight. + +**An upstream file's real size.** `acs_person_coverage_columns.MAX_ROWS` and +`acs_native_coverage_binding.MAX_SOURCE_ROWS`, both 6,000,000, bound how many +records the reader will stream out of the ACS archive. The 2024 ACS person file +holds 3,422,888 of them. That bound is a structural assertion about a file this +build does not produce and cannot check any other way, and the roster it feeds +is separately bounded by `MAX_SELECTED_ROWS`. Raising it would trade a real +check for nothing: a selection cannot exceed the source it is drawn from. + +**A fixed-width encoding.** The `PAYLOAD_MAX_BYTES = len(MAGIC) + 4 + +HEADER_MAX_BYTES + ROW_BYTES * MAX_HOUSEHOLDS + 32` family in +`asec_housing_status`, `asec_housing_universe` and `asec_income_observations` +computes a byte budget *from* a row ceiling, so moving the row ceiling moves the +wire format a reader depends on. `survey_observed_age.MAX_EXACT_FLOAT64_INTEGER` += 2**53 is the same kind of thing one level down: the module's own docstring +calls it "a representation limit, not a scientifically permitted maximum age", +and it bounds an age's value, not how many rows carry one. + +**A quantity no selection fraction grows.** The ASEC codec's +`asec_current_money.MAX_PERSONS` (1,000,000) and `MAX_HOUSEHOLDS` (400,000) +count the ASEC source scope's own `person_rows` and `household_rows`. The +catalogue measures those at 142,125 persons in 55,762 households, and a +full-source survey selection does not change them. **The transport lane's §5 +listed `asec_current_money.MAX_PERSONS` among the bounds a full-source build +meets; it does not, and this note corrects that.** + +## 5. What still binds, and whose argument it is + +**Nineteen bounds bind at full source at the base branch's values.** §3 moved +seven of them. The other twelve belong to the transport lane's argument — a +segmented stream under one explicit total — not to this one. + +(The census in §6 reports fourteen rather than nineteen because its agents read a +moving tree: five of the seven were lifted while it ran, and they correctly +report the post-lift values at which those five no longer bind. Each of those +rows records both numbers. The two the census still shows as binding — +`current_survey_geography.MAX_HOUSEHOLDS` and +`asec_demographic_source._MAX_PERSONS` — were moved afterwards, on its evidence.) + +### 5a. The test the rule applies + +Three of the remaining bounds *look* like row ceilings, and one of them was +moved here after that test was applied to it. The test is a single question, +answered from code rather than from the constant's name: + +> **Does the binding site materialise a per-row payload, or does it stream?** + +A bound in front of a materialised payload cannot be usefully raised on its own, +because the payload's own byte cap refuses first and at a smaller number. A +bound in front of a streaming digest has nothing behind it, and the rule applies. + +| bound | value | what is behind the binding site | verdict | +|---|---:|---|---| +| `current_survey_geography.MAX_HOUSEHOLDS` | 524,288 | `_projection_digest` streams one `_encode(values, maximum=4096)` at a time into a `hashlib.sha256`; the module's only bytes are a 64 KiB summary receipt | **streams — moved in §3** | +| `current_survey_household_roles.MAX_PERSONS` | 2,097,152 | `_projection_bytes` is `table.reset_index().to_json(orient="table").encode()` — the whole per-person table in one string — under `graph_current_survey_household_roles.MAX_ARTIFACT_BYTES` = 64 MiB | materialises — transport's | +| `current_child_property_income_source.MAX_ROWS` | 600,000 | `_json` is one `json.dumps(value)` under `MAX_PROJECTION_BYTES` = 64 MiB | materialises — transport's | + +The middle row is the clearest case for why the test matters, and it is measured +rather than reasoned: through the module's own encoder, a person row costs +**320.08 bytes**, so that 64 MiB cap admits **209,661 persons — 5.88% of source** +against a row bound that admits 58.8%. The byte cap refuses **ten times earlier**, +so raising `MAX_PERSONS` alone would move nothing at all. +`experiments/native-row-ceilings/roles_projection_size.py` measures it; the table +it encodes is invented but faithfully shaped, and the per-row cost is a +difference between two row counts so the fixed schema preamble cancels. + +A fourth bound needed the same test plus one more question, and passed both. +`asec_demographic_source._MAX_PERSONS` (600,000) is enforced at five sites that +count **two different quantities**: a fixed ASEC three-cohort roster of 432,523 +rows, and the retained ACS person roster, which a full-source build grows to +3,422,888. One constant over two rosters takes the larger, so the rule produces +14,000,000 from the ACS arm. Three things had to hold before it could move, and +each was read rather than assumed: + +- **It is not a fixed-width encoding.** `rows` lives as a JSON integer in the + header; the module's only `struct.pack` is `" **`survey_origin_budget.MAX_PAYLOAD_BYTES` (64 MiB) admits 87,838 households — +> 5.53% of source.** That is below one tenth, and below the 96,860-household +> preparation-receipt ceiling the transport lane lifted. A full-source payload is +> 1.13 GiB, 18.07× the cap. + +Measured, not estimated: `experiments/native-row-ceilings/origin_budget_size.py` +builds one faithful origin record through the module's own `_reference` and +`_json` at full-source household-id widths and gets 764 bytes per group, +including the two `household_ids` and two `group_indices` entries the header +carries for each group's two clone roles. + +**It also cannot be raised on its own, and that is a structural fact rather +than a preference.** `survey_origin_budget._json` is +`graph._bounded_json(value, MAX_PAYLOAD_BYTES)`, and `_bounded_json` opens with + +```python +_require(type(limit) is int and 0 < limit <= 64 * 1024**2, "TRANSPORT_LIMIT") +``` + +so the shared encoder refuses any cap above 64 MiB before it encodes a byte. A +larger number in this module would not loosen the bound; it would refuse the +module. The 64 MiB is the encoder's, and moving it is the transport change. + +The consequence for this lane is stated rather than glossed: **lifting +`MAX_GROUPS` is necessary and not sufficient.** `GROUP_COUNT_BOUND` is checked +in `_initial` before the payload is built, so at full source it fired first; with +it lifted, the refusal moves to `TRANSPORT_LIMIT` in the same module. That is the +correct outcome of one argument applied to one class of bound, and the remaining +ceiling is a real piece of work for whoever carries the segmented transport into +this module. + +## 6. The census + +`experiments/native-row-ceilings/census.json` is the census. Every `MAX_*` row, +household, person, group or byte bound in +`packages/microcosm-build/src/microcosm/build/us_runtime/` reachable from the +19-node financial graph and the 45-node pilot graph was read at this head — **146 +bounds across five module families** — each with its enforcement site, its +refusal code and exception type, what it protects, the count it meets at 1/10 +and at full source, and whether it binds. Every verdict that claimed a +full-source build meets the bound, or that the bound guards an encoding width or +an upstream file's real size, then went through an adversarial pass that read the +code again and tried to refute it — **42 verdicts**, and no headline binding call +was overturned. + +The result, read as the agents read it: **14 bounds bind at full source, 6 of +them at 1/10.** Five of the seven §3 moved are absent from that fourteen because +they were lifted mid-census and the agents correctly reported the post-lift +values; each of those rows carries both numbers. **At the base branch's values +the figure is nineteen**, of which §3 moved seven, leaving twelve for the +transport argument. §5a and §5b say which and why. + +## 7. Pins + +These modules are hashed into stage implementation identities, so the pins were +re-derived through their generators rather than assumed. + +| pin | generator | result | +|---|---|---| +| `graph_implementation_inventory.json` → `contracts` (all 124) | `graph_implementation._dependency_contract(payload, name, _covered_imports(name, inventory))` | unchanged by these edits — a constant's value is not an import, an unbound use or a resource access | +| `acs_native_coverage_binding._ACCEPTED["acs_pums.py"]` | `sha256(.read_bytes())`, the same call the check makes | **moved**, `6ecf79f0…` → `e79a2a4e…` | +| the other three `_ACCEPTED` entries | same | unchanged; this branch does not edit those modules | +| all ten `implementation_manifest(stage)` | `graph_implementation.implementation_manifest` | built | + +`experiments/native-row-ceilings/repin.py` reports all of this in one run and +`regenerate_accepted_pin.py` produced the moved value. Neither pin was +hand-edited, and both generators are idempotent. + +What moves and is not a committed pin: each US stage's `implementation_hash` is +over its whole module roster, so editing any inventoried module moves it and +with it every node key and store address. #935 recorded that this is true of any +change to these files including a comment. diff --git a/docs/us-native-scale-transport.md b/docs/us-native-scale-transport.md new file mode 100644 index 000000000..cb7ef210e --- /dev/null +++ b/docs/us-native-scale-transport.md @@ -0,0 +1,480 @@ +# US native build: receipt transport and per-node population retention + +Design authority for the two hard limits the 2026-09-16 cost attribution found +in the native US graph build +(`experiments/us-native-executor-cost-attribution-20260916.md` on the +`microcosm-exec-profile` worktree, §5 and §9): a whole-roster receipt that +refuses above 6.1% of the US source, and a per-node population retention that +needs about 300 GiB at full source. + +Every line number below is at `5ff889814`, the branch point, unless it names a +later head. Line numbers in the cost-attribution report are its own tree's and +are offset from these. + +## 1. What refuses today, measured rather than inferred + +The preparation receipt is **one JSON byte string**, built by +`prepare_authenticated_survey_population` and handed to `_encode` in a single +call (`survey_population_preparation.py:1679`) under +`MAX_PAYLOAD_BYTES = 64 * 1024**2` (`:50`), which refuses `PAYLOAD_LIMIT` +before each chunk append (`:104-109`). Twelve top-level keys; ten of them are +fixed-size. The two that are not are `"selection": _plan_document(plan)` +(`:1698`) and `"origins": origins` (`:1699`), which carry **one record per +source household and per person**: + +| block | built at | per-row shape | +|---|---|---| +| `selection.selected`, `.excluded` | `_plan_document` `:240-253` | one `_value`-flattened plan row per supplied household | +| `origins.households` | `_origins` `:946` | one 7-field dict per stacked household | +| `origins.entities` | `_origins` `:870` | one `[new_id, channel, source_id]` row per row of **each** of the six US entities | +| `origins.persons.rows` | `_origins` `:965` | one 10-element row per stacked person | + +A second, independent budget guards the same lists: `_bounded_append` +(`:816-819`) `_encode`s each row on its own and refuses `ORIGIN_LIMIT` against +the same 64 MiB constant. `_plan_document` opens its own budget (`:247`) and +`_origins` opens another (`:844`), so there are three 64 MiB ceilings on one +document, plus the final `_encode`. + +**The size law, read off the pilot's own artifact rather than modelled.** The +1/1000 preparation receipt survives in `_recovered/pilot-runs/` — 1,136,063 B at +`native19-required-20260912/run/financial-artifacts/preparation.json`, byte +identical to `native19-v2`'s copy and to that run's +`graph-store/objects/04/0481…/payload.bin`, sha256 `34b362d85d2f06ac…`. It +records `supplied_households = 1587376` with 1,584 selected. Re-encoding it +through the repository's own canonical separators reproduces 1,136,063 exactly. +Decomposed: `origins` 718,800 B (households 269,000; entities 224,755; persons +210,924; storage_transitions 13,268; age normalization 762), `selection` +393,395 B (selected 385,122; excluded 7,250; cells 923), `producer` 21,135 B, +everything else 2,733 B. The blocks that grow with the draw are +1,096,821 B over 1,584 selected households: + +> **bytes(f) = 39,242 + 692.4375 × 1,587,376 × f** + +At f = 1 that is **1,099,197,911 B = 1.024 GiB, 16.4× the limit**. Encoding it +costs about 53 CPU-s at full source (the report's §5 measured 33.3 µs per source +household), so the ceiling is a transport-shape decision and not a performance +trade-off. + +**Five ceilings, in the order they bind.** Each is computed from the same +committed artifact, or from the allocation payload's own measured 501,597 B at +1,584 households: + +| # | ceiling | refusal | households | share of source | +|---|---|---|---|---| +| 1 | preparation `_encode` | `PAYLOAD_LIMIT` | 96,860 | **6.10%** | +| 2 | `_origins` row budget | `ORIGIN_LIMIT` | 150,916 | **9.51%** | +| 3 | allocation payload stream | `ALLOCATION_LIMIT` | 211,924 | 13.35% | +| 4 | `_plan_document` row budget | `ORIGIN_LIMIT` | 270,284 | 17.03% | +| 5 | allocation structural pre-check `ALLOCATION_MAX_BYTES // 128` | `ALLOCATION_LIMIT` | 524,288 | 33.03% | + +A 1/10 build is 158,737 source households, so **#1 and #2 both bind and #3 +does not**; a full build needs all five, because 524,288 is below the source's +own 1,587,376. #5 bounds the intermediate maps `allocation_instructions` builds +rather than the payload — its implied 127-byte row budget is below the payload's +220-byte structural overhead alone, so it can never be the binding *payload* +constraint — but it does refuse a full-source selection, so its ceiling moves +with the others and keeps its code. + +**The store is not the constraint.** `ContentStore.put_bytes` +(`store.py:984`) and `put_frame` (`:919`) carry no byte cap, and `put_frame` +already writes one `.npy` per entity and column under +`objects//` with a `meta.json` payload table recording sha256 and +size per file (`store.py:140-153`), re-verified on every read by +`_verified_meta` (`:227`). Every 64 MiB ceiling in this pipeline is owned by +the US runtime, not by the graph. + +## 2. The new transport + +### 2a. What must not move + +`preparation_sha256` is `_sha(payload)` (`graph_survey_population.py:493`, +`:513`), it is folded through `_digest` into the CREATE and ALLOCATION node +params (`:1226`, `:1235`, `:1254`), and `params` reaches `node_key` through +`normative(node)` (`keys.py:288`). So the document's bytes reach every node key +and every store address. That is why the transport changes and the **canonical +byte stream does not**. + +The stream is already defined independently of its materialisation. +`_chunks(value)` (`:97-102`) yields the canonical JSON of the document; +`_encode` (`:104-110`) concatenates those chunks under a bound, and `_digest` +(`:113-116`) hashes the same chunks under no bound. sha256 is a streaming hash +over exactly those bytes in exactly that order, so + +> `_sha(_encode(document)) == _digest(document)` + +for every document `_encode` accepts, and `_digest` keeps returning that value +for documents `_encode` refuses. The new transport digests the stream and +materialises it in bounded segments; it does not re-shape the document. + +### 2b. The segmented, content-addressed receipt + +`_segment_stream(document)` replaces the single `_encode` call for the two +whole-roster receipts. It walks `_chunks(document)` once and: + +* accumulates into a segment buffer, and whenever appending the next chunk + would exceed `MAX_SEGMENT_BYTES` (= the old `MAX_PAYLOAD_BYTES`, 64 MiB) it + closes the segment — so **no accumulation in the encoder is ever larger than + the value the old ceiling allowed**, and `PAYLOAD_LIMIT` still guards each + one, with the same code, at the same number; +* records each closed segment's sha256 and size, and updates one running sha256 + over the whole stream; +* refuses `ROSTER_LIMIT` above `MAX_ROSTER_BYTES`, an explicit resource ceiling + that is *not* a transport shape: it is the total the process is willing to + spend, set far above full source (see §2d), and it is the only new refusal. + +The result is a `_RosterReceipt` handle carrying `digest` (identical to +`_digest(document)`), `size`, and the segment table. The preparation path also +writes the segments to a content-addressed spill under the run's `snapshots` +directory — `/.segment` plus a small `/header.json` naming +the protocol, the whole-stream digest, the size, and the ordered segment table — +which is the same header-plus-bodies shape `ContentStore.put_frame` already +uses, and which gives the receipt a re-verifiable on-disk form independent of +the graph store. + +**The write surface, disclosed rather than left to be found.** This module +previously wrote nothing but `snapshots.mkdir`. The spill adds +`Path.write_bytes` for each segment and for `header.json`, and +`graph_implementation`'s `_RESOURCE_CALLS` (`:126-150`) covers *reads* — +`open`, `read_bytes`, `read_text`, `read_csv`, `load`, `files`, `joinpath`, +`import_module`, `exec`, `eval` and their neighbours — so a write is invisible +to `resource_accesses_sha256` by construction and no committed pin would notice +a redirected spill. Every component of the path is therefore checked here +instead: `root` is the snapshot root `_root()` already validated as symlink-free +and disjoint from the source root (`SNAPSHOT_LOCATION`), `name` must be a Python +identifier (`ROSTER_NAME`), each segment's own filename is the sha256 of its +bytes, the spill directory must not be a symlink (`ROSTER_SPILL_LOCATION`), and +an existing segment or header must be a regular file of the recorded size +(`ROSTER_SEGMENT_CHANGED`, `ROSTER_SPILL_LOCATION`). Presence is decided by +`lstat` and not by `Path.exists()`, because a broken symlink does not exist and +would be written straight through — a hole the first draft of this code had and +its own test caught. + +`read_all()` reassembles, re-hashes each segment against its recorded digest +and the whole against `digest`, and refuses `ROSTER_LIMIT` above +`MAX_MATERIALIZED_BYTES`. The two consumers that need whole bytes — the +`preparation` and `allocation` opaque artifacts, which reach +`ContentStore.put_bytes` through `KernelResult.artifacts` — call it. So +`AuthenticatedSurveyPopulationPreparation.payload` stays `bytes` and **every +existing consumer is unchanged**: `_sha(entry[1])`, `type(value.payload) is +bytes`, the `CANDIDATE_MISMATCH` byte comparison, and the artifact publication +at `graph_survey_population.py:510`. + +This is deliberately honest about what it does and does not buy. It removes +every ceiling that binds below full source and it keeps the encoder's peak +allocation at one segment. It does **not** remove the final materialisation: +1.02 GiB of bytes exists once, at full source, because +`KernelResult.artifacts` is a mapping of `bytes` and `put_bytes` takes whole +bytes. Removing that last copy needs a streaming artifact channel in +`microcosm-graph`, which is main-only and is §4's sibling. At full source +1.02 GiB against a ~26 GiB working set is not what stops a build. + +### 2c. The row budgets + +`_bounded_append`'s `ORIGIN_LIMIT` (ceilings #2 and #4) is a bound on the +Python row list, not on a transport. It keeps its code and its per-row +`_encode`, and its ceiling becomes `MAX_ROSTER_BYTES` — the same explicit +resource ceiling — so the refusal still fires, on the same rows, with the same +message, at a number a full-source build does not reach. + +### 2d. The allocation payload + +`_allocation_payload` (`graph_survey_population.py:344-369`) is already a +stream with a per-row bound: it emits `_bounded_json(_instruction_document(row), +4096)` per household into one bytearray with a pre-append check against +`ALLOCATION_MAX_BYTES` (`:362`) and a post-loop re-check (`:367`). Its bytes are +byte-identical to the canonical whole-tree encoding, which +`test_us_graph_survey_population.py:304` pins — an identity that holds only +because `"households"` sorts before all six metadata keys — so the same +streaming-digest argument applies. + +It is built inside `SurveyPopulationAllocationKernel.run`, which has no +filesystem path and must not write to the store itself, so it takes the +in-memory half of §2b: bounded segments in a list, each guarded by +`ALLOCATION_LIMIT` at the unchanged 64 MiB, under `MAX_ROSTER_BYTES` in total, +materialised once. `allocation_sha256` (`:441`) is unchanged in derivation and +in value. + +Ceiling #5, the `// 128` structural pre-check, moves to +`ALLOCATION_ROSTER_BYTES // 128` = 33,554,432 households. It bounds +`allocation_instructions`' intermediate maps, and its old 524,288 is **below** +the source's 1,587,376, so a full-source selection refused there too. The code +is unchanged. + +**One refusal narrows, and this design says so rather than leaving it to be +found.** `ALLOCATION_LIMIT` fired when the *whole* payload passed 64 MiB. Under +a segmented stream that condition no longer arises: a segment closes instead. It +still fires, with the same code and the same expression, on the condition a +segmented stream can still reach — **one instruction row plus the reserved tail +larger than one whole segment**. `test_us_graph_survey_population.py` drives both +that refusal and the new total ceiling, and asserts the payload's bytes are +unchanged at four different segment sizes down to the smallest one that holds a +row. The same narrowing applies to the preparation receipt's `PAYLOAD_LIMIT`: +`_chunks` yields one JSON token at a time and a segment never splits one, so the +refusal now means "one token larger than one segment", and because +`_check_scalars` has already bounded every string to 1 MiB the shipped 64 MiB +segment always holds any token. Both are tested at a segment size below the +longest token. + +### 2e. Size law after the change + +`MAX_ROSTER_BYTES` is set to `64 * MAX_SEGMENT_BYTES` = 4 GiB, which is +3.9× a full-source preparation receipt and 7.6× a full-source allocation +payload (measured 316.665 B per household × 1,587,376 = 502.6 MiB). The +refusals therefore still exist, still fail closed, and no longer bind below +full source: + +| | before | after | +|---|---|---| +| preparation receipt | 96,860 hh (6.10%) | 6,206,000 hh (391% of source) | +| `_origins` rows | 150,916 hh (9.51%) | 9,658,000 hh | +| allocation payload | 211,924 hh (13.35%) | 13,563,000 hh | +| `_plan_document` rows | 270,284 hh (17.03%) | 17,298,000 hh | +| allocation map pre-check | 524,288 hh (33.03%) | 33,554,432 hh | + +A fourth whole-roster receipt the cost attribution did not name is in the same +module: `_current_survey_wage_projection` builds one row per selected wage +earner and encoded it the same way. It takes the same transport. It does not +bind at 1/10 and does bind at full source. + +## 3. Retention and the frame seal + +### 3a. What the observer costs and where + +`run_graph` calls `_population_observer(node_id, _observer_snapshot(updated))` +at `executor.py:2773-2774`, under the single guard +`if _population_observer is not None:`. That is **outside** the cache-load +attempt (`if resume != "forbid":`, `:2575`), outside the cold path +(`if result is None:`, `:2607`) and outside the store write (`if not hit:`, +`:2776`), so every reached node pays it, cache hits included; only the +unreached-node `continue` (`:2510-2524`) skips it. `_observer_snapshot` +(`:356-408`) detaches by round-tripping every entity and link table plus the +strata through `pickle.loads(pickle.dumps(..., protocol=5))` (`:373`) — a full +independent copy, measured by the cost-attribution report at **9.6–9.8 bytes +per cell per snapshot**. + +The financial runner retains all of them: `observed[node_id] = population` +(`graph_atomic_survey_financial.py:1644`), never cleared, read at `:1656` +(node ids), `:1796`, `:1818`, `:1821`, `:1824`, `:1944`, `:1948`, `:1974` +(the objects), `:1897` (object *identity*) and `:1945` (the stamp alone). It +reaches the frozen `_FinancialRunState.node_populations` field (`:127`) through +`_issue_run` and `_node_population_seals` (`:500-515`), and `_pure_run` +(`:676-691`) re-stamps every retained population on **every** `checked_view()`. + +### 3b. Why the snapshot cannot be addressed to the store + +The obvious move — write each snapshot to the content store and keep the key — +is refused by the store's own normalisation, in three independent ways, and this +design records them rather than discovering them later: + +* `store.py:456` zeroes the values beneath a null mask (`values[mask] = 0`), + while `survey_atomic_geography._population_stamp` (`:230`) deliberately folds + the physical storage parts *under* the mask — its docstring says so at + `:231-237`. A store round trip therefore changes every stamp. +* `store.py:619` refuses a `MultiIndex` outright, and + `test_graph_executor.py:4254-4255` builds one into the observed frame. +* `store.py:486-487` refuses `CategoricalDtype` and `DatetimeTZDtype`, and + `test_graph_executor.py:4262` observes a categorical column. + +There is also no store API that accepts a `Population` (only `put_frame` for a +`Frame` and `put_bytes` for bytes), and `DataFrame.attrs` — asserted preserved +at `test_graph_executor.py:4320-4321` — is not persisted by `_write_frame`. +**A store-backed snapshot is not a drop-in for the pickle.** Anything that +spills a snapshot must spill the same pickle bytes the executor already trusts +for detachment. + +### 3c. Object identity is pinned, so re-materialisation is not free + +`result.financial_population is observed[final_node]` +(`graph_atomic_survey_financial.py:1897`) is an identity check, not an +equality check, and the completion host has two more of its own. A policy that +dropped a population and rebuilt it from a spill would produce a different +object and fail these. So the retention policy is **declared-consumer**: the +nodes whose objects a caller holds are named, retained in memory, and the rest +may be spilled. For the base financial run those are the final node, +`financial.ATTACH_NODE`, and — when the tax rebase is enabled — the property +population. + +### 3d. What still needs every population, and the question it raises + +Two loops need the whole `Population` object, not a seal: +`atomic.same_replayed_population(population, observed[node_id])` at `:1796` and +`:1948`. They compare the executor's observation against the runner's own +independent replay, field by field and byte by byte +(`survey_population_replay.py:132`, `:179`). The replay cannot be built until +`run_graph` has returned, because it reads the manifest's loaded artifacts, so +between the observation and the comparison **nineteen detached populations must +exist somewhere** — RAM (~290 GiB at full source) or disk (~285 GiB). There is +no third place, and choosing to replace the object comparison with a content +seal is a change to a verification contract, not an optimisation. §5 puts that +to the owner as a question. + +### 3e. The seal: a vectorised per-column pass, proven equal + +`survey_population_preparation._frame_identity` is the only remaining per-cell +frame walker in the US runtime. Its cost was literally + +```python +for value in series: + digest.update(_frame_cell_encode(value)) + digest.update(b"\n") +``` + +— one `_frame_cell_encode` call and two `digest.update` calls per cell, over a +quantity that is a pure function of the cell values plus dtype metadata. (The +two neighbours that look similar are already vectorised and are left alone: +`asec_current_money_source._series_digest` (`:98`) dumps `_data`/`_mask` bytes +for numeric dtypes and caches string tokens, and +`asec_2024_native_population._frame_identity` (`:184`) delegates to it.) + +The replacement builds, per column, **the exact byte string that loop would +have fed the digest**, and updates the digest once: + +| dtype | how | why it is exactly equal | +|---|---|---| +| `float64` | `_float_cells`: a fixed-offset `(n, 37)` uint8 matrix with a per-position keep mask, cut down with one boolean gather | the record is `["float","` + optional `-` + `0x1.`/`0x0.` + 13 mantissa nibbles + `p` + sign + up to 4 exponent digits + `"]`; the mask deletes the parts a value does not spell, so `float.hex()`'s variable spellings come out of one padded matrix | +| `int8`…`uint64` | `_integer_cells`: `"\n".join(map(str, values.tolist()))` | `str` on a Python int **is** `int.__repr__`, which is the fast path's encoder; every integer-dtype value lies inside `-2**63 <= v < 2**64` and spells ≤ 20 bytes, so neither `_encode` nor `PAYLOAD_LIMIT` is reachable | +| `Int8`…`UInt64` | `_nullable_integer_cells`: the same, with `pd.NA` positions overwritten by `null` | `_cell` maps `pd.NA` to `None` and `_frame_cell_encode` spells `None` as `null`; unsigned widths keep an unsigned carrier so `UInt64` above `2**63 - 1` still spells | +| `bool`, `boolean` | `_boolean_cells`: index into the three spellings | three possible cells | +| `string` | `_string_cells`: `pd.factorize`, then `_frame_cell_encode` once per distinct value | `StringDtype` compares and hashes exactly, so grouping equal values cannot merge two whose encodings differ | +| anything else | the cell-at-a-time walk, unchanged | `object` and `category` deliberately do **not** qualify: `1`, `True` and `1.0` are equal and hash alike there while `_frame_cell_encode` spells them three different ways | + +`_cell` refuses a non-finite cell with `FRAME_NONFINITE`; the float branch +raises the same code for the column rather than at the first offending cell, +and the digest is discarded either way. + +**The equality is proved against an independent oracle that already existed.** +`packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py` +carries `_legacy_frame_preimage`, a verbatim predecessor traversal, and +`_assert_exact_frame_preimage`, which records every `digest.update` call through +a patched `hashlib.sha256` and asserts the **concatenated byte preimage** — not +merely the digest — equals the oracle's. That is a byte-for-byte proof, and it +is joined by an exhaustive float proof and a per-dtype sweep this lane adds. + +## 4. What `run_graph(population_retention=...)` on main would need + +`#901` recorded that the shared executor has no retention mode, and that is +still true at this head: there is no eviction, pruning or population dropping in +`executor.py` or `store.py`. The executor's own `populations` dict is keyed by +**version**, not node, and a `StructuralDelta.NONE` node overwrites its entry +(`:2769`), so the executor itself already retains one population per version; the +per-node retention is entirely the observer's. + +**One such mode was written and dropped, and it is not the one this work needs.** +`run_graph(population_retention="lazy")` existed on PR #893's original branch, +staged as layer 08 GRAPH-ATTACHMENT-METADATA: commit `6c0f24c77` "Add opt-in lazy +population retention" adds `packages/microcosm-graph/src/microcosm/graph/attachments.py` +(210 lines: `_StoredPopulation`, `_LazyPopulations(Mapping[str, PopulationView])`), +81 lines of `executor.py`, 41 of `keys.py`, 31 of `manifest.py`, and +`tests/test_population_retention.py` (528 lines). The #893 reconciliation dropped +it (`experiments/893-reconciliation-amendments-19-20-20260912.md:192`) because +"no call site outside the graph package passes `population_retention`; its only +consumer was the branch's own `test_lazy_snapshot_metadata_integration.py`, +dropped with it", and recorded the way back: "if it is wanted it is a +self-contained later PR on top of piece B, which it imports." + +**Read what it actually did before reaching for it.** `attachments.py` imports +`ContentStore`, `_write_frame` and `_payload_table` and its docstring says +"Private, non-portable population attachments and execution lifetimes. These +store references never enter node records or portable manifest identity." Its +`_LazyPopulations` is a `Mapping[str, PopulationView]` — so it made the +**manifest's attached population views**, one per structural version, lazy +against the content store. That is a different target from the observer's +per-node detached `Population` snapshots, and §3b's three store refusals are why: +a `PopulationView` is a portable Frame that already goes through `put_frame`, and +a detached snapshot is not. **Restoring layer 08 would not remove the nineteen +pickles.** It is still worth restoring on its own terms, and it is still the +closest existing shape for the keyword, but the work §3d describes is a second +thing. + +A `population_retention` keyword would have to carry these, all of them +established above: + +1. **A mode that skips the detachment, not just the retention.** The cost is + `pickle.dumps` + `pickle.loads` per node (`:373`). An observer that only + seals needs no detached copy at all, because sealing is read-only. The + executor would hand it the live population and the mode would be the + observer's declaration that it will not retain or mutate it. + `test_graph_executor.py:4368-4374` already pins the converse — with no + observer, no snapshot may be allocated — so the shape of that test is the + shape of the new one. +2. **A declared-consumer roster.** `_observer_snapshot`'s detachment exists so + the observer may mutate what it keeps (`test_graph_executor.py:4183-4248` + asserts a mutated snapshot moves no node key, no manifest and nothing in the + store). A retention mode that hands out a read-only view instead would make + that mutation raise — *more* closed, but a behaviour change to a documented + contract in `microcosm-graph`, so it belongs to a main-only change with its + own test rather than to a country runner. +3. **Invocation invariants that do not move.** Exactly once per reached node, in + `compiled.order`, on cache hits and cold execution alike + (`test_graph_executor.py:4153`, `:4166-4167`); an observer's own exception + still refuses the run with the observer's exception type (`:4172-4179`); + `ATOMIC_OBSERVER_ROSTER` and `COMPLETION_OBSERVER_ROSTER` still refuse any + gap or reorder. +4. **A precedent in the same package.** `graph_survey_puf55` already declines to + retain everything: its issued run entry "retains the final output, not all 245 + executor-observed snapshots" (`graph_survey_puf55.py:638-639`). The financial + runner's `node_populations=observed` is the outlier among the six observers in + this runtime, and the shape it would move to already exists a few files away. +5. **A statement about `_pure_run`.** `state.node_populations` is re-stamped on + every `checked_view()`, so a retention mode that drops populations must say + what `FINANCIAL_NODE_POPULATION_CHANGED` means afterwards. Today + `_node_population_seals` already returns `()` when there are no observations + and no completion boundary (`:502-503`), which is the existing shape of + "nothing retained, nothing to re-stamp". + +Item 1 is the whole memory wall and most of the avoidable CPU. It is main-only, +it lands in `microcosm-graph`, and the executor hunks of PR #938 are its +neighbours. + +## 5. The ceilings this lane did not touch, and where they sit + +A transport ceiling is not the only kind. The 19-node path carries a second +family — row-count and row-byte bounds — and this section records where each one +sits so the next lane does not rediscover them one build at a time. The counts +come from the recovered 1/1000 artifact's own rosters: 1,584 selected households +carry 1,584 stacked households and 3,464 stacked persons, i.e. 2.187 persons per +household, and the combined clone doubles both. + +| bound | value | binds at | +|---|---|---| +| `survey_population_domains.MAX_HOUSEHOLDS` | 100,000 | **never** — it bounds one `classify_households` batch, and `survey_catalogue_selection` streams batches of at most `_BATCH_HOUSEHOLDS = 10,000` households or `_BATCH_PEOPLE = 100,000` people (`:187`, `:197-201`) | +| `survey_population_domains.MAX_TOTAL_MEMBERS` | 1,000,000 | **never**, for the same reason | +| `survey_population_domains.MAX_MEMBERS` | 20 | per household, unrelated to the fraction | +| `acs_person_coverage_columns.MAX_SELECTED_ROWS` | 1,000,000 | above 1/10; 347,137 stacked persons at 1/10, 3,471,383 at full source | +| `asec_current_money.MAX_PERSONS` | 1,000,000 | above 1/10, same counts | +| `acs_pums.MAX_EXACT_HOUSEHOLDS` / `MAX_EXACT_PERSON_ROWS` | 1,000,000 | above 1/10; 1,587,376 households at full source | +| `survey_observed_age.MAX_ROWS` | 2,000,000 | above 1/10; ~694,000 cloned persons at 1/10, ~6,943,000 at full source | +| `survey_origin_budget.MAX_GROUPS` | 1,000,000 | not established — the group count was not derived here | +| `graph_survey_age_artifact.MAX_PEOPLE` | 10,000,000 | above full source's ~6.94M cloned persons, so not at all | + +**So a 1/10 build meets no ceiling this lane did not lift, and a full-source +build meets at least four more** — all of them row counts rather than transport +shapes, and all of them in the 1,000,000–2,000,000 range that a 1.59M-household +source crosses by construction. They are a separate change with a separate +argument, and this lane does not touch them. + +## 6. Open, for the owner + +1. **Columnar format for the receipt bodies.** This design uses the store's own + existing shape — a `header.json` payload table plus content-addressed bodies, + as `put_frame` already writes `.npy` files — and keeps the bodies as canonical + JSON segments, so nothing derived from the byte stream moves. A true + Arrow/parquet body needs **no new dependency**: `pyarrow>=15` is already + declared by `microcosm-build` (`pyproject.toml:20`, "Parquet is the artifact + format for the us_trade margin and entry tables"), pinned at 25.0.0 in + `uv.lock`, and already named in this very stage's `STAGE_DEPENDENCIES` + (`graph_survey_population.py:77`), so its version already enters the stage's + implementation hash. What a columnar body would cost is the byte stream: it + changes, and therefore `preparation_sha256` and every node key and store + address move, once and deliberately. +2. **Whether the retention policy is a run option or the only behaviour.** §3d + shows that nineteen detached populations must exist between the observation + and the replay comparison unless the object comparison is replaced by a + content seal. That is a verification-contract decision, and §4's item 4 shows + the runtime has already made it once, the other way, in `graph_survey_puf55`. +3. **Whether the roster spill should be readable by anything but this module.** + It is written, verified, and then only reassembled in the same call today. A + consumer that wanted to stream it — a future `put_bytes` that takes an + iterator, or an auditor re-deriving `preparation_sha256` without loading + 1 GiB — would read `header.json` and the segments it names, which is why the + header exists at all. diff --git a/docs/us-native-soi-agi-targets.md b/docs/us-native-soi-agi-targets.md new file mode 100644 index 000000000..17c0b6954 --- /dev/null +++ b/docs/us-native-soi-agi-targets.md @@ -0,0 +1,77 @@ +# Native integration of SOI Table 1.1 targets + +This compiler-only port adopts the target representation from +[Microcosm #960](https://github.com/PolicyEngine/microcosm/pull/960), reviewed at +`b563d11be314bdd608f736c553f435b9236820df` and retrieved September 19, 2026. +[Issue #958](https://github.com/PolicyEngine/microcosm/issues/958) distinguishes +these target changes from the separate full-vector tail support work. + +The integration base is the clean native continuation +`59daeda3bf4a745eca2119e59520c6f691afe252`, descended from the active native API +source `5258a4c7f6d05cb1e68806d51b38996ea116bfb5`. At review, the compiler, its +existing tests and fact-to-target guide on that base were byte-identical to +fetched main `16c8e78d2f60d629da5d70294f643bce5c4597e2`. A sibling worktree carries +this port; the measurement and owner-PR worktrees are unchanged. Only compiler +source, compiler tests and documentation are adopted. The PR's population, +prototype reweighting and reform-score scripts are excluded. + +## Accepted facts and year alignment + +The rescue applies to stale `irs_soi.table_1_1` facts with all filing statuses, +national geography, measures `return_count` or `adjusted_gross_income`, and a +lower AGI bound of at least $100,000. Other tables, state/status slices and lower +cross-period bands remain excluded. The +[IRS table index](https://www.irs.gov/statistics/soi-tax-stats-individual-statistical-tables-by-size-of-adjusted-gross-income) +identifies Table 1.1 as all returns by size and accumulated size of AGI, and +Table 1.2 as a separate marital-status table. The $100,000 edge is the model's +existing tax-unit-count approximation threshold, not a statutory filing limit. + +Each stale class is represented as its share of the same-vintage Table 1.1 +national total, multiplied by the latest eligible national total of that +measure. Controls cannot come after the build year or before the class year. +An absent or zero same-vintage denominator leaves the class unbound. The +existing dynamic target selection chooses the latest eligible class vintage. + +For a 2022 class aligned to a 2023 control and aged to a 2024 build: + +| Field | Amount target after aging | +|---|---| +| `name` / source-record identity | Original 2022 source record | +| `ledger_fact_period` | `2022` | +| `uprating_from_period` | `2022` | +| `uprating_to_period` | `2023` | +| `source_period` | `2023`, the existing ager's alignment origin | +| `target_period` / `aged_to` | `2024` | + +The amount receives the realized 2022-to-2023 control ratio once, then the +remaining 2023-to-2024 CBO AGI ratio once. Counts receive the realized national +return-count ratio; the amount ager does not apply a CBO dollar factor to counts. +The nominal AGI edges remain the declared source edges. Same-period facts bypass +this rescue and preserve their existing behavior, including the existing +below-$100,000 behavior. No target IDs, API arguments or default compiler options +are changed. + +## Integration boundary + +The existing `compile_us_fiscal_target_registry` entry point consumes this code +when the separately admitted native release source includes the commit. No +Chronicle feed, the shared 32,867-target record, parity manifest, calibration +registry snapshot or frozen comparison surface is regenerated here. A future +build must record its actual newly compiled surface and requalify the relevant +frozen identities. The PR's separately reported target counts are not counts +verified on the shared native Chronicle artifact. + +These targets do not create high-income support or repair its income +composition. They cannot establish support adequacy, calibration feasibility or +scientific improvement. The separate full-vector tail and ownership work remains +a launch prerequisite under #958. No dataset, calibration, model or reform run +was performed for this port. The held-out top-rate benchmark remains held out; +this change adds no target family for it. + +Validation uses invented facts to exercise compiler acceptance, exclusions, +future/control-year selection, missing denominators, original identity and the +complete two-stage amount-aging chain. The official TY2023 workbook was acquired +and hashed during review, but no installed XLS reader was available, so the +upstream test's published numeric fixture was not independently cell-verified. +That source-review limitation does not change the values compiled from supplied +facts; production code introduces no hardcoded target amounts. diff --git a/docs/us-native-source-memo.md b/docs/us-native-source-memo.md new file mode 100644 index 000000000..a5847bf05 --- /dev/null +++ b/docs/us-native-source-memo.md @@ -0,0 +1,254 @@ +# Persistent memo for US source derivations + +The historical native US profile below spent most of its CPU outside the +graph, re-deriving values from pinned source archives. This note records what +`microcosm.build.us_runtime.source_memo` memoizes and the rules that keep a +recalled value equal to a recomputed one. The resumed 2026-09-23 session did +not run an actual-archive benchmark or a 1/1000 graph measurement. + +## Where the outside-graph CPU goes + +`experiments/native-source-auth-memo/attribute_outside_graph.py` partitions +every sampled stack in the `outside_graph_executor_calls` bucket of the +2026-09-19 1/1000 attribution run at `9af56aa8a`. The input is the harness's +`completed-aggregate-report.json`, SHA-256 `4e7be8ad…`. The script's output, +`attribution-9af56aa8a-1-1000.json`, is checked in with the attribution script. +This is inherited evidence; the resumed session did not rerun the graph or +reproduce the report. The values are statistical process-CPU estimates at a +0.25 s sampling interval, not exact function CPU. + +| Phase | Process CPU-s | Outside graph | Share | +|---|---:|---:|---:| +| Cold | 1,999.4 | 1,681.6 | 84.1% | +| Required replay | 1,652.1 | 1,624.7 | 98.3% | + +Each sampled stack is charged to exactly one issuer: the first owner entry +point found on it, taken in a fixed precedence order. Cold phase: + +| Issuer | CPU-s | +|---|---:| +| ACS native coverage (`issue_acs_native_coverage`) | 469.4 | +| ACS source catalogue (`issue_acs_source_catalogue`) | 467.5 | +| ASEC source catalogue | 176.8 | +| ASEC native population | 174.6 | +| Atomic geography requalification | 108.4 | +| Predictor requalification | 100.8 | +| Preparation assembly and validation | 68.7 | +| Verification-epoch close | 43.4 | +| Other (selection, runner, borrow checks, seals) | 72.1 | + +The two ACS issuers account for 936.9 of the 1,681.6 CPU-s. Their cost is +dominated by work that is a pure function of the archive bytes: + +| Function (inclusive, cold; overlapping) | CPU-s | Call sites | +|---|---:|---:| +| `acs_housing_universe_source._reconstruct` | 381.5 | 2 | +| `acs_housing_universe_source._archive` | 220.1 | 2 | +| `acs_population_catalogue._collect` | 216.5 | 1 | +| `acs_person_coverage_authentication._inventory` | 153.9 | 3 | +| `acs_pums.load_acs_pums_tables` | 148.9 | 1 | +| `acs_housing_universe_source._select` | 121.0 | 2 | + +The ASEC figures predate the CSV guard fast path (`b4f9b8eeb`), which this +branch's base `47960af43` carries. In the `9af56aa8a` profile, +`asec_coverage_authentication._capture` took 221 inclusive CPU-s across its two +call sites. No new ASEC timing is reported here. + +## What is memoized + +Six derivations, each keyed by the exact bytes it reads: + +| Namespace | Stands in for | Key inputs | +|---|---|---| +| `acs_housing_universe_source.full_projection` | `_archive` of both archives: the canonical full lexical projection and member inventories | both archive SHA-256s | +| `acs_housing_universe_source.selection` | `_select` and the receipt built from it | full projection and inventory digests, pins, the exact selection tuple's digest, implementation and definition digests | +| `acs_person_coverage_authentication._inventory` | one archive's member inventory and selected rows | archive SHA-256, role, digest of the sorted selection | +| `acs_population_catalogue._collect` | the catalogue records, vacancies and counts | full projection digest, both archive SHA-256s | +| `acs_pums.load_acs_pums_tables` | household and person tables, with read metadata | both archive SHA-256s, vintage, household limit, chunk size, ordered selection digest, live owner and parser identities | +| `acs_person_coverage_authentication._coverage_columns` | selected literal coverage columns and their receipt | person archive SHA-256, exact encoded person-key frame, chunk size, live owner and parser identities | + +In one cold 1/1000 preparation, the housing owner runs twice: once for the +catalogue and once for the selected native population. The second +`_archive` pair and the repeated selected inventories are recalled within the +run. A later run can recall all six derivations when their inputs and identities +match and their completed values pass the encoding proof. + +Owners change nothing else. They still capture and pin-check their sources, +write and re-read the projection file, construct and seal every issued +object, and run every live-object, mutation and producer check. The recorded +producer and implementation identities change only because the owner files +changed. + +## Rules + +**Keys.** A key is `sha256("microcosm.us.source-memo.v1\0" + canonical(document))`. +The document holds: + +- the namespace; +- a runtime identity: interpreter version, optimisation level and platform, + the zlib runtime, digests of the stdlib `csv`, `json` and `zipfile` modules, + the binary that provides `_csv`, `_json` and `zlib`, and the memo module + itself; +- the owner's code identity; +- every input as `{role, sha256, bytes}`; +- the canonical parameters. + +`canonical` is sorted, compact JSON encoded as strict UTF-8. ASCII escaping +would spell a lone-surrogate pair and the astral character it resembles +identically. Strict UTF-8 refuses to encode the pair, so the value never +becomes a key. File inputs are hashed through one no-follow descriptor at call +time. Stat metadata is never trusted. + +**Code identity.** Each owner passes the identity its receipts already record: +the scoped implementation hash for the housing owner, and `_producer()` for +coverage and the catalogue. It adds `live_code(*modules)` over the modules the +derivation executes, which records: + +- each module file's SHA-256; +- a check that every function and method the module defines still runs the + code compiled from those bytes; +- the identity of every function bound in the module's namespace; +- the module's immutable constants, such as limits, pins and field lists. + +A monkeypatched constant or function changes the key. Loaded code that no +longer matches its file raises, and the memo is bypassed. + +Frame derivations also include NumPy, pandas and PyArrow versions, digests of +the selected parser and array-provider modules, and the pandas options that +choose string dtypes. The live parser binding is checked on every identity +formation, including the first lookup. The pandas C-engine path binds +`read_csv`, the source-defined functions and methods in `TextFileReader`, +`CParserWrapper` and `ParserBase`, their dispatch aliases and class bases, and +the native `TextReader` constructor and read/close methods. Changed parser +bindings bypass the memo; parser +defaults and NA tokens participate in the key. This covers the ACS CSV parser +path, not arbitrary monkeypatches throughout pandas or NumPy. + +**Entries fail closed.** An index file `index//.json` names +content-addressed `blobs/
/` files. It carries an HMAC-SHA256 made +with a per-user key that must live outside the memo root, so a copied or +foreign memo confers nothing. A lookup accepts an entry only if all of these +hold: + +- the index is a bounded, regular, non-symlink file in canonical form; +- its MAC verifies; +- its key document equals the requested one byte for byte; +- every blob is a regular, non-symlink file of the recorded size and SHA-256. + +Anything else is a counted miss, never partial trust. The owner recomputes from +source, and the fresh entry atomically replaces the rejected one. + +**Only completed, exactly representable values are recorded.** An exception +from the computation propagates unchanged and records nothing, so every refusal +keeps its code. After a miss, the code identity and file inputs are +recomputed. A value is stored only if neither moved and the owner's proof shows +that the encoding decodes to exactly the computed value. The lexical owners +check their exact tuple, list, string, integer and ordered-dict shapes. The +frame owners also prove their metadata with a typed codec and compare each +decoded frame's dtypes, labels, index and values. A value outside the owner's +supported shapes, or one with a lone surrogate, is not stored. + +**Exact frame scope.** The codec accepts plain pandas DataFrames without attrs, +with duplicate labels allowed by the frame flags, a RangeIndex, and unique +string column names in an object or pandas StringDtype index. Numeric columns +are NumPy bool, integer, unsigned integer or floating dtypes; their raw bytes +preserve byte order, signed zero and NaN payloads. StringDtype columns retain +their Python or PyArrow storage and their `pd.NA` or NaN missing-value mode. +Object columns retain exact Python `None`, bool, int, float, str and bytes +values, plus the `pd.NA` singleton. Floating values use their IEEE bits, so +signed zero and NaN payloads survive there too. Arbitrary objects and NumPy +scalar objects are refused. The typed metadata codec preserves tuple versus +list, dict insertion order and float bits. +NumPy dtype metadata, unsupported extension dtypes, frame subclasses and other +index shapes bypass storage rather than being converted into a different +representation. + +**Trust boundary.** Whoever holds the HMAC key is trusted to have run these +owners. A MAC-valid entry whose blobs decode wrongly is caught only if decoding +fails. Keep the key private (it is created `0600`) and outside shared storage. + +## Enabling it + +The memo is off by default, and an ordinary call runs exactly the unmemoized +path. There are two ways to enable it: + +```python +from microcosm.build.us_runtime.source_memo import source_memo + +with source_memo("/path/to/memo", key_path="/path/outside/memo.key"): + ... # native preparation or graph run +``` + +or set these environment variables: + +```bash +export MICROCOSM_US_SOURCE_MEMO=/path/to/memo +export MICROCOSM_US_SOURCE_MEMO_KEY=/path/outside/memo.key # optional +``` + +The default key is `$XDG_CONFIG_HOME/microcosm/us-source-memo.key`, or +`~/.config/microcosm/us-source-memo.key`. It is created on first use. +`source_memo(None)` disables the memo inside an enabled context. Context +activation is a context variable, so a new thread sees only the environment +variables. `source_memo.statistics()` reports hits, misses, rejections, +bypasses, bytes stored and seconds per namespace. + +The memo is a cache. Deleting its root is always safe. It never prunes itself. +A code change produces new keys, and old entries stay until the root is +removed. + +## Evidence + +`packages/microcosm-build/tests/test_us_source_memo.py` covers: + +- **Memo behaviour:** activation and isolation, content-addressed hits, and + key sensitivity to bytes, code and parameters. +- **What is never recorded:** refusals, values that cannot be represented + exactly, and inputs or code that moved during the computation. +- **Bypass:** identities that cannot be formed skip the memo. +- **Tampering:** blob digest, size or absence; index MAC, fields, form or + garbage; symlinks; a foreign key; a misfiled entry; a decode failure. Each is + a counted miss that the next store repairs. +- **`live_code` binding.** +- **Owner identity:** + - The housing source, the ACS catalogue and the whole ACS+ASEC preparation + payload are byte-identical with the memo off, cold and warm. + - A warm preparation executes none of `_archive`, `_select`, + `_inventory_uncached` or `_collect`. + - A monkeypatched limit, or changed archive bytes, is never answered from + the memo. + +Additional fixture suites cover the frame derivations: + +- `test_us_source_memo_codec.py`: exact numeric, string and object columns, + missing markers, float bits, typed metadata and refusals outside the codec's + supported shapes. +- `test_us_source_memo_frames.py`: off/cold/warm table and coverage equality, + warm reads skipping their parser or scan, parameter and archive-byte changes, + unsupported key dtypes and invalid selections. +- `test_us_source_memo_parser_identity.py`: live parser replacements before + the first lookup and after warming, changed defaults and string options, + forged function identities and changed class dispatch. + +The resumed-session test commands and results are recorded in +[`experiments/native-source-auth-memo/README.md`](../experiments/native-source-auth-memo/README.md). +They use invented fixtures and do not certify actual-data artifacts. + +`experiments/native-source-auth-memo/bench_acs_owners.py` can time the ACS owners +on staged archives, in one process per phase. No successful off/cold/warm +receipt set was produced in the resumed session. + +## Measured + +The actual-archive owner benchmark and 1/1000 graph-run measurement remain +pending. Process inspection was blocked by the sandbox, so the required check +that no other native run was active could not be completed. The task's explicit +admission rule therefore prohibited both runs even though the initial +available-memory reading was about 79 GiB. See the experiment README for the +measurement status and reproduction procedure; no speedup is claimed. + +## Not memoized + +- The ASEC owners. Their capture cost was already cut at the base. +- Every live-frame seal and requalification. These digest live objects, not + source bytes, so there is nothing persistent to key on. diff --git a/docs/us-native-spm-amount-source.md b/docs/us-native-spm-amount-source.md new file mode 100644 index 000000000..0fce043ba --- /dev/null +++ b/docs/us-native-spm-amount-source.md @@ -0,0 +1,38 @@ +# Native SPM amount source + +`qualify_current_spm_amounts(preparation, spm_inputs)` projects current ASEC +`SPM_ENGVAL` and `SPM_CHILDCAREXPNS` through the existing authenticated money +owner and qualified original SPM membership. It returns descriptive person and +unit tables with evidence. It issues no source capability and does not attach +canonical consumer inputs. Callers must retain both genuine owners and +requalify after relevant I/O. + +The projection joins exact current-year native keys and IDs, checks original +household/person coordinates, and preserves complete literal SPM membership. +Amounts repeated on person records represent one unit amount; they are never +summed. Replicated amount bits, validity, status and zero origin must agree. +An absent money member leaves the whole unit unknown. Historical restated +amounts cannot fill current-year observations. ACS units remain unobserved. + +Missing money uses the maintained canonical positive-zero storage encoding, +with missing status and invalidity retained; that storage value is not an +observed zero. Frozen-storage zeros also remain unknown unless their original +evidence authenticates a Census-encoded zero. Unresolved annual membership +does not qualify a nonzero amount merely because it is present. + +The existing current-money domain and resource pins remain authoritative for +admission. In particular, this adapter does not resolve the conflicting energy +range in the ASEC dictionary or bypass the original owner's refusal. Childcare +is labeled reported uncapped expenditure. It is not claimed to be an observed +amount before subsidy. + +The pure `project_spm_amounts` function accepts supplied descriptive evidence +for inspection and testing. Its output and `spm_amount_values_seal` hash confer +no authority. The genuine borrower checks both retained owners, exact active +implementation dependencies and source bytes before and after projection. + +This layer performs no fitting, clone transport, graph attachment, calibration +or consumer export. Choosing unit rather than person donor weighting, completing +ACS amounts, and resolving consumer naming remain separate modeling work. Passing +invented fixtures establishes these contracts, not actual-data quality or release +readiness. diff --git a/docs/us-native-spm-input-bridge.md b/docs/us-native-spm-input-bridge.md new file mode 100644 index 000000000..287938570 --- /dev/null +++ b/docs/us-native-spm-input-bridge.md @@ -0,0 +1,98 @@ +# Native survey SPM source inputs + +The native SPM bridge qualifies two source inputs for income year 2024: +`is_spm_independent_minor_role` on people and +`spm_unit_spm_universe_status` on SPM units. It does not compute thresholds, +resources or poverty, certify a dataset, calibrate weights, or authorize a +release. A detached Frame or copied receipt does not carry source authority. + +## Source meaning + +ACS requires explicit native SPM construction and the named +`acs_spm_household_only_analysis_2024` profile. The profile retains separate +choices for admitting modeled and approved-inference membership. Its per-unit +classification uses the reviewed construction evidence, without regrouping. +Ambiguous secondary relationships make the household unresolved. Missing roles +remain missing. Group-quarters records are outside this **restricted ACS +household-only analysis**, not outside every possible SPM analysis. + +ASEC requires the versioned +`ASEC_2025_INCOME_2024_SPM_POLICY`. Census specifies an unrestricted SPM +person universe in the [2025 ASEC technical documentation, Appendix D, +D-3](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf). +That instruction supports inclusion only after authenticating the exact source +people and complete original SPM unit. It does not justify including arbitrary +rows labeled ASEC. Role and count reconciliation are additional integrity +checks; they do not supply the universe policy. + +The ASEC observations were collected in 2025 and refer to 2024 income. The +bridge preserves survey-time age and relationships. It does not backdate age, +infer residence history, reuse `POV_UNIV` as SPM scope, or transfer the ACS +household-only exclusions to ASEC. Omitting the ASEC policy leaves its annual +scope unresolved. A missing required role remains unknown and prevents engine +projection for that unit. + +## Retained source and final membership + +The source qualifier borrows the live preparation retained by the issued +enrichment run. It validates source ownership, implementation and source bytes, +and retains nullable roles, annual scope and reasons. Its validator must run +before and after consuming the borrowed values. + +The pure projection helper does not grant authority. It verifies the original +person axis, survey/source years and the receiving two-clone mapping. Every +receiving SPM unit and household must correspond to exactly one original group +and clone, with its complete person roster. Moving, mixing, splitting, +truncating or duplicating groups refuses. Existing output columns cannot be +overwritten. Projection returns new columns without changing the populations, +weights, strata or mass history. + +The source nullable role is preserved separately. Only an explicitly `OUTSIDE` +person may receive a chosen Boolean storage placeholder at the final engine +boundary. Null `INCLUDED` or `UNRESOLVED` roles refuse. Known roles with +`UNRESOLVED` scope retain that status for the country's `SPM_UNIVERSE_REQUIRED` +gate. The annual status is never carried into a different model year. + +The adapter honors the consuming country's explicit `spm.DATASET_SOURCE_INPUTS` +declaration, including both source inputs. Its runtime and import-free metadata +paths reject malformed, overlapping or unsupported declarations. Formula-owned +thresholds and resources remain rejected dataset columns, and the two source +inputs receive no invented defaults. + +## Enrichment graph + +`run_us_survey_enrichment` accepts three optional SPM arguments: +`spm_acs_profile`, `spm_asec_scope_policy` and +`spm_outside_role_placeholder`. Omitting all three keeps SPM disabled. Enabling +it requires a typed `ACSAnalysisProfile` for 2024 and an explicit Boolean +OUTSIDE placeholder. The ASEC policy may remain `None`, preserving unresolved +scope. Incomplete or untyped configuration refuses before enrichment fitting. + +The host qualifies its real retained preparation and preflights the complete +receiving membership. Three ordinary nodes then follow the hours attachment: +`survey_spm.source` retains private source evidence, `survey_spm.project` +transports it at the person and SPM-unit grains, and `survey_spm.attach` adds +only the two declared source inputs to the existing population version. +Source identifiers, nullable observations and mappings remain in private +artifacts. Public parameters and the final receipt record the chosen profile, +policy, representation and artifact digests. + +The host revalidates the retained source on cold and required-cache paths and +independently replays all three nodes against the complete population. Final +pure source and output checks follow the last source and artifact I/O. SPM +attachment does not make a run release eligible. Public qualification can still +inspect the issued enriched owner; projecting onto it again refuses because +the two input columns already exist. + +## Verification boundary + +Focused tests use invented source records and detached pure transport fixtures; +they do not constitute a population measurement. Genuine source-owner tests are +separate from pure classifier and projection tests. Optional-assembler skips or +expected failures are not evidence that native construction passed. + +The graph hooks and pure fragment checks do not replace a combined final-owner +cold/cache proof. That execution is a separate integration gate. Do not +interpret these helpers or the earlier development-cohort country result as +qualification of the full native population, a production wrapper, calibrated +estimates or an app deployment. diff --git a/docs/us-native-ss-report-completion.md b/docs/us-native-ss-report-completion.md new file mode 100644 index 000000000..f3b173971 --- /dev/null +++ b/docs/us-native-ss-report-completion.md @@ -0,0 +1,45 @@ +# Native Social Security report completion + +The optional `current_survey_ss_completion` qualifier and +`graph_current_survey_ss_completion` fragment complete **source reports** for +selected original ASEC and ACS people. They do not populate canonical individual +beneficiary inputs, join an enrichment host by default, or qualify a release. + +The qualifier borrows a genuine `AuthenticatedSurveyPopulationPreparation`. +Its donor population retains every original current ASEC person and the original +DESIGN weights, independently of receiving support selection. Eligible category +labels are positive reports whose source reason basis resolves completely to one +of the four maintained components. Known zero reports and below-15 NIU reports +are not labels. Publisher allocation codes remain in source evidence; no +allocation category is silently excluded. + +The approved development predictors are source age, current wage income, current +self-employment income, and the source Social Security report total. Required +unknown predictors refuse. The ASEC 2024 annual and ACS 2024 rolling-window +transport assumption remains scientifically unqualified. No age-only retirement +rule, assigned geography, prior wages, clipping or recipient fitting is used. + +When unresolved positive reports exist, seven graph nodes publish the full source, +model columns, resolved donor support, original recipient matrix, actual fitted +categorical artifact, raw probabilities, and a report-completion artifact. +`CategoricalConfig` and seed are explicit. The probability and report nodes own +no cells and join the donor version: the compiler makes a FILTER depend on every +member of its base version, so placing a fit consumer on the full-source version +would cycle through the donor FILTER. Every declared class needs positive +DESIGN-weighted support. Maintained categorical fit/probability kernels supply the +model, typed dependencies and cache behavior. Reason-allowed probabilities are +renormalized as conditional mean report shares by `complete_positive_basis`; +completed report components must conserve each known source total exactly. + +When no unresolved positive reports exist, only source and report nodes execute. +That path needs neither class support nor a fitted model. Both paths preserve +complete, zero and unknown source component bits. The versioned report artifact +keeps source arrays, raw probabilities and completed amounts separate and records +that reports may combine family payments. Its decoder reconstructs the completion +and verifies exact source and model bindings. + +The private boundary retains the genuine preparation. An integrating caller must +validate before graph/store work and after the final I/O, including required +replay, then perform the final pure seal. Detached Frames, projections, models, +receipts and cached artifacts cannot recreate that source authority. A production +host and canonical beneficiary interpretation remain separate future work. diff --git a/docs/us-native-survey-demographic-inputs.md b/docs/us-native-survey-demographic-inputs.md new file mode 100644 index 000000000..a2f26479a --- /dev/null +++ b/docs/us-native-survey-demographic-inputs.md @@ -0,0 +1,75 @@ +# Native survey demographic inputs + +The enrichment host offers two independent options, both disabled by default: +`demographic_inputs=True` binds `is_female`; `race_hispanic_inputs=True` binds +`cps_race` and `is_hispanic`. Each fragment borrows the original survey +preparation, binds values once per original person, and copies them through the +existing clone mapping. They do not add people, change weights, redraw values, +or assign geography. + +The race/Hispanic fragment captures source literals from the same pinned ASEC +person member and ACS person archive authenticated by the retained preparation. +It checks the complete captured members, source identities, original key roster +and final parent identity. Its projection is descriptive evidence, not a new +source credential. The host retains it and requalifies it after relevant I/O. + +## Source definitions + +`cps_race` represents the published CPS `PRDTRACE` categories. ASEC supplies +those codes directly. Hispanic origin uses the direct `PEHSPNON` response with +the `PRDTHSP` detail-universe crosscheck. The fragment preserves `PXRACE1` and +`PXHSPNON` allocation histories. These observations come from survey 2025; +their associated income cohort is 2024. See the [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), PDF pages 27–28 and 31–32. + +ACS 2024 Hispanic origin maps directly from `HISP`. Race uses `RAC1P` together +with `RACWHT`, `RACBLK`, `RACAIAN`, `RACASN`, `RACNH`, `RACPI`, `RACSOR`, and +`RACNUM`. The fragment checks agreement between the broad category, detailed +indicators and number of major groups. Native Hawaiian and other Pacific +Islander indicators form one major group. All combinations of the five groups +represented in CPS map to its published categories, including its residual +three-race and four/five-race codes. `FRACP` and `FHISP` preserve allocation +status. See the [2024 ACS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf), PDF pages 63, 106, 111, 127 and 129. + +“Some Other Race,” alone or in combination, has no exact CPS counterpart here. +It remains unresolved. A missing, malformed, unlabelled, contradictory or +allocation-unresolved source item also remains unknown. Allocation histories +that end without a published value do not establish a known response. Published +allocated values are retained with their allocation status; they are not +described as unallocated responses. No unknown is replaced by false or zero. + +## Graph inspection + +The visible `survey_race_hispanic` fragment has source, raw-column, canonical +binding and clone-attachment nodes. Source literals, observation year, +allocation and mapping-status columns accompany the canonical nullable outputs. +The graph's private source artifact retains source evidence without exposing +raw native person keys in public node parameters. A missingness mask belongs to +the canonical column; mapping status explains why a value is unresolved. + +The older `acs_release_predictors` mapping has a different contract: it uses +representative CPS codes for particular coarse donor-model categories. That +equivalence is not an exact source observation and is not reused here. + +The maintained SCF wealth model also uses an internal feature named `cps_race` +with its own coding. Its recipient helper maps known Hispanic origin to 3 and +non-Hispanic White/Black/Asian/Other to 1/2/4/7. Those are model-feature categories, +not canonical PRDTRACE. A future SCF graph must call or reproduce that explicitly +declared consumer transformation from canonical race plus Hispanic origin; it +must not feed the canonical 1–26 codes directly into that feature. This fragment +does not implement or change SCF consumption. + +## Possible future conditional harmonization + +An additional explicitly modelled operation could draw a CPS-compatible category +for an unresolved ACS person, conditional on reviewed common predictors and +the observed ACS category. Such an operation would need a scientific mapping +convention, authenticated donor population, model/seed declaration, consistency +constraints, validation and distinct imputation provenance. Original ACS +literals and unresolved-source status would remain available. Model output +must not overwrite the historical source claim or be relabelled an exact +crosswalk. This fragment implements no such convention or imputation. + +Tiny invented-source tests exercise capture, all supported category combinations, +unknowns, exact clone transport and cache replay. They do not certify national +data completeness, source comparability, a full enrichment host or release +eligibility. Native ACS required/default loader APIs remain unchanged. diff --git a/docs/us-native-survey-development-handoff.md b/docs/us-native-survey-development-handoff.md new file mode 100644 index 000000000..e3b9c0795 --- /dev/null +++ b/docs/us-native-survey-development-handoff.md @@ -0,0 +1,195 @@ +# Native survey development handoff + +`tools/build_us_fiscal_refresh_release.py` now exposes +`prepare_native_survey_development_input(run, checkpoint_directory)`. Its input +is a **retained, issued `SurveyEnrichmentRun`** in the same process that ran +`run_us_survey_enrichment`. The output is a `NativeSurveyDevelopmentInput` +containing a usable `Frame` and a machine-readable coverage report. + +```python +from tools.build_us_fiscal_refresh_release import prepare_native_survey_development_input +from microcosm.build.us_runtime.native_survey_handoff import ( + load_native_survey_development_checkpoint, +) + +# enriched_run remains alive from the native PUF/enrichment continuation. +result = prepare_native_survey_development_input( + enriched_run, "/owned/new/native-development-checkpoint" +) +frame = result.frame +missing_inputs = result.report["missing_inputs"] +remaining_gates = result.report["required_release_evidence"] + +# Validated reuse while the original issuer remains alive: +reloaded = load_native_survey_development_checkpoint( + "/owned/new/native-development-checkpoint", run=enriched_run +) +assert reloaded.owner_live_verified +``` + +The writer creates a fresh directory containing `population.h5` (the maintained +Frame checkpoint format) and `handoff.json`. It performs complete readback +comparison: entity tables and row order, schema/linkage, dtypes and knownness, +non-owned fields, geography, strata, typed weights and mass history. Frame +metadata uses the graph store's typed metadata encoding. Nullable values follow +the maintained replay comparator's documented null backing-buffer normalization; +known values and masks retain their meaning. No additional support clones, +weights, default inputs, fitted values or legacy assembly receipts are created. +The final owner check must succeed before the API returns. Existing output +directories are refused. + +The loader also works without `run` for downstream development inspection. In +that case `owner_live_verified` is false. Byte identities bind the two files; +they do not issue source authority. The Frame checkpoint is not a +`USSingleYearDataset` H5 or simulation-ready multispine pool. Do not send it to +`--base-h5` or `--pool-manifest`. Its Frame can be used by existing Frame-based +consumers after those consumers' own input checks, while the source run is kept +alive whenever source qualification is required. + +The separate `prepare_native_survey_engine_input(run, declaration=..., consumer=...)` +entry selects an exact, closed projection from a live owner. Its optional +`PolicyEngineUSEngine` consumer calls `validate_input_representation(frame, +period=...)` against that adapter's selected live registry and country source +declarations. This checks owning entities, selected-year formula ownership, +annual/eternity periods, actual enum names, knownness and physical dtype +compatibility, including lossless integer IDs and memberships. It copies the +entity tables and materializes the typed household weight for inspection; it +does not apply defaults, rewrite inputs, construct a dataset or calculate. + +Success adds only `consumer_representation_compatible: true` to the projection +report. Runtime admission, source applicability, SPM universe qualification, +scientific gates, simulation readiness and release eligibility remain separate. +Ordinary float32 rounding and representable infinities are not scientific +certification; unsupported nulls, integer overflow, finite-to-infinite conversion +and string truncation refuse without filling or remapping. `UNRESOLVED` can be +a valid enum representation while still preventing SPM measurement. Omitting +the consumer preserves the previous import-free, unqualified projection path. +Both paths retain the final live-owner and exact-storage comparisons. + +The builder's private `_calibrate_native_input_frame` bridge can then solve +already-materialized targets with the maintained dense or L0/refit calibrator. +It requires an explicit formula-ownership provider and a closed solve-only +option set; warm starts, checkpoints and exact-k identities are not accepted. +Numerical option validation remains with the maintained calibrator; this bridge +does not define another solver parameter policy. +Before solving, the target frame must retain every projected input column, +structural ID, membership, row order, stratum and typed initial weight. Target +materialization may add calculated columns; those columns never enter the +exported input frame. + +The bridge works on detached copies, attaches weights using the maintained +helpers and verifies the result against its supplied parent. It reconstructs a +complete float64 weight vector in original household order. L0 keeps exactly +the maintained selected support, including zero post-refit weights, without a +second pruning threshold. The returned `comparison_specification` records the +exact solve options, registry and loss-weight identities, both frame identities, +and the supplied specification digest. The existing comparison `binding` hashes +those specification bytes, so the pair must be retained together. Target +materialization may have different metadata and mass history; the exported +input frame preserves its input parent's metadata and mass-history prefix. +Callback changes to caller configuration cannot +rewrite those captured inputs; changes to either supplied frame refuse. +The returned measured result and comparison binding do not issue native owner +authority, establish target/scientific qualification or authorize publication. + +The report preserves the owner's receipt and source content identities, the +native population version and graph manifest identity, and the amount model's +source/draw flags. It merges the maintained pool input roster, release-source +column roster and historical input profile, retaining each named missing or +unknown-containing input and its declarations. Presence is not proof of +applicability, source validity or non-default statistical signal. An optional +`export_contract=` accepts the consuming adapter's actual `ExportContract` and +reports missing, forbidden, formula-owned and unexpected columns without +projecting or default-filling the Frame. No country engine is imported or run by +this handoff. + +The approved native scope excludes the legacy prior-year income family: +`employment_income_last_year`, `self_employment_income_last_year` and +`previous_year_income_available`. The handoff does not generate, fit or require +these as native completion targets. Existing columns, if any, remain preserved +for inspection. The [input-coverage diagnostic](us-input-coverage-diagnostic.md) +now offers an explicit `USInputProfile.NATIVE_NATIONAL_CD` name profile with +159 inputs: it excludes the two prior-year fields still in the 161-name +national/CD roster; prior-year wages were already absent. This resolves only +the name-profile selection. Native scientific/source gates and historical +release requirements still need separate reconciliation; the handoff continues +to name that outstanding gate. + +## Follow-on release integration + +This closes the live-owner to exact development-Frame/checkpoint seam. Release +admission is still unsupported. Its concrete successors are: + +1. Attach and qualify each missing input named by this candidate's report. The + native hours source/transport work, SPM construction/role path, and remaining + source families must become part of the receiving owner's checked lineage. +2. Bind applicability and source-signal gates to that owner, including the + usual-hours gate, and reconcile the prior-year-income scope. The current + report's `release_eligible=false` is descriptive, not an irrevocable decision + about a future qualified owner. +3. Supply the selected engine's input projection and country runtime evidence. + The full checkpoint deliberately preserves fields that may be forbidden or + formula-owned in an engine export; drop them only in the owned projection. +4. Pass the resulting checked Frame directly into maintained target + materialization and common-population calibration. Replace the native + admission branch with checks for that actual evidence; do not route through + legacy assembly/clone stages or mint their receipts. +5. Run the maintained Chronicle/model, national/CD target fit, export + readback/input-mass/target parity, reform smoke, and matched incumbent/default + improvement gates. The builder's existing exact-k arm additionally expects + its agreement and ladder provenance, which this handoff does not fabricate. + +Serialization proves none of those scientific or publication gates. This entry +writes no release manifest, certification or publication pointer. + +## Development base and verification + +This continuation starts from `5258a4c7f6d05cb1e68806d51b38996ea116bfb5` after +fetching and inspecting `origin/main` on September 19, 2026. Main lacked the +native `SurveyEnrichmentRun` API and associated owners; the branches diverged by +41 upstream commits and 407 continuation commits. Work occurs in a separate +worktree. The active native measurement source and harness are untouched. + +Lightweight invented tests cover full storage readback, missing-input and +engine-contract inventories, unissued-owner refusal, changed artifacts and +output reuse. The genuine-owner integration test reuses the existing enrichment +suite's actual issued owner; run it with the compatible continuation source, +not against a frozen active measurement tree. No native data or country model +calculation is needed by the lightweight suite. + +### Optional veterans’ benefits development route + +The retained native enrichment host accepts `groups=("veterans_benefits",)` with +`full_original_amount_donors=True`. Existing default groups and donor selection +remain unchanged. This route qualifies `VET_VAL` and `VET_YN` from the exact +retained 2025 ASEC person member for 2024 income. Positive “yes” answers and +explicit “no” plus zero are known; NIU, recipient zero, missing answers and +contradictions remain unknown. Age 15 is the lower reporting boundary, without +an upper-age, veteran-status or VA-healthcare filter. + +`I_VETYN` and `I_VETVAL` retain their literals, published codes, conditional flag +universes and allocation status. A zero flag outside its universe does not +establish an unallocated observation. Allocation provenance does not create +income zeros or discard otherwise known publisher-allocated amounts. The +six-digit veterans amount range does not change the five-digit UC/WC ranges. + +The model uses full-original current ASEC DESIGN donors with qualified amounts +and predictors, including donors absent from the selected receiving support. +It draws once per eligible original ACS person and carries that draw to both +existing clones. ASEC originals and their clones retain the qualified source +amount and unknownness. The ASEC2025/income2024-to-ACS2024 transfer is an explicit +development assumption; held-out quality, donor adequacy and release equivalence +remain unverified. + +When veterans is selected, `survey_amounts.canonical.version` keeps all receiving +support after prior enrichment, and `survey_amounts.canonical.attach` writes only +selected veterans and child-support canonical outputs. Child-only runs retain +their existing node IDs. An enabled canonical-state stage follows this version. +Carried canonical values establish no source authority: supported float32/float64 +storage is replaced from qualified source/model values with exact-loss checks; +unsupported storage and unrelated ownership collisions refuse. The parent source +branch, other columns, entity axes, memberships, weights and ledger are preserved. + +Invented-source and tiny real-fit/replay tests qualify this component boundary. +They do not establish full financial/PUF/enrichment-owner acceptance, a completed +native input profile, actual-source quality or publication eligibility. diff --git a/docs/us-native-verification-once.md b/docs/us-native-verification-once.md new file mode 100644 index 000000000..abd74366b --- /dev/null +++ b/docs/us-native-verification-once.md @@ -0,0 +1,373 @@ +# Verifying native sources once per run + +A native US build spends most of its time re-proving things it already proved. +The 2026-09-15 pilot measurement on the nineteen-node financial graph at +`Fraction(1, 1000)` (3,168 expanded households, 5,362 s wall) attributes +**1,143 s to node execution and the remaining ~79 % to admission and +verification of the 3.48 GiB of staged source** — repeated. QRF fits inside +that run cost 0.06–0.07 s each. The graph should compile to roughly the +pre-graph runtime plus receipts; today it compiles to the pre-graph runtime +plus the source tree, once per accessor use and once per executed node. + +This note records the five mechanisms that cause it, what each one guarantees +today, what changes, why the guarantee survives, and the new cost class. + +## The rule the whole change obeys + +> Memoisation changes **when** a check runs. It never changes **what** the +> check computes. + +Every digest, key, seal and receipt value in this repository keeps its exact +value across this change. Nothing in `_validate`, `_validate_state` or +`source_content_key` produces a receipt field — all three only compare live +values against values frozen at issuance — so moving a comparison in time +cannot move a recorded value. The one place bytes are genuinely rebuilt +(`_update_series`, mechanism 5) is proven byte-for-byte against a verbatim copy +of the pre-change implementation. + +Two guarantees are stated up front because every section below leans on them: + +1. **Verified before first use.** Nothing is memoised until it has been + verified in full, exactly as today. +2. **Proven unchanged at the end.** Every memo is re-verified in full, + unconditionally, before the run hands anything back. A source or a live + object that changed while a memo was warm still makes the run refuse. + +## The verification epoch + +Memoisation is **opt-in and scoped**. `microcosm.build.us_runtime.verification_epoch` +provides a context manager. Outside an epoch there is no memo at all and every +capsule behaves byte-for-byte as it does today — which is why every existing +capsule test, including the two that trace `_producer` and `_file_identity` +firing inside a validation, keeps passing unchanged. + +Inside an epoch each capsule validation is split into two tiers: + +* **Tier A — every access, never memoised.** The issuance binding proof, the + live final authority and the attached owner payloads, the producer encoding + (`_encode(_producer())` / `_encode(_implementation())`), and a *signature*: + the stat identity `(st_dev, st_ino, st_size, st_mtime_ns, st_ctime_ns)` of + every file the memoised tier reads, plus the structural witness of every live + object the memoised tier digests (per table: the column axis, dtypes, row + count, and for each column and index the buffer address, shape, strides and + writeable flag). The signature is *input*, never a verdict: no Tier A check + refuses because a stat identity or a witness moved. +* **Tier B — memoised on Tier A's signature.** Everything else: the source + catalogues, ACS native coverage, the nested ASEC native validation, the + whole-roster `_source_files` re-hash, the roster stat comparison itself, and + the pure final seals other than the two Tier A repeats — the nested owner + seals, the plan document digest and the three frame identities. + +A signature mismatch is **not** a refusal. It is a memo miss: the complete +unmemoised validation runs, and whatever refusal it would have raised today it +raises now, with the same code, at the same access. This is why the roster's +stat identities are read into the signature rather than compared in Tier A: a +cheap comparison that refused first would answer an appended ACS archive with +`SOURCE_STAT_CHANGED` where an unmemoised borrow answers it with the ACS +catalogue's own refusal. +`test_an_in_epoch_refusal_carries_the_code_it_carries_today` pins that by +refusing the same mutation twice, once inside an epoch and once with no memo at +all: an appended `selection-request.json` raises `SOURCE_CHANGED`, an appended +`acs/csv_pus.zip` and an appended `asec/pppub25.csv` raise +`PREPARATION_VERIFICATION_REFUSED` (a foreign owner's own refusal, translated +by `_checked`), and a touched roster file raises `SOURCE_STAT_CHANGED`. + +On leaving an epoch — and on leaving every nested epoch, so a run that ends +early through an inner scope is still covered — every capsule the epoch +validated is re-validated **in full, with the memo bypassed**, and the result +is recorded. A close records a new signature only when nothing moved while it +was validating: `_validate` compares the roster stats and then runs a whole +trailing `_pure_final`, and at an **inner** nested close the memo it records +survives into the outer epoch, so a signature taken after validating would +absorb a file that moved in that window and answer every outer borrow up to the +outermost close from the memo. Each close therefore takes the signature before +validating and again after, and when the two differ it records no memo answer +at all — the next borrow is a miss that pays the complete validation and +refuses with the code that validation raises. The entry itself is kept, because +the outermost close re-validates every capsule the memo still holds. +`test_a_roster_stat_moved_inside_an_inner_close_refuses_at_the_next_borrow` and +`test_a_native_source_moved_inside_an_inner_close_refuses_at_the_next_borrow` +pin both capsules by moving a source from a profile hook as that close's own +validation returns; against the previous ordering both fail with DID NOT RAISE. + +Leaving the refusal to the next borrow is the whole answer only while a borrow +can still follow. At the **outermost** close there is none: the memo is cleared +as that close returns, so a source that moved inside its window — including one +removed, after which no signature can be taken at all — would end the run +without refusing. The outermost close therefore pays that deferred validation +itself, in the same place and with the same error translation as the +unconditional pass above, whenever its own signature moved or could not be +taken. The refusal is the one that validation raises, never a code the close +invents, and a close that leaves a clean signature — which is what a close +leaves when nothing moved — runs nothing extra. +`test_a_roster_file_removed_inside_the_outermost_close_refuses_at_the_close` +and `test_a_native_source_removed_inside_the_outermost_close_refuses_at_the_close` +pin both capsules by removing a source from the same kind of hook, and assert +the close raises exactly what an unmemoised borrow raises for that removal: +`SOURCE_ROSTER` and `NATIVE_BINDING_REFUSAL`. + +`verification_epoch()` yields the close's record: the protocol label, the +capsule count, the number of memo hits, the number of signature misses and the +number of unconditional final re-validations. It is filled in as the epoch runs +and completed as the epoch closes; `epoch_record()` returns the innermost open +epoch's record, or `None` outside an epoch. + +Both native runners bind the record they open and hand it to `run_graph`, which +attaches it to the manifest as `RunManifest.verification_epoch` — outside the +manifest key, outside its JSON, and outside every node receipt and cache +record, exactly like `source_identities`. The epoch closes before the runner +returns, so the record a caller reads there is the closed one: +`test_a_real_run_records_its_epoch_in_the_manifest` asserts that an actual +nine-node run over invented sources reports memo hits, at least one signature +miss, and one unconditional final re-validation for every capsule it memoised. +`test_nineteen_node_financial_cold_and_required_replay` asserts the other +runner, on both manifests it returns — its own epoch's record on the outer +manifest and the nested population epoch's on the prefix's — for the cold run +and the required replay alike: the epoch's protocol label, one final +re-validation for every capsule memoised, and nothing of it in `to_json`. + +## The five mechanisms + +### 1. `AuthenticatedSurveyPopulationPreparation._checked()` + +`survey_population_preparation.py:1138-1162`. Every accessor — `.frame`, +`.context`, `.selection_plan`, `.receipt`, `.to_bytes()`, `.checked_view()`, +`validate()`, and the module-level `verify_survey_population_preparation` that +the graph registers as a per-node population observer — calls `_validate(state)`. + +**Guarantee today.** At the moment of every borrow: the capsule is the one that +was issued and its payload is unchanged; both source catalogues, the ACS native +coverage binding and the nested ASEC native population re-verify their own +source bytes; the ten-file source roster re-hashes to `state.files`; the +producer modules re-encode to `state.producer`; the roster's stat identities +re-derive to `state.file_stats`; and the pure seals (live authority, attached +evidence, nested seals, plan document, prepared frame identity, both source +frame seals) match. + +**Cost today.** One `_checked()` re-hashes the ACS archive pair fourteen times +(four in the ACS catalogue's source checks, eight across the native binding's +four snapshot dictionaries, two in `_source_files`) — about 11.1 GiB of +SHA-256 — plus three passes over the seven ASEC files, plus the whole roster. +It runs once per executed node through the population observer, plus once or +more inside every kernel that borrows the preparation. + +**Change.** `_validate` becomes Tier A + Tier B as above. Tier A keeps the live +authority, the attached owner payloads and `_encode(_producer())` as refusals on +every access, and reads `_file_stats(state.root)` into the signature on every +access, adding the paths that `_file_stats` does not cover but the I/O arm +reads: the ACS catalogue's `source_dir` entries, its two private snapshot copies +and its projection path, the native binding's four snapshot dictionaries, and +the `source_files` rosters of both nested ASEC capsules. Tier B is the rest, +including the comparison of those roster stat identities against the ones frozen +at issuance. + +**Why the guarantee survives.** Producer tampering and live-callable rebinding +still refuse at the same access (`_producer()` calls `_live()`, and it is never +memoised). Every on-disk change to any file the I/O arm reads — truncation, +append, in-place rewrite, replacement, rename, roster addition or removal, +symlink substitution — moves at least one of `st_size`, `st_mtime_ns` or +`st_ctime_ns`, or the directory's own stat identity, so it is a memo miss and +the existing refusal fires at the same access it fires today. Every structural +change to a live frame — a replaced column, a reindex, a dtype change, a +rebound buffer, a flipped writeable flag — moves the witness and is likewise a +memo miss. What is deferred to the epoch's unconditional final re-validation is +exactly two things: an in-place value mutation of a live buffer, and an on-disk +rewrite that leaves all five stat fields identical. On this machine the second +is not reachable without privilege: an unprivileged same-length in-place +rewrite followed by `os.utime` restores `st_size`, `st_ino` and `st_mtime_ns` +but not `st_ctime_ns`, and `setattrlist(ATTR_CMN_CHGTIME)` refuses with +`EPERM` (measured 2026-09-15, APFS). That is an observation about one +filesystem, not a guarantee, which is why the final re-validation is +unconditional rather than conditional. + +**New cost class.** Per accessor use: O(modules) producer encoding + O(roster) +`lstat` + O(columns) witness — microseconds. Per run: one full validation at +first use plus one at each epoch exit, instead of one per accessor use, plus +two signature passes per memoised capsule at each exit, which bracket that +exit's own validation and cost the same `lstat` walk a borrow pays. + +### 2. `AuthenticatedAsec2024NativePopulation.frame` + +`asec_2024_native_population.py:449-499`. `.frame`, `.context`, `.receipt`, +`.to_bytes()` and `validate()` all route through `_state()` → `_checked()` → +`_validate_state(state)`: every frame access is a full re-authentication, +including a full SHA-256 of all seven ASEC source files. + +**Guarantee today.** On every borrow: the implementation encoding matches, the +descendant and parent frame identities and the frame context match, the parent +capsule re-validates, the coverage/anchor/field owners re-verify, all seven +source files re-hash to their pinned digests with a before/after `fstat` guard, +and — because a file check or a foreign owner call may yield — the producer and +both frame identities are re-checked *after* the file loop as well as before. + +**Change.** Same two tiers. Tier A keeps `_encode(_implementation())` and adds +the stat identity of the seven `state.source_files` paths and of their parent +directories (which `_file_identity` does not cover: it opens `O_NOFOLLOW` on +the final component only). `_State` here is a `NamedTuple` and a test pins its +immutability, so the memo lives in a side table keyed by `id(owner)`, holding a +weak reference to the owner beside the signature and the state. An entry +answers only when that reference still resolves to the same object, dead +entries are dropped as the epoch closes, and the whole table is cleared when +the outermost epoch exits. + +The owner's own close runs after this capsule's. At the outermost close this +capsule's memo is already cleared, so the owner's final validation reaches the +complete file check; at an inner nested close the memo is still live with a +refreshed signature, so the owner's validation is a memo hit — unless a source +moved while this capsule's close was validating, in which case that close +records no signature and the owner's validation is a miss that re-runs the +complete check. Nothing is skipped in net either way, because this capsule's +exit has just re-validated it in full. + +**Why the guarantee survives.** As in mechanism 1. The documented +yield-tolerance pattern is preserved intact: a memo hit runs neither the file +loop nor the checks that bracket it, and a memo miss runs the whole of +`_validate_state` unchanged, so the bracketing pair is never split. + +**New cost class.** Per access: O(modules) + seven `lstat`s + O(columns) +witness. Per run: two full validations, each close's own bracketed by the two +signature passes described above. + +### 3. Per-node source content keys in the executor + +`executor.py:2452-2457` re-derives `source_content_key(name, source_paths[name])` +for every source a cold-executed node declares, and `keys._directory_identity` +(`keys.py:31-51`) `read_bytes()`es every file of a directory source and hashes +the whole tree. Cost is O(executed source-declaring nodes × source bytes): the +3.48 GiB tree is re-read per node. + +**Guarantee today.** The check runs after the kernel returns and **before** +`_validate_result`, `_apply_result`, the population observer and `_write_node`. +Its real stake is the last one: no artifact computed from mutated source bytes +is ever persisted under a node key that claims the pre-mutation bytes, which +would poison the content-addressed store permanently. + +**Change.** `run_graph` gains a per-run cache in its own locals — never inside +`source_content_key`, which stays pure because four existing assertions call it +twice on one path with the bytes changed in between and require different +answers. The cache is keyed by source name and a *stat signature*: for a +regular file, its `_stat_identity`; for a directory, the sorted recursive +roster of relative names with each member's `_stat_identity` and the stat +identity of every directory in the tree, plus the resolved identity of any +member that is a symlink — `_directory_identity` selects members with +`is_file()` and reads them with `read_bytes()`, both of which follow the link, +so the signature follows it too and the two sides walk the same members. The per-node check derives the +signature (an `lstat` walk, no byte reads); on a match it reuses the cached +key, on a mismatch it re-derives the content key in full and raises the same +`NodeRejected` if it moved. Immediately before the manifest is built, every +source is re-derived **in full, cache bypassed**, and a mismatch raises. The +run-end re-derivation is written inline rather than through +`_source_paths_and_keys`, because a build test counts calls to that exact code +object. + +**Why the guarantee survives.** The refusal still precedes `_write_node` for +every mutation that moves any stat field — which is every ordinary mutation, +including one that adds, removes or renames a file inside a directory source, +since the roster is part of the signature, and including a change to the bytes +behind a member symlink, whose own five stat fields never move when its target +is rewritten. The narrowed case — a byte rewrite +that preserves all five stat fields — is caught at run end, before any caller +receives a manifest, but after intervening nodes have written store records. +That residual is stated rather than hidden: a run refused by the run-end check +may leave store entries behind, and those entries are self-consistent under +`_require_record_shape`, so the operator must clear the store for that run. +Against this, the executor gains a guarantee it does not have today: sources +are re-verified at run end even when the mutation happens during a node that +declares no source, which today is invisible. + +**Record.** `RunManifest` gains `source_identities`, an attached mapping modelled +exactly on `populations` and `mass_ledgers` — `repr=False`, `compare=False`, +excluded from `content_addressed` and absent from `to_json`. It carries the +run-end re-derived key of every source. It moves no manifest key, no manifest +JSON byte, no node receipt and no cache record. + +**New cost class.** Per node: an `lstat` walk of the source tree. Per run: two +full content derivations instead of one per executed source-declaring node. + +### 4. The ACS record fence + +`acs_person_coverage_authentication._records` fences raw logical CSV records +out of a zip member before any UTF-8 decoding or CSV allocation. It is a +pure-Python `for byte in block` loop over 4,096-byte blocks, and the member it +fences is the ACS person CSV inside `csv_pus.zip` — 2.4 GB expanded across +`psam_pusa.csv` and `psam_pusb.csv`. With `peek` and +`acs_person_coverage_columns.lines` it is 57–65 % of the admission phase in +every measured run. + +**Guarantee today.** For every byte, in this order: the record length must be +below its cap (64 KiB for the first record, 400 KB afterwards) or +`CSV_RECORD_BYTES`; an unquoted comma resets the token counter, any other byte +advances it and must stay within 64 KiB or `CSV_TOKEN_BYTES`; a double quote +toggles quote parity, which **persists across records**; an unquoted LF ends a +record, an unquoted CR ends it and absorbs a following LF. Record boundaries +are located by quote parity only; CSV validity remains the unchanged strict +literal parser's job. + +**Change.** The same state machine, driven by `bytes.find` and `bytes.count` +over 1 MiB blocks instead of a Python loop over individual bytes. Terminator +cursors are cached per block so an absent carriage return is searched for once +rather than once per record. Every completed record, and every in-flight record +at a block boundary, passes through a bound check: a record no longer than +64 KiB cannot have violated either cap, because the token counter resets at +each record start and so never exceeds the record's own length, and the record +cap is never below 64 KiB. Any record longer than that — which real ACS data +never produces — is replayed byte by byte through the original fence, so the +refusal code and the point at which it fires are exactly the original's. + +**Why the guarantee survives.** The yielded record sequence and the refusal +raised are proven identical, not argued: `test_us_acs_record_fence_scan.py` +compares the shipped `_records` against a verbatim copy of the pre-change loop +over 4,200 randomised byte strings, every boundary case (bare CR, CRLF split +across a block boundary, quoted terminators, odd quote parity carried across +records, BOM, empty lines, trailing CR at EOF) and every bound case on both +sides of every cap. The same comparison runs against the real staged +`csv_pus.zip`. + +**New cost class.** Linear in member bytes at C speed: measured 10.8 MB/s → +522.8 MB/s on the real archive, digests identical, a 48.5× reduction. + +**Pin.** `acs_person_coverage_authentication.py` is one of two modules whose raw +file SHA-256 is pinned in `acs_native_coverage_binding._ACCEPTED`, so this +change re-pins it. The new value is derived by `shasum -a 256` on the changed +file and recorded in the lane report. + +### 5. The kernel context digest + +`_context_digest` (`executor.py`) digests every column of every table twice per +node — once before the kernel runs and once after, to prove the kernel did not +mutate its input context. `_update_series` took a fast path only for masked +extension arrays exposing `_data` and `_mask` ndarrays. On pandas 3.0.3 an +ordinary float, integer or boolean column is a `NumpyExtensionArray` with +neither, so it fell through to `to_numpy(dtype=object)` and a per-value Python +loop with two `hashlib` updates per value. + +**Guarantee today.** The digest is a total function of the column: its dtype +string, the object projection's dtype and shape headers, and one framed payload +per value. + +**Change.** `_object_stream` builds exactly those bytes with numpy for the three +kinds whose boxed value is always one exact Python type — `float` packed to +eight native IEEE-754 bytes, `bool` rendered as `b"b1"`/`b"b0"`, and `int` +rendered as a variable-width decimal. It returns `None` for everything else, +including a `Categorical`, whose values array is an integer-coded view and +would otherwise be mistaken for an integer column. Both `_context_digest` calls +remain: the second one is the mutation guard, and reusing its result would +delete the guard. + +**Why the guarantee survives.** +`test_graph_executor_series_stream.py` compares the shipped helper against a +verbatim copy of the pre-change body **at the byte level**, not only at the +digest, across every column kind the projection can carry, the float specials +(signed zero, both infinities, the subnormal extremes), non-canonical NaN +payloads with their sign bit, both `int64` endpoints, empty, sliced and +strided columns, labelled indexes, and a 200-frame random sweep. + +**New cost class.** Measured 0.47 s → 0.09 s for a 6,928 × 240 frame; 183 → +18 ns per value for floats, 226 → 14 for booleans, 383 → 119 for integers. +Still linear in rows, but with no Python-level work per value. + +## What this note does not claim + +Nothing here is a statement about dataset quality, calibration or release +eligibility. The PR-CI / certification boundary is unchanged: these are code +contracts, and green checks do not certify any data artifact. diff --git a/docs/us-property-completion-routing.md b/docs/us-property-completion-routing.md new file mode 100644 index 000000000..debe0ee45 --- /dev/null +++ b/docs/us-property-completion-routing.md @@ -0,0 +1,82 @@ +# Property completion routing + +`current_property_completion_routing.build_property_completion_routing(qualified, clone_frame)` describes the remaining ordinary-interest and dividend work over the original ACS/ASEC source population. It assigns no amounts, fits no model and changes no Frame. Its `PropertyCompletionRouting` return value is descriptive; constructing it grants no source, complete-parent or release authority. + +The inputs are the existing `QualifiedPropertyIncomeSources` description and the complete initial clone Frame. The operation checks original source order, integer identities, source-native agreement, source ages, component knownness, the exact two clone roles and original household DESIGN weights. It preserves the complete input physical descriptions. Actual source custody remains external: a country host must retain and requalify the preparation and source issuers around relevant I/O. Neither the description's type nor a separately supplied DESIGN vector authenticates that custody. + +## Routes and retained reasons + +Each original person receives one route: + +| Route | Meaning | +| --- | --- | +| `unsupported_under15_measurement` | The source age is below15. No child amount or adult extrapolation is supplied. | +| `source_review_required` | A required source amount/receipt or ACS anchor is malformed, outside its published domain, inconsistent or has an invalid adjustment. | +| `carry_known_components` | Both independent ASEC components are known. Joint-fit exclusion does not erase them. | +| `existing_acs_anchor_decomposition` | The adult ACS aggregate anchor is observed and qualifies for the existing decomposition branch. The separate components remain unresolved at this stage. | +| `acs_anchor_completion_review` | An adult ACS anchor remains missing. This operation supplies no fallback. | +| `asec_component_completion_review` | At least one adult ASEC component remains unresolved. Any independently known other component is preserved. | + +Under15 is the first route; malformed, positive-outside-universe and contradictory observations remain visible as overlapping reasons. Adult source errors precede generic missingness. Joint-donor exclusions remain separately named reasons and never by themselves turn otherwise known O/D into unknowns. Unknown new source statuses refuse classification instead of silently entering a generic completion branch. + +The long component table preserves amount, individual knownness and the exact reporting/literal statuses. `observed` denotes a qualified observed amount; `derived` distinguishes a qualified known-nonreceipt zero; `unresolved` retains missing amounts. An observed ACS aggregate is recorded separately from its unresolved individual components. No component receives a `modeled` origin here; only a later operation consuming actual checked model outputs can assign that origin. Allocation and disclosure flags remain source provenance rather than donor filters. + +## Outputs and support + +- `person`: one row per original, with source/native identity, source age, original household, independent O/D knownness, separate ACS anchor and exclusive route. +- `components`: two rows per original for the two required component families. +- `reasons`: all overlapping required-source and joint-fit diagnostics. +- `clones`: exact original/native/source identity and clone roles mapped to the initial receiving person IDs. +- `summary`: raw original-person and unique-household counts, household-inherited DESIGN person mass and union-household DESIGN mass for each route/reason. Clones do not multiply these summaries. Zero-weight support remains visible in raw counts. +- `payload`: deterministic JSON with the tables and explicit statements that it neither assigns amounts nor authenticates a source or complete parent. Integer IDs remain integers, including those above2⁵³; unknown amounts serialize as null. + +These are support diagnostics, not prevalence estimates or calibrated totals. Overlapping reason masses must not be added as if the reasons were disjoint. + +## Integration boundary + +`property_completion_artifact_output()` supplies the typed `completion_routing` artifact declaration. This additive slice does not connect a new node or change the accepted38-node host. The next host integration should emit the payload at the existing property source-projection boundary, bind the actual retained source and clone descriptions, and recompute/compare it during requalification and reconstruction. That host change requires its own source/graph identity and observation checks. + +This operation does not create a child-income measurement model, fill an unknown adult input, assert that the PUF covers a missing source population or relax the existing completeness gate. + +## Opt-in host artifact + +`PropertyIncomeOptions(completion_routing=True, ...)` requests a fourth typed +artifact, `completion_routing`, from the existing +`survey_property.source_projection` node. The default remains disabled and +retains the previous canonical option payload. Neither setting changes the +numerical model declarations, source universes, clone attachment or tax split. +An implementation source edit does change executable cache identities. + +The private artifact contains the complete original-person, component, reason +and initial-clone tables. The source receipt contains an explicitly allowlisted +aggregate summary and its private artifact digest. The actual graph HTML and +text views expose receipts, so private identifiers and per-person values must +never enter this summary. Aggregate DESIGN support is descriptive; it is not +calibrated representation, a release gate or a disclosure certificate. + +Source execution reuses its existing qualification and builds routing once. +Independent host reconstruction checks exact diagnostic bytes and the expected +aggregate receipt on cold execution and required replay. Ordinary branch checks +retain their existing three source-artifact checks and do not recompute routing. +The host retains all declared artifact hashes and canonical options; its checked +parent/source lifetime remains the authority boundary. The pure routing result +and the artifact grant no source or complete-parent authority. + +The bounded integration controls exercise qualified invented source descriptions, +option/declaration behavior, tamper refusal and the real public serializers. +Actual 38-node host acceptance is a separate test scope; this feature does not +claim native completion, PUF readiness or a released dataset. + +The repository's existing 38-node tax-host fixture now enables completion +routing on both its cold and required calls. Its controls retain full population +and unknown-input checks, verify the private typed artifact and aggregate receipt +on replay, and refuse changes to the retained artifact bytes. The local native +harness predicates can share those fixtures through a separately pinned acceptance +wrapper; they are not a machine-specific dependency of repository tests. + +The bounded acceptance passed 27 focused controls and three actual invented +cold/required host tests at source `c06d29ea8`. The latter reuse one 38-node pair, +check the full saved Frame after final source revalidation, and reject changed +retained artifact bytes. They preserve the fixture's false tax-input completeness +verdict. Exact scope and receipt hashes are in the +[13 September integration record](../experiments/us-launch-consolidation-20260913.md). diff --git a/docs/us-property-income-graph.md b/docs/us-property-income-graph.md new file mode 100644 index 000000000..d06b4c869 --- /dev/null +++ b/docs/us-property-income-graph.md @@ -0,0 +1,58 @@ +# Property income model graph + +`microcosm.build.us_runtime.graph_property_income.property_income_nodes` +declares ten numerical nodes: four conditional QRF fits, four chained draws, +one operation that writes the raw draws as columns, and one signed +reconciliation. It uses the existing QRF and reconciliation implementations. +The fragment consumes separately supplied original donor and recipient +populations; it does not qualify native sources or attach results to clones. + +The shared columns in `property_income_constants.py` describe ordinary +interest, retirement-account interest, dividends and signed broad property +receipts. The first three are nonnegative; broad property receipts may be +negative. `property_reported_total` is a required predictor and the recipient +reconciliation anchor. The source bridge supplies the ASEC reported aggregate +for donors and the adjusted ACS INTP amount for recipients. It must preserve +their different reference periods and account for unresolved income routes. + +The training wrapper requires finite float64 predictors and components, +original DESIGN weights, and a component sum equal to the supplied donor +aggregate. It delegates actual fitting to `LegacyQRFTrainKernel`; it does not +filter, repair or reconcile donor observations. The owning source preparation +must establish original household membership, design-weight mapping, eligible +donor support and source knownness before invoking this fragment. A weight +kind label alone does not establish original source authority. + +Recipient draws preserve the complete target order and earlier raw draws as +conditioning inputs. The column operation binds every raw value to its apply +checkpoint, exact recipient index, sibling producer, and final model history. +The graph executor supplies the authenticated artifact edges. Direct calls or +copied checkpoint bytes do not authorize source or model admission. Person IDs +remain int64 independently of the pandas index used by QRF checkpoints. + +Reconciliation retains raw draws in separate `draw_` columns and exposes each +component's adjustment, active bound, residual and objective. Scales and +tolerances are required parameters. Changing them recomputes reconciliation; +changing recipient anchors recomputes recipient draws while reusing donor fits. +Negative or zero net income never forces all positive components to zero: +positive interest and a property loss can coexist at either total. + +Missing anchors and under-15 source blanks require an explicit upstream +decision; this fragment does not manufacture zeros. Original ASEC observations +are not reconciled to ACS respondents. The next country integration must join +the complete source-qualified basis, draw once per original ACS person, retain +the result on both initial clones, and derive the declared tax splits before +running the dependent PUF successor. Retirement-account earnings are not +ordinary taxable/exempt interest. Broad property receipts are not yet a +rental-income tax leaf. PUF-owned alternatives must not subsequently be forced +back onto the survey anchor. + +The initial invented test uses 44 donor persons and four recipients, including +negative/zero totals, large int64 identities, original membership, metadata and +a zero-weight household. It executes the real twelve-node graph including two +source nodes, compares draws with direct weighted QRF, compares reconciled +values with the pure projection and reopens the store in required-cache mode. +Twenty-two tests also cover delegated training-code identity, dependency reuse, incorrect weight kinds, +unknown/inconsistent donors, altered draw history and invalid declarations. +They establish numerical and graph contracts, not native source acceptance, +statistical adequacy, tax correctness or release readiness. diff --git a/docs/us-property-model-receipts.md b/docs/us-property-model-receipts.md new file mode 100644 index 000000000..1c16e9618 --- /dev/null +++ b/docs/us-property-model-receipts.md @@ -0,0 +1,65 @@ +# Property model receipt verification + +`verify_property_model_receipts(nodes, donor_population, recipient_population, +artifacts)` in `graph_property_income_receipts.py` reconstructs the existing +four fit and four apply receipt dictionaries. The caller passes the ordered +property fragment, the two original `Population` branches, and authenticated +artifact bytes keyed by `(node_id, output_name)`. Additional non-model fragment +nodes and their artifacts may be present; the country host verifies those. + +The caller must first bind the store bytes, typed producer outputs and exact +population dependencies. This helper loads the existing trusted model format, +which contains pickle. A self-consistent digest or a returned receipt does not +authenticate arbitrary model bytes or establish source authority. The owning +host retains responsibility for source qualification, complete population +identity, physical seals, borrowed inputs and final I/O checks. + +## Checks + +The helper reconstructs the shared graph declarations for the four ordered +property targets and compares their complete normative bytes. This binds input +slices, population versions, predictors, target order, seed, phase, fit options, +model/training sibling edges, prior raw-draw edges and output artifact types. +Training stays donor-only. The existing model protocol initializes its exact +chain-start state against the original design-weight donor without fitting. +Each trusted model must match the complete ordered training history, actual +consumed float64 donor bytes, donor pandas index, resolved weights, configuration +and preceding/succeeding training states. The first three property components +remain nonnegative and the four-component sum must equal the observed donor +aggregate. Both branches require finite float64 model columns and unique int64 +person IDs. + +Ordinary population apply checkpoints have a pandas-index identity but no +independent recipient-feature digest. The verifier therefore performs four +additional `apply_target` operations using the retained fitted models, current +recipient features, raw prefix and exact initial draw RNG. It compares the +resulting raw bytes and complete application states with the published +artifacts. It validates every model/raw hash in the ordered application history. +It never calls `fit_target` or a model-fitting convenience method. + +This replay establishes output consistency with the supplied feature values; +a model can produce the same output for different inputs. Exact feature and +person-ID provenance remains the caller's authenticated population binding. +The existing protocol uses pandas row labels for model state and raw draws; +the verifier does not substitute entity IDs or invent a new identity digest. +Graph owners must continue to bind the distinct int64 person axis themselves. + +## Scope and cost + +The return value uses the existing raw kernel receipt schema; the host adds and +checks the executor capability/input-writer envelope through its existing +`_states` path. It adds no status, +source issuer, receipt artifact type, model format or host framework. The +helper does not fit, reconcile, attach clones, assign tax treatment or modify +its input frames. Cost per call is one chain initialization, four trusted model +decodes and four deterministic apply replays over the recipient branch. The +owning host decides when to perform this validation; no native-scale runtime +claim follows from the small tests. + +Tests reuse the actual twelve-node property graph fixture with two trees, +44 invented donor persons, four recipient persons, non-default pandas row +labels, int64 IDs above `2**53` and zero-weight records. Original cold and +required-cache kernel receipts match the verifier. Separate controls alter +donor/recipient axes and values, design weights, graph declarations, model and +training bytes, model/raw history and draw RNG. Country engines, native sources, +the full PUF host and publication are outside these checks. diff --git a/docs/us-property-tax-leaves.md b/docs/us-property-tax-leaves.md new file mode 100644 index 000000000..afc520911 --- /dev/null +++ b/docs/us-property-tax-leaves.md @@ -0,0 +1,124 @@ +# Property tax leaf rebase + +`graph_property_tax_leaves.py` provides three deterministic graph nodes for an +already checked population with property components and incumbent tax leaves. +It does not qualify sources, fit models, attach clones, grant a checked-run +handle or wire the country/PUF host. The owning host verifies the original +component observations, clone pairing and complete frame before using it. + +## Numerical operation + +`split_property_tax_leaves(person)` returns four float64 columns indexed by exact +int64 `person_id`. It imports the maintained fractions from `cps_carried`: + +| Input component | Primary leaf | Complement leaf | +| --- | --- | --- | +| `property_ordinary_interest` (`O`) | `taxable_interest_income = O * 0.680` | `tax_exempt_interest_income = O - taxable_interest_income` | +| `property_dividends` (`D`) | `qualified_dividend_income = D * 0.448` | `non_qualified_dividend_income = D - qualified_dividend_income` | + +The complement uses subtraction after computing the primary. This is a declared +operation order, including for the dividend complement that legacy code computes +by multiplying the complementary fraction. These fractions are modelling +assumptions; they do not establish a person's observed tax treatment. + +Interest and dividend knownness are separate. A finite nonnegative input produces +its two leaves; a missing input produces two NaNs, replacing any earlier tax +predictions. Zero stays an observed numerical zero. A negative or infinite O/D +input, an infinite retirement component, incompatible dtype or ambiguous person +identity refuses. `property_retirement_interest` is retained as an auxiliary +component and never enters either tax-interest leaf. Its missing value does not +invalidate known O/D values. Broad property receipts, net anchor sign, source age, +donor-fit eligibility and earlier predictions never supply missing O/D values. +No under-15 zero completion or missing-adult prediction occurs here. + +## Three graph nodes + +Call `property_tax_leaf_nodes(frame, population=base_version, +projection=projection_input, reconciliation=reconciliation_input, atol=..., +rtol=...)`. Both tolerances must be explicit finite nonnegative Python floats. +The two `ArtifactInput` aliases must be `projection` and `reconciliation`; the +host supplies their actual producer IDs and nominal types. Register +`PropertyTaxReceivingKernel`, `PropertyTaxLeavesKernel` and +`PropertyTaxLeafGateKernel`. + +When compiling before the earlier financial/property nodes execute, the host +may pass `anticipated_outputs=tuple_of_owned_descriptors`. The factory combines +those existing declarations with the real checked Frame's entity/axis/group +shapes and dtypes. It creates no placeholder values or synthetic population. +It rejects duplicate descriptors, unknown entities, structural ID fields, +masked outputs and incompatible or unclaimed rewrites. At execution the host +re-declares the nodes from the actual complete post-property Frame and requires +the same normative declarations before reconstructing the results. The default +empty descriptor tuple still requires a complete actual input Frame. + +1. `survey_property.tax_receiving` opens a new population version using + FILTER-all. This inherits the prior owners and permits explicit replacement of + the four incumbent leaves. The executor adds a real FILTER conservation record + to the population mass ledger. It preserves the existing `Frame.mass_log`. +2. `survey_property.tax_leaves` replaces the four leaves on that version and + emits `diagnostics` of type `microcosm.us.property_tax_leaf_diagnostics/v1`. +3. `survey_property.tax_leaf_gate` recomputes the leaves and their knownness, + checks exact leaf bits, partition sums within the declared tolerances, and the + complete diagnostics payload. It emits `verification` of type + `microcosm.us.property_tax_leaf_verification/v1`. + +The gate binds ordered person IDs without floating-point conversion, current +O/D/retirement bytes, all four leaf byte digests, fractions and operation order, +individual knownness, and the typed projection/reconciliation artifact keys and +payload hashes. It includes every physical person, including dependents and +zero-weight members. `complete` is false if any person lacks O or D; a successful +numerical check with missing inputs reports `evidence_absent`. Complete numerical +inputs report `pass`. Neither outcome establishes source, model, tax-validity or +release authority. The artifact explicitly leaves full-frame verification to +the host. + +## Supported receiving shapes and host responsibilities + +`Frame.select(all)` currently refuses link tables and normalizes group indices. +The declaration factory therefore refuses links, nondefault group indices, +orphan groups, empty person frames and groups lacking a readable non-ID column. +The last condition ensures the kernel context exposes group IDs for the +receiving check. Orphans are already invalid at Frame construction; the factory +also detects a later membership mutation. The kernel rechecks exposed group +membership and logical group indices immediately before returning its keep mask. +It never silently prunes a group or switches structural operation. + +For supported frames, person rows/order/IDs, entity tables, weights and their +kinds, schema, strata, metadata and frame mass log are preserved, apart from the +four declared leaf replacements. The new population version, owners and +conservation record are intentional changes. Kernel contexts do not expose the +complete schema, metadata or population ledger, so the host must retain and +verify those independently. It must also qualify the complete supported shape +before adding these nodes, bind actual artifact producers and stored payloads, +and recheck the original owners and output after its final relevant I/O. + +For independent expected-result reconstruction, the host can call each kernel's +`run(KernelContext)` on the retained current population and authenticated typed +artifact aliases. The existing executor `_project_context` and `_apply_result` +provide the canonical context projection and result application. FILTER results +must be materialized through the validated keep mask before `population.patch`; +passing its raw `keep` result directly to that lower-level patch function omits +the executor's frame selection. Ordinary rebase/gate results use the existing +patch path. These deterministic kernels consume no random draws. Expected +artifact bytes and receipts can then be compared with the actual store and node +states, and expected full populations with the host's retained observations. +This fragment adds no separate executor, authority registry or public run issuer. + +Future PUF integration must depend on this verification artifact and require +complete tax leaves over every physical member the PUF consumer needs. The +fragment does not add that host dependency itself. Matching checked inputs give +matching outputs on paired clones; the fragment does not infer or repair clone +pairing. Capital gains retain their earlier conditioning and are not rebased by +this change. + +## Verification scope + +The 32 invented controls exercise a real four-node graph including its source, +ContentStore cold execution and required-cache replay. They cover negative, +zero-net and positive anchors with loss offsets, independent O/D unknowns, +known O/D with unknown retirement/broad components, integer IDs above `2**53`, +row permutations, empty pure input, malformed values, altered leaf bits, +diagnostics and artifact bindings, full-frame preservation, zero-weight records, +the population conservation record and unsupported receiving shapes. No native +survey, country engine, PUF execution, fitting or publication is part of this +acceptance. diff --git a/docs/us-puf55-original-development-placement.md b/docs/us-puf55-original-development-placement.md new file mode 100644 index 000000000..e3e33833e --- /dev/null +++ b/docs/us-puf55-original-development-placement.md @@ -0,0 +1,90 @@ +# Original-arm PUF55 development placement + +The survey enrichment host exposes optional original-arm application through +`run_us_survey_enrichment(..., original_application_seed=...)`. Omission preserves +the existing path. An explicit integer seed must differ from the retained +clone-one seed. The original arm reuses the genuine route fits and runs every +selected profile target with that application seed; it does not refit the donor. +This development path does not qualify a release. + +The selected PUF55 profiles contain **52 person outputs and 3 tax-unit outputs**. +The six SCF mortgage-detail fields in the full65 compatibility roster are not +PUF55 targets. The person outputs form three disjoint categories: + +- Five possible new singleton values: `long_term_capital_gains_on_collectibles`, + `non_sch_d_capital_gains`, `partnership_income`, `s_corp_income`, and + `partnership_self_employment_net_earnings`. +- Twelve qualified fixed values remain unchanged. Eight are survey-preserving + financial leaves; four use the explicitly recorded development assumptions + for pension taxability, regular IRA routing, rental proxy, and farm amount. + These are not newly claimed observed taxable amounts. +- The other 35 person outputs receive no new placement in this slice. This is + not a claim that all 35 lack existing values. + +The three unit outputs are `domestic_production_ald`, +`unrecaptured_section_1250_gain`, and `health_savings_account_ald`. + +`puf55_original_placement.placement_result` returns detached full-axis columns +and a descriptive receipt. Its checks require exact ordered entity IDs and +complete memberships, all twelve fixed values known for every original member, +unchanged known person values, and exact fixed conditioning totals. Mixed-known +units remain unresolved. Singleton means exactly one member in the complete +retained person table, not one eligible member or a capped source return size. + +A candidate output must be absent in the genuine pre-PUF canonical parent and +introduced by the checked arm-one attachment. Its current owner must still be +that attachment, with original-arm nulls intact. A preexisting column is left +alone even if null. Nullness is an integrity check on authenticated ownership; +it is not source authority, a source NIU-to-zero rule, or legal applicability. +The new values are explicitly modeled development transport. Source knownness +and that transport's scientific acceptance remain distinct. + +The pure helper preserves inherited dtypes and requires exact representability +of each new value. Signed outputs are not clipped. A draw outside the maintained +nonnegative domain remains unresolved for that output; no clipping, alignment, +tail cap, multi-person allocation, QBI-support placement, or tuition allocation +is performed. Blocked cells, clone-one values, all other columns and weights are +preserved. Per-output write counts and reasons are descriptive measurements. + +`graph_puf55_original_placement.original_placement_nodes` declares a keep-all +version after the supplied current terminal, then explicit full-column rewrite +ownership for the candidate outputs. The attachment consumes every route's +55 model/training/raw-draw/conditioning/v2-state dependencies, plus qualified +fixed inputs and the exact original recipient matrix. Its result helper invokes +the existing strict complete conditioning merger. There is no fabricated +completed prefix or finalizer-only restoration of survey values. + +The existing enrichment owner admits and rechecks the genuine financial and +PUF owners and qualifies the original fixed inputs. Its private original-arm +binding is not a new issuer. Declarations use predecessor metadata, while the +executor observer supplies the actual completed receiving Population. The +central terminal selector follows state, canonical child-support/veterans, +race, sex, immigration, SPM, then hours according to the enabled fragments. +The receiving version comes from that selected node, including later versions; +it is not assumed to be the earlier PUF version. + +The binding compares declared kernel contexts with the executor's projection of +that complete terminal, including typed weights and strata. The existing host +checks every artifact against actual compiled producer keys. It reconstructs +all preceding branches, compares the actual terminal, rebuilds the keep-all +and placement, compares persisted attachment bytes and complete Populations, +and retains its source requalification, live-code and final storage checks. +A caller-supplied digest, a descriptive placement receipt, or the private +`_capture_checked_puf_ancestors` helper alone cannot admit a receiving run. + +Whole-host qualification still requires all real route fits/applications and a +required replay on admitted source owners. The bounded integration verification +is staged: actual three-target training/application/replay, the complete 110 +application-node declarations for two profiles with explicitly synthetic +metadata suppliers, and invented terminal/context/placement preservation tests. +The declaration check is not evidence of 110 fitted or executed applications. +The genuine full51 parent previously exceeded the bounded test's CPU budget by +itself, so that parent and a whole-host run are reserved for a separately +admitted integration run. Applicability, coverage, and the development +assumptions also need the eventual no-worse comparison before release. + +The local tests use invented Frames for eligibility and patch arithmetic. A +separate complete-envelope test uses synthetic 55-target payloads derived from +an actual three-target test template solely to exercise the strict codec path. +It is not evidence of 55 fitted models, a genuine financial owner, a complete +host run, or a calibrated dataset. diff --git a/docs/us-survey-completion-host.md b/docs/us-survey-completion-host.md new file mode 100644 index 000000000..432662161 --- /dev/null +++ b/docs/us-survey-completion-host.md @@ -0,0 +1,139 @@ +# Optional US survey completion host + +This source implementation adds opt-in completion to +`run_atomic_survey_financial`. It has not been imported, collected or executed +in this task. Component acceptance does not establish combined-host acceptance. + +Select an exact `ChildPropertyOptions` with explicit scenario ID, scope, +`SupportRequirements`, `JointTransport` and keyed stream. Also select +`property_income` and `rebase_property_taxes=True`. `household_roles=True` +requires child completion; its default is `False`. Existing `person_status` +remains independent. Unsupported combinations refuse before source execution. +There are no scientific defaults or new public issuers. The result remains an +existing-issued `AtomicSurveyFinancialRunValues`. + +The host first issues and retains the actual property parent (35 nodes, or 39 +with status), with tax rebasing disabled. It retains its preparation, source +objects, original legacy/property populations, manifests, store, keys and kernel +registry. Registries are copied before extension. The union executes in the same +store and requires every already-issued parent node to hit cache. + +| Selected path | Union nodes | +| --- | ---: | +| Existing financial / property / property plus tax | 19 / 35 / 38 | +| Existing status variants | 23 / 39 / 42 | +| Property, child, tax | 45 | +| Property, status, child, tax | 49 | +| Property, status, roles, child, tax | 51 | + +A full-row completion FILTER creates a distinct version. Qualified household +roles optionally add two nodes after incumbent reconciliation. Child completion +adds six nodes, including the generic empirical fit and draw. A separate tax +FILTER, split and gate follow child verification. Tax declarations use the +independently reconstructed completed frame, including its actual role/status +and imputation columns. All three tax nodes bind the actual typed child +verification artifact. The country child verifier owns canonical child evidence; +the tax seam binds its payload hash. + +Before compilation, the helper binds exact ordered node identities for receiving1, +optional roles0/2, child6 and tax3. Compensating child/tax count drift refuses even +if the union total is unchanged. The portable child count comes from retained +nodes. Its source-knownness flag comes from the independently reconstructed +child verification bytes that the materialized verifier checks against actual +store evidence; the document retains that same payload's SHA-256. + +The private helper independently applies the maintained receiving FILTER and +role reconstruction before constructing `ChildPropertyBoundary`. Its parent +callback returns that same retained Population after pure checks of the issued +base, source ownership, executable closure and complete receiving/role stamps. +Full base/source and role qualification occurs at host boundaries. Ordinary +child borrows do not repeat role parsing. The child fragment itself still +requalifies its source on each `_current()` call. + +Both cold execution and required replay load artifacts through the actual +compiled producer, manifest, store, keys and numeric scope. Both invoke the real +child materialized verifier with actual executor-completed output, both actual +support Populations, every actual verification input and the actual verification +output. Required hits skip kernels, so that final verifier is mandatory. Expected +full-population states, receipts, owners, weights and ledgers are compared using +the maintained graph state checker. Final source/store I/O precedes pure seals. +A failed enabled-host check permanently revokes its completion boundary. + +Original property verification always receives its original property output. +Status preservation follows the actual FILTER lineage; structural operations +legitimately change inherited owners. Qualified survey reference persons confer +no tax/SPM role, filing status or eligibility. Existing property completion +routing is labeled `pre_child_source_gaps`; those counts are not post-child +completion counts. Public completion metadata is aggregate only. + +Only qualified jointly unmeasured eligible children receive paired modeled O/D, +using full original ASEC teenage donors and original household design weights. +Source observations and all nonrecipient/complement cells remain unchanged. +Modeled zero remains `imputed_zero`, distinct from observation. Tax completeness +requires finite nonnegative ordinary interest and dividends for every physical +person, including dependents and zero-weight rows. Unknown adults remain +incomplete and the maintained PUF admission gate must refuse before fitting. + +The authored finite tests include separately selectable cold/required 49-node +(roles disabled, status enabled) and 45-node (roles/status disabled) pairs, one +shared cold/required 51-node fixture, and one incomplete-adult fixture. The positive +fixture composes status/role/money literals and attachments before issuance, +keeps its teenage donor outside the selected sample, and includes housing roles +only. It does not erase unsupported GQ incumbents. It covers exact declarations, +paired draws, unchanged complement/weights/geography, real tax completeness, +detached custody, altered artifacts/identity and final-I/O revocation. The 45/49 +pairs explicitly select the partial ACS+ASEC fixture, preserve incumbent role +columns without role writes, call actual checked views, and assert count, +declaration/manifest/payload parity and required hits. They do not parameterize +the 51-node adverse suite. All 16 functions / 32 source-expanded cases are UNRUN, +including those fixture-composition assumptions. + +Lineage assertions cover all nine prefix nodes and actual financial, property +and enabled status writers before receiving. Both FILTERs must preserve entity +dimensions, tables, strata and weights, append one ledger record with +`operation="filter"` and `policy="conserve"`, and conserve total, stratum and +partition mass. Assertions follow actual structural owner/version transitions +through role/child attachment and tax-leaf writes. FILTER uses `Frame.select`, +whose Frame constructor copies tables; role column patching additionally uses +explicit deep table copies. The parent-mutation adversary checks the retained +base stamp immediately after mutation and retains permanent child revocation. + +The 49-node control also replaces a retained completed Population with equal +values under a different identity and checks refusal and permanent revocation. +Other controls reuse real observed 51-node evidence to challenge a base cache +miss (which does not change manifest content identity), missing observations, +and final source/key/implementation/artifact identities. Final identity controls +re-read the authentic store and sources without executing kernels. The child +artifact and verification-identity adversaries can reconstruct/refit; they are +not fit-free checks. Source-construction adapters are restored before issuance +to prevent later fixture composition from wrapping an earlier fixture's closure. + +Remaining review scope: T4's financial-only parent refusal needs an authentic +19/23-node run; the retained tax-rebased 51-node parent refusal is authored. T5 +does not exhaust all initial base-key/artifact, final declaration/manifest or +role-reconciliation adversaries. T6's actual source composition and checked-view +behavior need guarded execution; source confirms PUF calls the completeness gate +before source admission. T7 GQ and zero-weight scientific coverage stays open: +the positive fixture excludes unsupported GQ and its source helper fixes positive +weights. A coherent new source fixture requires separate owner review. Unknown +values must remain unknown. The status+tax42 complement check now uses the retained pre-tax property +population, where the original status writer still owns its columns. Final +completion and tax populations retain separate full custody and reconstruction +checks. The isolated repair passed source review; its guarded runtime check and +combined-host acceptance remain pending. + +A new guarded resource proposal is still required. Source-visible work includes +base issuance, one receiving/role reconstruction, independent child numeric +reconstruction, union execution, artifact loading twice during extension, and a +mandatory materialized child verifier. Each host requalification invokes the +base checker, optional full role qualification and source-closure reads. Child +construction has its initial qualification plus `_current()`; each executed +country child kernel calls `_current()` twice, and materialized verification +calls it twice even on required replay. These are call-site descriptions, not a +measured file/read/memory budget. The child 64 MiB artifact and status 2 GiB +serialization limits remain upstream constraints. No national capacity, native +PUF, fiscal, scientific age-transfer or release acceptance is claimed. + +The combined source and runtime-module classification passed independent source +review. Root owns the final source closure, identity/golden reconciliation and +independent runtime gates. Source approval does not establish runtime acceptance. diff --git a/docs/us-survey-development-inputs.md b/docs/us-survey-development-inputs.md new file mode 100644 index 000000000..a68969951 --- /dev/null +++ b/docs/us-survey-development-inputs.md @@ -0,0 +1,46 @@ +# Source-qualified survey development inputs + +The native financial host accepts +`source_qualified_development_inputs=True` to publish four current ASEC income +mappings before PUF application. The default is `False`. + +The option adds `survey_development_inputs.source_projection` and +`survey_development_inputs.attach`. The first node publishes the authenticated +routing basis, source literals, knownness and named development assumptions. +The second maps each qualified original person to its complete pair of survey +clones. Both nodes bind the declared parent inputs and genuine producer keys. +Existing canonical columns cause refusal. + +| Person output | Maintained development mapping | +| --- | --- | +| `taxable_private_pension_income` | Apply the maintained pension fraction to qualified pension and annuity amounts when both are known. | +| `taxable_ira_distributions` | Preserve the qualified regular-IRA route amount when the distribution and routing are known. | +| `rental_income` | Use the qualified net-property amount as the existing rental proxy. | +| `farm_operations_income` | Preserve the qualified farm amount, including a known negative amount. | + +The implementation reuses +[`_development_person_values`](../packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_observed.py) +and the current ASEC income-routing qualifier. Graph parameters and typed +artifacts record the existing rule names and values. They retain the raw totals +and unresolved source taxability or component statuses. These development +mappings do not establish observed taxable amounts or release-science acceptance. + +American Community Survey (ACS) rows remain unknown. Ambiguous, off-route and +otherwise unqualified ASEC values remain unknown as well. A not-in-universe code +does not establish a zero. The attachment preserves identifiers, other columns, +typed weights, strata and population accounting. + +The new attachment follows the selected base financial/property/tax output. +When child completion is enabled, the completion receiving version carries the +four additions from the genuine base. Household roles, child property and the +final tax gate remain the later writers. The output selector returns that final +tax gate for completed runs. Compact retention keeps the development attachment +in the base and retains the existing completion owner. + +The tests execute genuine invented source preparation and routing, then real +source and attachment kernels with cold execution, required replay and +independent artifact/Population reconstruction. They also exercise the actual +child, household-role and tax declaration factories through the compiler and +check the compact roster. These component tests do not execute the complete +financial/property/PUF host. That continuation requires separate verification +before dataset-release claims. diff --git a/docs/us-survey-person-status-graph.md b/docs/us-survey-person-status-graph.md new file mode 100644 index 000000000..87efa31bf --- /dev/null +++ b/docs/us-survey-person-status-graph.md @@ -0,0 +1,90 @@ +# Descriptive person status in the common US graph + +`run_atomic_survey_financial(..., person_status=True)` adds four operations +after the complete survey/PUF-support clone families and atomic geography, before +financial enrichment. The default remains `False`. The existing nine-node +spine/geography prefix and its returned populations remain separate and unchanged. +The optional graphs have 23, 39 or 42 nodes, compared with the respective +19-node financial, 35-node property and 38-node property/tax paths. + +| Node | Input and output | +| --- | --- | +| `survey_person_status.source_projection` | Borrows the actual retained preparation and source qualifier. Creates a private branch of original support and a typed artifact containing exact original identities and literal items. | +| `survey_person_status.literals` | Exposes the complete union of ACS/ASEC raw items and distinct allocation/edit flags on original support. | +| `survey_person_status.recode` | Calls the existing descriptive recoder and checks its result against the retained qualifier. Declares every parse/code field, difficulty/student observation, knownness, universe and period field. | +| `survey_person_status.bind` | Joins the descriptive observations and context onto both initial clone families by exact original-person, native-person, survey-arm and clone-index identity. | + +The receiving Frame carries ten descriptive observations and their explicit +context, rather than repeating all raw strings per clone. Raw/edit/parse fields +remain on original support. The projection artifact is therefore required for +the audit lineage: its typed producer edge, digest and the clone-origin linkage +must be retained by a subsequent export. A receipt alone cannot authorize the +source. Row values, private paths and native identifiers are absent from public +node parameters and receipts. The private JSON artifact represents integers as +decimal strings under `integer_literal`, never lossy JavaScript numbers. + +All column declarations are fixed, including when the selection contains only +one survey arm. Missing opposite-survey fields remain null. Code columns use +nullable `Int64` and observations use nullable Boolean storage, with knownness +separate from the value. No `is_blind`, `is_disabled`, annual full-time student, +prior-wage, sex or household-role writer is added. The +[source definitions and reference periods](us-survey-person-status.md) remain +unchanged. + +The shared host adds a typed status-binding dependency to its existing financial +donor node. A narrow adapter checks that dependency, then supplies the exact +original declaration and artifact slice to the existing donor kernel. This +ordering is visible in the graph. The original source qualifiers continue to +receive the untouched geography population. + +Verification covers the complete output. The status boundary reconstructs every +status value, dtype, null mask, writer and position from qualified source literals. +The existing financial/property verifiers independently verify the remaining +population, including unrelated values, structural links, weights, design +weights and mass ledger. The host retains complete output seals and rechecks +them after its last source requalification. It refuses both a status-only change +with an unchanged financial complement and an unrelated financial-cell change. + +When property-tax rebasing creates a receiving version with FILTER, that +structural node owns the carried status columns in the new version. The host +checks the original status writer on its retained pre-tax parent, then verifies +the complete tax reconstruction and final population seal. It does not accept +arbitrary replacement writers or treat the carried columns as new source facts. + +The fragment validates the actual qualified object on admission. Before and +after each executed kernel callback it checks live code, defaults, closures, +declared artifacts, input/output seals and the original preparation/student +owner relationships. The final host borrow requalifies the original source +members before the last pure checks. Cold execution and required replay use the +same declarations and full materialization comparisons. + +## Validation boundary + +The private projection has an explicit 2 GiB serialized-byte ceiling. Each raw +survey token retains the source qualifier's 64-character bound; the existing +ASEC student-controls issuer also retains its 600,000-row ceiling. There is no +claim that these limits admit the complete ACS/ASEC universe; the inherited +survey graph's preparation/allocation artifacts each retain their 64 MiB limits. +Projection JSON is +materialized in memory before its byte-limit check, and its working memory can +exceed the final serialized size substantially. The small invented fixture will +check a conservative 16 KiB-per-original serialized bound; that is an authored +test expectation, not a measured native-data size. Before a maximum-local or +full-national claim, measure the actual bytes per original and peak memory on +successive small samples and consider a separately reviewed columnar/chunked +private artifact. Repeating that artifact in every clone is not an option here. + +Authored tests include pure schema/binding controls, an actual 13-node +spine/geography-plus-status fixture, optional 23-node common-host replay, and +39/42-node property/tax preservation using invented authenticated sources. +These tests are unexecuted in this source-only change pending review and a +separately pinned bounded run. The later PUF and survey-enrichment hosts are not +modified here; their combined end-to-end preservation remains a separate +integration acceptance. Input-coverage counts describe the input registry, not +the number of Frame columns. + +The root previously reported acceptance of 77 person-status source tests +(75 pure/source controls and two actual owner cases), and 100 canonical-sex +tests (88 plus 12). Those are root-reported source-boundary results, not this +author's independent acceptance, validation of this new graph, a native-file +build or release certification. The frozen native V4 graph is unaffected. diff --git a/docs/us-survey-person-status.md b/docs/us-survey-person-status.md new file mode 100644 index 000000000..31310cda3 --- /dev/null +++ b/docs/us-survey-person-status.md @@ -0,0 +1,100 @@ +# Survey person-status observations + +The additive person-status qualifier retains published difficulty and enrollment +items from the original ACS 2024 and ASEC 2025 person sources. It does not assign +`is_blind`, `is_disabled`, or `is_full_time_college_student`. It is independent of +the existing legacy eligibility operator. The optional +[four-node graph fragment](us-survey-person-status-graph.md) exposes these +descriptions on the common US population; it does not add release coverage or +establish eligibility. + +`qualify_current_survey_person_status(preparation)` accepts the actual retained +`AuthenticatedSurveyPopulationPreparation`. It captures its pinned original +members, selects people through exact original household/person/line identities, +and checks each original age against the unchanged `A_AGE` column. It does not use +a normalized or imputed `age` as the item universe. Twenty-two-digit ASEC native +person keys remain strings; stacked person IDs retain their integer index. + +The result contains the original Frame reference, an origin table, literal source +columns, descriptive observations, and an immutable receipt. The result owns no +source authority. Its `validate()` method rechecks the actual preparation and +student owners and compares all retained value seals after the last owner I/O. +A consuming host must keep those owners, validate after its own last relevant +I/O, and independently verify any materialized descendants and replay. Serialized +rows or a copied receipt do not authorize a source or a graph execution. + +## Meaning and period + +| Observation | Definition | What it does not establish | +| --- | --- | --- | +| `survey_vision_difficulty` | The published answer to blindness **or serious seeing difficulty**, with source and period retained. | Statutory blindness, measured visual acuity or visual field. | +| Other `survey_*_difficulty` items | The particular hearing, cognitive, ambulatory, self-care and independent-living questions. | A program-specific disability determination. | +| `survey_any_applicable_difficulty` | Any known yes among applicable items; false only when every applicable item is a valid no. Completeness and universe contradictions are separate diagnostics. | A medical, duration or eligibility finding. | +| `survey_publisher_disability_recode` | `PRDISFLG` or `DIS`, retained separately from the item-derived battery. Agreement is reported only when both are known. | An independent clinical observation. | +| `survey_full_time_college_student_last_week` | A coherent ASEC enrollment, college and full-time combination for the preceding survey week in 2025. | Annual student status in the 2024 income year or five calendar months of attendance. | +| `survey_college_attended_last_3_months` | ACS school attendance with undergraduate/graduate level in the preceding three months. | Full-time workload. ACS does not measure that here. | + +ASEC difficulty items use the published `PRPERTYP = 2` universe, rather than a +replacement age threshold. The CPS publication may retain an earlier rotation +response; its survey year does not imply a fresh March interview answer. ACS +hearing/vision apply at all ages, cognitive/ambulatory/self-care at age 5+, and +independent living at age 15+. Out-of-universe items remain unknown, not false. +A positive or other non-NIU code outside that universe prevents an item-derived +battery answer and remains visible as a contradiction. + +For ASEC student items, the printed enrollment universe is ages 16–54; the +named zero explicitly includes NIU, children and Armed Forces. The descriptive +recoder therefore also rejects positive enrollment routes for those named +person types and leaves an unresolved type unknown. This is an additional +published-code consistency check, not an alteration of the existing control +issuer's age-only assertion. Missing or unreadable student literals remain +unresolved even outside the age universe. + +All values retain literal/code/parse states. Printed but unnamed numeric range +members remain unresolved. ASEC `PXDIS*` transitions distinguish no change, +longitudinal retention, allocation and changes to blank; nonzero is not a common +allocation flag. Valid published yes/no values survive unknown edit provenance +without becoming claims about an unallocated original answer. ACS `FSCHP` retains +its enrollment scope and the separate `FSCHGP` flag records grade-attending +allocation. The initial source-review packet overlooked `FSCHGP` on dictionary +p.129; this implementation corrects that omission using the same pinned PDF. + +## Source contracts + +- [ASEC 2025 dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`: student values pp.23–24, difficulty values pp.26–27, person type and summary p.28, student allocation pp.28–29, edit transitions p.30 (one-based PDF pages). +- [ACS 2024 dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf), SHA256 `929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff`: difficulty p.36, attendance/level pp.43–44, summary p.56, allocation pp.124–125 and129–130. +- [ACS 2024 household questionnaire](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf): attendance items10a/b, p.12; difficulty items18–20 and their age gates, p.14. This does not claim separate verification of the group-quarters instrument or every interview mode. +- [BLS disability FAQ](https://www.bls.gov/cps/cpsdisability_faq.htm): rotation-period retention and the distinction between the statistical battery and SSA eligibility. + +The minimal readsets are declared once in `current_survey_person_status.py` and +recorded in the result receipt. `SCHL` is excluded because it measures completed +attainment. ASEC `A_ENRLW`/`A_FTPT` are retained as current-cohort literals as well +as cross-checked against the maintained `asec_student_controls` issuer. Calling +that issuer consumes its existing **three-cohort** contract (income years +2022–2024); only the matched 2024 cohort is selected into this projection. The +extra current-cohort `A_HSCOL` and allocation fields are authenticated from the +pinned original member rather than assumed to be covered by the control issuer. +Where the parent retains known `A_HSCOL` or `PEDIS*` cells, the qualifier also +checks their numeric representation against the captured member. Missing carried +cells impose no observation; matching NIU or unnamed numeric codes only establish +representation consistency. They remain unknown under the named observation +domains. Per-field comparison counts are recorded without changing the parent. + +Code columns have nullable integer dtype even when every code is unknown. The +receipt records actual table columns, named observations, difficulty mappings, +allocation vocabularies and the applicable source periods alongside the table +digests. This metadata describes the projection; a stored receipt cannot replace +the retained owner or its validation callback. + +The source modules perform no downloads, no fitting, no tax/benefit calculations, +no cloning, and no graph or dataset publication. A later legal or annual bridge +must be a separate declared operation with its assumptions and validation. + +## Validation scope + +The committed tests use invented literals and privately pinned small source +fixtures. They cover universe boundaries, partial knownness, named allocation +transitions, enrollment routing, exact keys/ages, copied owners and mutation +seals. This patch was prepared under a source-only boundary: static checks passed, +but these new tests have **not yet been executed**. Bounded runtime acceptance and +any integration into the country graph remain separate work. diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md new file mode 100644 index 000000000..e01ab94ee --- /dev/null +++ b/docs/us-uk-release-path.md @@ -0,0 +1,379 @@ +# US and UK release path + +Planning snapshot, 14 September 2026. This is a release work plan, not a +certification of either country. Component tests, historical candidates and +current-candidate acceptance are distinct evidence. + +## Products and common construction + +The US product must support national and congressional-district analysis. The +UK product must support national, constituency and local-authority analysis. +Each country should have one fully constructed population from which full and +smaller analysis files are derived. Record the survey, monetary, policy and +boundary reference periods explicitly; do not infer them from a filename. + +After constructing and harmonizing the complete multispine, including its +initial clones, assign one atomic area to each resulting household and derive +larger geographies from versioned mappings. Distinct clones may receive different +areas, constrained by observed source geography and keyed by stable clone +identity. Subsequent enrichment retains the assigned location. Once construction +and enrichment are complete, +calibration changes household weights; scope selection and pruning retain the +original values, missingness and relationships for every retained household. + +For a shared set of weights and consistent measurement definitions, additive +local totals must reconcile to national totals. Separately refitted compact +views need declared approximation tolerances and their own diagnostics. A +common source population alone does not guarantee identical estimates. + +## Release sequence + +| Stage | US work | UK work | Exit evidence | +| --- | --- | --- | --- | +| Source and period coverage | ACS and ASEC survey multispine; PUF tax detail; SCF and other required auxiliary sources; declared aging | Raw FRS 2024–25; SPI/HMRC, WAS, LCFS/ETB and other required stages; declared uprating | Source register, licensed-input boundaries, source definitions, missingness, units and periods; every applicable input has a producer | +| Complete population | Harmonized relationships and tax/benefit units; geography; financial and PUF enrichment; remaining benefit, disability, housing and asset inputs | Integrated FRS stages and current relationship/claimant definitions; income, wealth, consumption and benefit inputs | Small real end-to-end build; source-channel coverage; plausible distributions and joints; no silent engine-default substitutions | +| Geographic support | Block assignment after initial multispine cloning; versioned county/PUMA/CD derivation; enough household types in each district | Shared atomic-area assignment after full spine cloning, and derivation for each nation; adequate constituency/LA support | Observed geography respected; distinct stable clone draws; subsequent location invariance; support and fit measured at advertised levels | +| Model and calibration | Locked US model; national/state/CD target matrix from identified facts | Locked UK model and companion input changes; national/local matrix with corrected UC and other target definitions | Baseline engine evaluation, calibrated population, target-level errors, support/weight diagnostics and explicit outstanding failures | +| Independent quality | Held-out distributions and geographic cells; tax/benefit totals and representative reforms; comparison with the incumbent | Current release battery, held-out areas and complete incumbent comparison; resolve remaining support and target-fit failures | Candidate-specific reports; explained regressions; thresholds agreed before choosing a candidate | +| Scale and compact views | Increase local build size after smaller checks; derive and refit the intended full/sparse family | Choose a size that meets local support and quality needs; complete compact/exact-count release evidence | Measured time, memory and cache replay; retained-input invariance; full/compact error and performance comparisons | +| Release and consumption | Exact-file dashboard, methods/papers, immutable package and real PolicyEngine loading | Exact-file dashboard and certification; immutable cut, loader/pointer integration and PolicyEngine-UK adoption | Fresh export/readback, reproducible manifest and checksums, consumer calculation smoke test, reviewed release and rollback path | + +## Current position + +The 45-node US combined completion-host test stopped at its 1,800-second CPU +limit without a completion receipt. Its 49- and 51-node status/role variants +remain on hold while we investigate repeated verification costs. The child property-income component has passed all 58 +pure tests and all 14 actual-owner cases on invented inputs. The local host +repair now also passes all 50 small custody tests, including private-support +mutation and permanent-revocation checks. These tests establish component and +boundary behavior; the combined host and a native release file remain unverified. + +The latest native property/tax attempt reached its CPU limit without a final +acceptance receipt. Paired measurements did not justify adopting either the +scanner or digest candidate: each slowed relevant workload classes. The +existing implementations remain. Repeated whole-frame and source verification +are a plausible bottleneck from source inspection. A small experiment using +the actual functions passed seven correctness controls and measured a median +18.36 ms per frame-identity call on 2,048 invented people. Actual call counts +and whole-build attribution remain unmeasured. No whole-build speedup, native +rerun or larger resource budget follows from these measurements. +Native PUF qualification still requires completion of applicable unknown inputs. +Adult routes outside the child model need aggregate source-incidence counts +before choosing whether another producer is necessary. Retirement +candidate accounting has passed 80 controls but does not yet supply fiscal +inputs. The +[dated evidence](../experiments/us-launch-consolidation-20260913.md#14-september-completion-checks) +records the accepted checks and remaining work. + +UK work proceeds independently on the shared same-kind weight update and +metadata/mass-log/column-order interfaces needed by its country graph. The +separate [UK contract proposal](https://github.com/PolicyEngine/microcosm/pull/918) +has passed 101 invented runtime checks. Native FRS integration and an accepted +UK file remain unverified; that proposal establishes no new UK candidate or +release acceptance. + +The US housing participation graph is incorporated in the integration branch, +with 64 focused and nine actual invented full-host tests passing. It preserves +source observations, original household design weights and paired clone draws, +with awards and caps owned by the country model. Native housing outcomes and +the remaining fiscal-input inventory still need verification. See the +[housing integration evidence](../experiments/us-launch-consolidation-20260913.md#other-integrated-source-work). + +| Stage | US demonstrated result | UK demonstrated result | +| --- | --- | --- | +| Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; maintained two-route PUF and subsequent amount/health hosts pass scoped invented controls; native PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | +| Geography | Small native post-clone block assignment and financial extension passed on 11 September; strict required replay passed on 12 September; complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | +| Calibration | Declared national/state/CD measurement and dense grouped calibration stages pass on invented inputs; fully enriched native solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | +| Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | +| Full/compact files | Small prefix replay and 36 supplied-parent full/pruned/local export checks pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | +| Delivery | No accepted new-architecture release or default consumer adoption | Assembly implementation and historical assembled cuts exist; current certification and default promotion pending | + +The 11 September native pilot passed the corrected survey → full initial +support clone → atomic block assignment → financial-enrichment order. At a +1/1000 source sample, its 1,584 source households and 3,464 people become +3,168 households and 6,928 people after cloning. The nineteen-node graph fits +and attaches seven financial fields. Its closed receipt is tied to frozen +source `2ca11c85a`, and reports 5,362 seconds elapsed and 9.31 GB peak RSS. +This supersedes the earlier assignment-before-cloning milestone for ordering; +it does not establish full PUF enrichment or national/CD calibration. + +On 12 September, a strict required replay of that frozen source passed with a +fresh output directory and a read-only retained store. It verified the source +bytes again, reused all nine prefix and nineteen financial-graph nodes, and +reproduced the four non-manifest exports byte for byte. The portable manifest +also matched after excluding only operational timing and cache-hit fields. +The closed run took 3,981 seconds and peaked at 10.99 GB RSS; source, resource, +native-input, owned-file and thread-control checks remained unchanged, with no +unexpected access refusals. See the +[replay record](../experiments/us-financial19-required-replay-20260912.md). +Meanwhile the maintained graph's observer-mutation defect has been fixed and +independently reviewed. The shared graph consolidation merged in +[PR #913](https://github.com/PolicyEngine/microcosm/pull/913) at +`a9cc63e737fd4619a304117dc2ec7dcd86d97901`; the release branch has incorporated +that main revision without changing its existing graph implementation. Those +software changes still need their +own native candidate verification; this replay certifies only its frozen +financial stage, not the newer source or a release. + +The maintained PUF55 host is now adopted. Its 31 small controls and two actual +full-host controls pass on invented inputs: both conditioning routes, 110 fits +and 110 applications, cold execution, required replay, whole-population +preservation and numerical finalizer equality. The full-host run took 742 +seconds and 0.74 GB peak RSS. Those 64 invented donors do not establish the +runtime or quality of the full native donor. A separate current-source native +packet uses a fresh writable store and the actual retained financial run. +See [the PUF host record](../experiments/us-puf55-host-adoption-20260912.md). + +That native packet's second attempt closed with an assertion failure after +about 82 minutes, during its financial-stage acceptance and before any PUF +execution. The source and resource pins remained unchanged and no unexpected +access refusal was reported. Source review identified an invalid harness +assumption: the atomic host first materializes CREATE and ALLOCATION, so those +two nodes legitimately hit when its expanded graph runs in the same store. +Requiring every prefix node to miss rejects that valid execution. The refusal +record does not identify its precise assertion frame, so this diagnosis does +not establish completed native financial acceptance. The corrected v3 harness +passes two actual invented financial cold/replay controls and thirteen bounded +source and diagnostic controls. After independent review of its exact packet, +one fresh native run started with unchanged calculation source, native inputs +and resource limits. On 13 September, the owned process was no longer present +and neither a final receipt nor a PUF manifest existed. Its last recorded +progress was about 5.76 hours wall time, 6.41 hours process CPU and 28.29 GB peak +RSS. The exit code and stopping cause are unavailable; the CPU total exceeding +the configured budget does not establish that cause. This attempt is closed +without verified native PUF output. A successor requires a fresh reviewed +packet; the preceding source, input and failure evidence remain preserved. + +The maintained financial graph now retains an eighth field: the tax-exempt +interest remainder of its existing observed or modeled total. The split remains +a modeling assumption. Twenty-five financial/source controls and three full +invented PUF replay, finalizer and retained-output controls pass; the PUF stage +preserves the original-channel remainder. The historical native financial +receipt above still covers seven fields. See +[the interest correction](../experiments/us-survey-interest-conservation-20260912.md). +A separate reviewed source-hashing optimization preserves exact digest bytes +and passes thirty source tests. Its repeated-string benchmark improvement is +not a measured native build improvement; see +[the scoped benchmark](../experiments/us-source-string-seal-20260912.md). + +The new fiscal measurement stage passes all 66 interpreter/stage checks on the +combined integration, including an actual US-engine dividend calculation. It +uses declared target bindings, household-aligned CSR measurements and explicit +national/state/CD masks. Missing consumed predicates refuse, indicator sums +are numeric, and local calculation helpers participate in cache identity. +Specialized unsupported bindings refuse; this is not full fiscal-registry +coverage. + +The dense grouped calibration stage is now integrated. Independent review of +its numerical adapter found no actionable defects, and all 167 selected +calibration tests passed. The combined integration also passed all 42 tests +for the measurement and dense stages together. The adapter uses the existing +grouped Adam solver, keeps initially zero weights fixed, enforces group bounds +and refuses a result exceeding the declared original row caps. It returns +calibrated weights and schema-8 diagnostics. Numeric bound arrays do not prove +their source: the complete-population host must authenticate the original +sampling references and retained parent before and after use. Native fit, +full-size memory and calibration acceptance remain unestablished. + +The dense fiscal kernel now forwards an optional host-owned target-snapshot +observer to the actual grouped solver. Tiny invented graph tests verify equal +weights, artifacts, receipts and cache identity with observation on/off, plus +silent required replay and final mutation refusal after the sink. The integrated +suite has 473 passes and one explicitly recorded historical thread-configuration +skip; five final calibration-identity checks also pass. Source admission, +production cadence costs and a dashboard consumer remain separate. See +[the fiscal snapshot evidence](../experiments/fiscal-target-snapshot-host-20260912.md). + +Fiscal leaf-policy infrastructure now distinguishes required producers from +explicitly documented proposed assumptions. It passes 91 source/mock and +invented fiscal graph checks. The default remains strict all-producer; +assumption execution and inactive-leaf exemptions remain unsupported until +complete-parent admission and exhaustive dependency proof exist. No US +assumptions or new dataset inputs are enabled. See +[the policy contract and boundary](fiscal-leaf-policy.md). + +Complete-population ancestry remains an explicit requirement. The PUF host now +retains its actual checked result for downstream consumption through +`check_survey_puf55_run`. Constructing a dataclass or decoding its receipt cannot +issue that authority. Consumers must recheck the original run before use and +after their last relevant I/O. The corrected checker passes 25 controls on +actual invented cold/required execution, including extra manifest attachments, +source/store corruption and late output mutation. Independent review approved +the source and mechanical test colocation; 45 cases share one expensive fixture. +See [the retained-output evidence](../experiments/us-puf55-checked-output-20260912.md). + +The fixed post-PUF country host now assembles source-qualified unemployment, +three health-cost amounts and nine current health-coverage inputs. It retains +the checked PUF owner and attaches both fragments to the same receiving +population, preserving existing columns, geography, design anchors and mass +ledger. ASEC observations and unresolved values are retained on both clones; +modeled ACS draws are deliberately shared within each original person's clone +pair. Reporting universes remain explicit; no prior-wage or age-only zero +fallback is introduced. The narrower ACS coverage concepts remain nullable +where the source does not establish equivalence. + +The accepted invented fixture reports 74 present input names out of 161, up +from 61, leaving 87 missing. It executed 26 new enrichment nodes over a validated +245-node prefix, then required all 271 replay hits and complete Frame readback. +Its six logical controls passed across five tests in the main attempt and a +corrected focused parent-revocation follow-up, not one all-green full run. +Independent review approved that scoped evidence. After integration with the +eighth interest field and merged shared graph, 80 cheap source, attachment and +health-fragment controls pass. Declaration checks still give 271 nodes: the +eighth interest field adds an output to the existing financial attachment, +not a new fit or graph node. This integration did not repeat the full combined +host or measure fresh native coverage. See +[the combined integration record](../experiments/us-survey-enrichment-integration-20260912.md) +and [source/model decisions](us-current-survey-amount-successor.md). + +Actual graph tracing also confirms that the legacy full ASEC carry and ACS +gap-fill are not executed by this host. Remaining pension, retirement-account, +property, farm and other-income inputs therefore need source-qualified producers +and appropriate ACS completion. The ACS property-income aggregate should +condition a joint decomposition before the dependent PUF fits; separately adding +rent afterward would not conserve that observed anchor. Source measurement +bridges and overlapping income categories must be resolved explicitly before +claiming reconciliation. The broader ACS retirement total is not merely pension +plus regular-IRA income. + +Two reviewed building blocks for that completion are adopted in the release +branch. The [ACS anchor qualifier](../experiments/us-current-acs-income-anchors-20260912.md) +preserves original INTP/RETP amounts, adjustment factors, original ages and +allocation flags through the retained source owner; its 35 invented-source +controls pass. The [signed reconciliation helper](../experiments/signed-income-reconciliation-20260912.md) +projects joint component draws onto a caller-qualified total while allowing +declared signed components and preserving loss offsets; its 45 numeric controls +pass. An explicit property-income option now integrates the source bridge, +four conditional component models, signed reconciliation and clone attachment +into the existing financial host. Its actual invented 35-node construction and +required replay pass, preserving the full preceding population. The new columns +remain available beside the retained earlier population. A further explicit +three-node extension rebases four interest/dividend tax inputs and checks their +conservation and completeness. Its 38-node cold/required host passes eleven +invented-source controls; unknown inputs remain unknown and an incomplete gate +refuses PUF qualification. Retirement completion and complete PUF/native adoption +remain separate work. The focused PUF +handoff retest passes after correction of an invalid invented source record; +see the [scoped host evidence](../experiments/us-property-financial-host-20260912.md). +The [13 September integration record](../experiments/us-launch-consolidation-20260913.md) +also covers retirement source qualification, UK lineage and the CI repairs. + +The remaining critical sequence is complete PUF/input integration, a small +real full build, model and calibration evaluation, then progressive scale +and release verification. The existing age-development runner is a separate +prefix demonstration, not calibration of the nineteen-node financial result. See +[the US review guide](us-launch-review.md) for scoped acceptance records. + +UK work already includes raw-source enrichment, national/local calibration, +size experiments and release assembly. Its recorded P50/P95b 55,000-household +candidates fail area-support and target-fit checks; the skipped holdout does +not establish an incumbent-comparison pass. Historical dense assembly also +does not satisfy today's release battery. See +[the size evidence](../experiments/355-uk-dataset-size-receipts.md) and +[the current dense release runbook](uk-dense-release-assembly-runbook-762.md). + +The current UK route clones for geographic support and then uses the existing +OA ladder. It has not adopted the new shared atomic geography graph keyed to +stable post-clone household identity. Its existing placement after cloning is +consistent with the user's 10 September ordering correction; the source, +identity and shared-operator migration remain separate work. Maria's +[PR #900](https://github.com/PolicyEngine/microcosm/pull/900) updates Chronicle +household targets and the Northern Ireland lookup; that source work does not +by itself migrate the assignment owner. Anthony's +[PR #896](https://github.com/PolicyEngine/microcosm/pull/896) provides staging +telemetry and synthetic smoke evidence, not a calibrated release. + +The shared UK adapter now has accepted supplied-array tests for England/Wales +Output Areas, Scottish Output Areas and Northern Ireland Data Zones. These +checks cover source conventions, observed-region constraints, stable post-clone +identity and row-order/subset invariance. Ordinary UK imports also preserve the +public export contract without eagerly loading unrelated stages or country +engines. All 52 checks passed with unchanged source bytes and no unexpected +access attempts; see the [scoped acceptance record](../experiments/uk-atomic-area-lazy52-20260910.json). +No native UK area source, whole-spine build or calibration result is certified +by those checks. Maria can use these shared primitives in the existing UK graph +integration rather than starting a second country runtime. + +UK release coverage must be re-established on the chosen integrated build and +locked model. The declared 145-input contract's older candidate evidence must +not certify the current stage set. The +[migration epic](https://github.com/PolicyEngine/microcosm/issues/665) tracks +source and calibration work; +[exact-count release parity](https://github.com/PolicyEngine/microcosm/issues/898) +and [default promotion](https://github.com/PolicyEngine/microcosm/issues/823) +remain separate consumer-release requirements. Existing open issues are leads +for candidate-specific verification, not proof every historical defect remains. + +## 15 September decisions + +- The native property and tax graph ships without the child-property + completion. Persons under 15 carry zero interest and dividend income, as the + incumbent default does: in Build P, 0 of 37,248 persons under 15 have + taxable interest or dividends, against 918 of 8,669 persons aged 15 to 17 + with taxable interest. The completion's transfer from 15-to-17-year-old + donors is an unmeasured assumption that claims no source observation. It + returns only as a count-calibrated policy anchored on an external tabulation + of children's unearned income, such as IRS SOI's Form 8615 statistics. + Turning it on runs the ten completion nodes; the 35 base nodes replay from + the store. A pilot may declare a candidate-scope policy to exercise the + nodes; a release does not. +- Native runs keep the 21,600-second CPU cap. A cap hit closes the run with + its complete CPU-by-code-chain ledger and authorizes no larger rerun. The + cost is removed at its source before any native run repeats: sources are + authenticated once per run with memoised validation, source keys are derived + once per run and re-derived at run end, the ACS record scan runs at C speed, + and column digests hash buffers rather than values. Measured on the + nineteen-node pilot, node execution is 1,143 of 5,362 seconds; the rest is + repeated admission and verification of 3.48 GiB of sources. +- Main must demonstrably build the certified default from raw sources and pass + its preflight and certification gates. The last from-scratch build is Build P + (28 July 2026, policyengine-us 1.764.6); every later default is a + supplied-parent enrichment of that file. The from-scratch build and + certification run on this integration tree before #893 merges and on main as + a standing check afterwards, and a pull request of this size carries that + receipt. The complete native dataset is not the merge gate; it earns default + status separately by beating the incumbent on held-out cells. + +## Team meeting decisions + +This document is the shared US/UK release-path reference. The country review +guides and run receipts provide supporting detail; update the sequence and +ownership here when they change. + +The 10 September coordination sequence is: + +- Maria's #900 source/target reconciliation merged on 10 September at 14:54 UTC; + Anthony's #855 target hierarchy/diagnostics and corresponding dashboard + consumer follow it. Its merge clears that dependency, not all release gates. +- Maria owns registering every existing UK stage as a graph node, stacking on + #893. Changes to UK stage order follow in a separate PR. The shared atomic-area + adapter work supports that integration and does not create a second UK graph. +- Anthony's UK staging #896 can proceed independently. His earlier runtime + proposal #885 is superseded in intent, not another required implementation. +- US construction and verification continue alongside these changes. Complete + graph coverage in both countries is required for the broader advertised + launch; passing tests for individual stages does not establish that coverage. + +The latest UK full rebuild and evaluation were reported as underway at the +meeting. Its new report must replace historical quality evidence only after the +actual candidate and results are available; the approximate four-hour runtime +reported for that build is not a US or release ETA. + +1. Confirm the release years, advertised geographies and full/compact products. +2. Assign one accountable owner to each country candidate and one owner to each + shared dependency: runtime, calibration, diagnostics and release consumers. +3. Review every row above for the latest actual candidate. Name the missing + artifact and next measurable result, rather than reporting code completion + as population acceptance. +4. Agree calibration, holdout, incumbent and representative-reform criteria, + including which differences need explanation and which block release. +5. Set dates for the next complete small candidate, the full quality review and + the production-sized candidate using measured runs rather than linear timing + extrapolation. + +Country data work, target/source reconciliation, independent validation, and +dashboard/consumer integration can run in parallel. Each final dashboard and +release verdict must refer to the same candidate bytes. Cross-sectional release +does not require retirement dynamics, every possible forecast year, or a +complete Chronicle/Orrery redesign. Keep those workstreams active without +making them prerequisites for the first accepted cross-section. diff --git a/docs/us/puf-original-application.md b/docs/us/puf-original-application.md new file mode 100644 index 000000000..208c2b1bd --- /dev/null +++ b/docs/us/puf-original-application.md @@ -0,0 +1,37 @@ +# Original survey-arm PUF application + +`us_runtime.puf55_original_application` declares all 55 actual conditional +applications per selected original-arm route. It reuses the exact maintained +route fits and consumes the source-qualified, typed fixed-value artifacts. +The fitted-model seed remains the clone-one seed. The caller must choose a +separate, explicit `original_application_seed`; the helper invents no offset. +Existing clone-one declarations and v1 decoders are unchanged. + +Every target still draws for every recipient. Its raw draw is retained; the +conditioning output substitutes only cells whose qualified fixed-input mask is +true. Later targets consume that conditioning output. The v2 history binds the +matrix bytes and producer, actual model/conditioning/fixed-input producers, +complete ordered target prefix, raw/conditioning hashes, known counts and the +separate fit/application seeds. + +`merge_puf55_original_conditioning` checks the exact closed 55-target routes and +returns a detached tax-unit table and a descriptive receipt. It does not +unpickle models, issue source authority, attach columns, or certify scientific +coverage. Its caller must authenticate each typed artifact against the genuine +compiled graph and retained run, and requalify the original financial/fixed +inputs at the maintained pre/post I/O fences. Merely constructing an +`ArtifactValue` or a matching digest does not satisfy those duties. + +Eight shared financial inputs are qualified survey-preserving values; they can +include earlier modeled or split values. Optional pension/IRA/rental/farm fixed +inputs retain the explicit development rules and unresolved source statuses +from `puf55_survey_observed`. None becomes a claim of observed taxability. +An aggregate mask is true only when all members are qualified. A false mask +never permits overwriting known people in a mixed unit. + +Population attachment is deliberately separate. This slice does not implement +mixed-unit allocation, pruning, tail caps, tuition allocation, QBI finalization, +a new host entry, or release-science acceptance. Its component proof runs three +real invented-data fits/applications and required replay. Separate synthetic +codec-only 55-target envelopes exercise roster and merger refusal checks; +those envelopes are not evidence of 55 fitted models or a native owner. diff --git a/docs/validation-cost-and-borrow-boundaries.md b/docs/validation-cost-and-borrow-boundaries.md new file mode 100644 index 000000000..ff809a960 --- /dev/null +++ b/docs/validation-cost-and-borrow-boundaries.md @@ -0,0 +1,67 @@ +# Validation cost and borrow boundaries + +The US graph must detect changed sources, substituted artifacts and changes to retained Populations without repeating the same expensive work at every nested call. This note records the current decision and the proposals still under review. It does not change dataset release requirements. + +## Adopted: reuse source compilation + +The ACS native coverage owner keeps a bounded process-local cache of compiled code. Its key includes exact source bytes, filename, mode, flags, inheritance setting, effective optimization level and compiler identity. Every source read and loaded-function, global, alias and closure check remains fresh. Cached code is neither source authority nor a portable receipt. + +The change passes 25 focused controls and the original invented financial fixture, including cold execution, required replay and final checks. One paired profile took 216.82 wall seconds versus 308.26 seconds before the change, a 29.66% reduction. CPU time fell from 303.90 to 214.69 seconds. These instrumented observations establish an improvement for that fixture; they do not establish native-data performance. See [the experiment](../experiments/us-acs-compilation-cache-adoption-20260910.json) for exact source, test and verification identities. + +The native pilot already in progress retains its original frozen source snapshot. + +## Next: remove duplicate work within a check + +A source review confirmed nine direct Frame storage traversals in a healthy `verify_acs_native_coverage` call: three calls to `_verify_frame`, each performing three storage hashes. Some repetitions occur across supplemental axis and resolved-weight checks, so collapsing all nine into three is not automatically equivalent under the existing mutation checks. + +The next candidate will share the first storage calculation with the supplemental fingerprint calculation, while preserving the independent final prepared-Frame storage pass and all three outer verification points. That would reduce nine direct traversals to six. It needs a regression that mutates a cell during the supplemental check, unchanged receipt and fingerprint comparisons, and a paired measurement before adoption. + +## Keep separate: code, files and retained objects + +Three different facts currently appear together in nested validation: + +- Loaded-code identity: the interpreter's functions, code, globals, aliases and closures. +- Agreement with files: source bytes, archive snapshots and stored graph artifacts still match their recorded identities. +- Retained-object identity: the actual Frame, Population, weights and ancestry still match their issued state. + +An independent Fable 5.1 review recommended making these checks explicit and performing expensive file verification primarily at public entry and emission boundaries. Moving file checks changes when an altered archive is detected, so it needed a documented entry contract and tests at the boundaries before implementation. + +**Adopted 2026-09-16, as a scoped epoch rather than a boundary rule.** `survey_population_preparation.verification_epoch()` opens a verification epoch around a graph run. Outside one, nothing changes: every borrow re-authenticates in full. Inside one, each borrow still pays the cheap tier — loaded-code identity through `_encode(_producer())`, the attached owner payloads and the live authority — and the expensive file and retained-object tier is skipped only while a signature is unchanged: the stat identity of every path the four foreign verifications and `_source_files` re-read, including `_file_stats` over the whole source roster, the identity and length of every borrowed payload, and a read-free witness of each live Frame's storage. The roster's stat identities are read on every borrow, but into that signature rather than as a comparison of their own: a signature that moved is a memo miss, not a refusal, so the complete validation the miss runs is what refuses, with the code it raises today. Leaving the epoch, at every nesting level, re-validates every memoised capsule in full with the memo bypassed, so a change no signature can see still refuses before the run returns anything; a close whose signature moved while it was validating records no memo answer, and at the outermost close, where no borrow follows, it pays that validation itself. The contract, the residuals it accepts and the argument for each are in [verifying native sources once per run](us-native-verification-once.md); the boundary tests are `packages/microcosm-build/tests/test_us_native_verify_once_epoch.py`. + +**Adopted 2026-09-19: cache immutable declaration summaries.** The ACS owner now +retains ordered tuples of top-level definition names and imported aliases, keyed +by exact source bytes. It does not retain mutable ASTs. Every check still reads +the current source and compares its declarations with the current loaded objects; +the existing loaded-code checks also remain. Source changes therefore select a +different entry, and a warm entry cannot hide a changed live alias. + +The cache accepts only the original parser function, code, compiler, exact typed +defaults and AST flag values. Altered parser context bypasses it. Up to 128 source +keys of at most 128 KiB bound retained source bytes to 16 MiB; this is not a total +memory bound. Failed parsing is not cached. Like the existing code check, this +operates within a trusted Python process, not against arbitrary private-state +tampering. + +The candidate passes 210 focused source, coverage, membership and row-ceiling +tests, including warm-cache parser drift, changed source and alias refusals. +Eight repeated code-only producer checks took 2.033 CPU seconds with declaration +caching disabled and 0.385 seconds with it enabled. Their complete producer +documents matched within this candidate. This does not establish a native-run +speedup or equality with producer identities from earlier source versions. The +active native retention experiment keeps its separate frozen source. See +[the declaration-cache experiment](../experiments/us-acs-declaration-cache-adoption-20260919.json) +for reviewed source and verification identities. + +## Composition review + +The same review raised kernel-registry sharing, registration order and recipient ordering. These require owner-level checks before being treated as bugs. For example, each financial run currently constructs a fresh survey prefix internally; it does not accept an existing prefix for a second extension. The PUF raw-route merger separately derives and verifies the complete receiving tax-unit axis before restoring row order. Neither observation alone proves every composition boundary correct. + +The budget returned-view correction is now adopted. Detached documents and grouped bounds are constructed after the last borrowed I/O, then checked against the issued payload and retained owners. Four targeted controls passed, including nine callback branches; the post-run verification rechecked 501 source/owned files, 5,983 model-code files and 12 resources. The earlier eight financial-successor controls remain evidence for their original source revision. These are separate component runs, not twelve tests of one integrated revision. See [the correction experiment](../experiments/us-budget-detached-view-correction-20260910.json). + +A second Fable source review found no confirmed correctness or ownership bug in the public PUF recipient interface. It recommended explicit cold-cache reuse assertions, exact comparisons with the original issuers, and clearer documentation of the original-source reads required by the recipient kernels. The frozen two-positive pilot remains unchanged; those additions belong to subsequent controls. + +The immediate implementation priorities remain actual public PUF recipient and graph checks, authenticated donor/model binding, one whole-cohort PUF finalization and attachment, then native calibration and release verification. Performance work proceeds alongside those tasks. + +## Review scope + +Fable 5.1 completed a read-only source review on September 10, 2026, using the saved Hivesight login. Its requested scratch-cache read was denied and that limitation was disclosed in its result. It supplied architectural advice, not runtime approval or a release verdict. The recommendations above are distinguished from independently verified source findings and adopted changes. diff --git a/experiments/893-reconciliation-amendments-19-20-20260912.md b/experiments/893-reconciliation-amendments-19-20-20260912.md new file mode 100644 index 000000000..f1627eb2b --- /dev/null +++ b/experiments/893-reconciliation-amendments-19-20-20260912.md @@ -0,0 +1,722 @@ +# #893 reconciliation to main's amended graph interface — lane report + +> **12 September Codex continuation:** this is the historical layer proposal +> at `80fef2217`, not an approved extraction plan. The final battery +> `bt0nx8fcc` was killed during its first pytest group: CI-group verification +> and coverage check exited zero, but no final consumer result was produced. +> Independent review found that G4 exposes mutable admitted populations to +> observers and that the layer order omits hard packaged-resource dependencies. +> Fix and review both before extracting or landing layers. PR912 is also held +> for fixes from Astra's executed review; the branch-head assumption below is +> superseded until its corrected head lands. +> +> Resource repair `a4b6fe88c` restores all three omitted ASEC JSON contracts +> from their exact historical blobs, matching unchanged loader pins. Eight +> targeted tests and wheel-content verification pass. These three resources, +> `acs_2024_housing_universe.json`, and current-money domains/consumers/price-basis +> resources must accompany their first PR-1 loaders or an earlier prerequisite; +> Python import analysis alone cannot establish extraction order. No engine +> defaults were admitted and no source/engine identity was refreshed. + +> **Later 12 September correction:** `6f66fa545` closes the demonstrated G4 +> mutation defect by handing observers detached snapshots, including nested +> table values, metadata, schema, weights and ledgers. Independent review is +> clean; 524 graph tests, five focused cases and all four existing synthetic +> consumers passed, including financial and calibration teardown checks. An +> earlier source-edit-during-test failure remains preserved and is excluded. +> Entry 7.4 below is historical: the current observer need not be trusted to +> leave its snapshot unmodified. Retaining snapshots still costs memory. +> +> `cb27bdb0b` makes the offline explanation display executor-owned unreached +> and gate-exception states independently from cache hits. Sixteen graph +> explanation/actual-gate cases and two free-form diagnostic controls passed. +> Its new `explain → availability` import is recorded in the implementation +> inventory; availability was already bound as a whole module in all ten +> using scopes. No scope, dependency, resource, or unbound-use exemption was +> added. All eight prepared-resource/full-manifest tests passed afterward. +> +> Corrected PR912 head `5e33d6971` is undergoing exact-head CI and Fable review. +> Graph-only extraction is being prepared against that revision, incorporating +> G1–G4 and numbered charter entries. Neither that extraction nor the remaining +> country layers are declared accepted by this historical report. + +> **Subsequent 12 September integration checks:** corrected PR912 head +> `5e33d6971` received Fable approval; its merge remains conditional on CI. +> The local integration merged that exact revision in `9e61d0d04` and +> recomputed the combined source/seed identities. All 734 graph and fitting +> tests passed. The six-file spec/build sweep collected 248 tests: 247 passed +> and one failed because its invented PUMA fixture still used schema 1 and +> omitted the required joint tract/CD population support. The fixture now +> supplies consistent schema-2 support and explicit invented provenance; that +> single test passed when rerun. The original failed sweep and intermediate +> incomplete fixture repair are preserved. This is separate-run evidence, +> not a claim that the complete 248-test sweep was rerun successfully. +> +> Shared graph consolidation is now public as PR913, temporarily based on +> PR912, with independent Fable review in progress. Its runtime matches this +> integration's graph implementation. Native strict replay, PUF55 host +> acceptance and complete national/CD release acceptance remain separate +> work and are not established by these software checks. + + +Date: 2026-09-12. Worktree `~/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910`, +branch `microcosm-us-launch-verified-lanes-20260910` (PR #893's branch is +`microcosm-us-launch-integration-20260909`). Started at `069d5ed9a`. No push, no +new branch, no stash, `uv.lock` untouched. Every commit below is on the +checked-out branch. + +## 1. Outcome + +- `origin/main` (`0c3f4f651`, amendment 19) and + `origin/amend-keyed-seed-and-uniform-draws` (`23ba24770`, amendment 20 / PR + #912) are merged. **#912 was merged from its branch head**, so once it lands + the dispatcher should re-run `git merge origin/main`; for the graph and fit + packages, `docs/graph-interface.lock` and `docs/graph-acceptance.md` that is + expected to be a no-op (this tree already carries `23ba24770`'s bytes for all + of them, verified with `git diff --quiet 23ba24770 HEAD -- `). +- Every frozen or amended file equals `23ba24770` byte for byte: `decl.py`, + `kernel.py`, `keys.py`, `serialize.py`, `view.py`, `artifact_edges.py`, + `randomness.py`, `population.py`, `fit/qrf.py`, the lock, the charter, + `test_graph_kernel_contract.py`, and every `test_acceptance_*` file. The + interface-lock test passes (it is part of the 518-test graph run below). +- Four not-yet-amended graph pieces the US runtime consumes were re-applied + as separate named commits on top of main's files; four pieces with no + consumer were dropped (section 3). +- The five graph tests the brief named are green; the whole graph package is + green (518 passed); the fit package is green (191 passed); the spec-engine + check, `ci_test_groups --verify` and ruff are clean. Build-side results are + in section 5. +- The layer map (section 6) partitions the branch's remaining 407-file diff + against `origin/main` into the eleven staged layers, the later closure and + post-staging work, and the graph pieces, with per-layer file lists, test + files, import dependencies, and a proposed landing order. Section 7 has a + draft charter amendment entry per kept graph piece. Section 8 lists what + needs Max. + +Lane commits (newest first): + +``` +07566eb17 Re-pin the US implementation inventory for the reconciled graph package +c704a1be6 Pin that unreached propagates through a structural node and its version +2e1857a4b Bring the #893 reconciliation journal up to the piece commits +a16eeacf8 Re-apply the private population observer on run_graph +d1019762b Re-apply executor execution states: gate exceptions leave their consumers unreached +db1b7821a Re-apply complete Frame metadata storage and non-finite JSON refusal in the store +752ab840f Re-apply the raw-byte source codec on main's graph package +cff8fbf32 Re-pin the calibrate and simulate H1 parity cases the branch's kernels move +dc621c14c Re-pin the seed protocol and compiled seed map digests on the merged tree +051357909 Merge origin/amend-keyed-seed-and-uniform-draws (23ba24770, amendment 20) +3010b7788 Merge origin/main (0c3f4f651, amendment 19) into the #893 integration branch +3ad1b1aae Open the #893 reconciliation lane journal +``` + +## 2. The two merges and the per-file resolution + +Baseline before any change (`uv run --no-sync pytest packages/microcosm-graph/tests`): +5 failed, 362 passed, exit 1 — exactly the five the brief names +(`test_acceptance_b_ownership::test_b2_executor_enforces_ownership`, +`test_acceptance_h_parity::test_h1_kernel_parity`, +`test_graph_executor::test_fit_qrf_tolerance_source_hash_pin_is_current`, +`test_graph_kernel_contract::test_context_numerics_default_empty_and_carry_scopes`, +`test_graph_serialize::test_generated_parity_graphs_bind_real_kernels_and_direct_bytes`). + +### Merge 1: `git merge origin/main` (commit `3010b7788`) + +Git reported 8 content conflicts plus one in `PROGRESS.md`; `store.py`, +`__init__.py` and `PROGRESS.md` auto-merged. Resolution: + +| Path | Resolution | +| --- | --- | +| `packages/microcosm-graph/src/microcosm/graph/{artifact_edges,decl,executor,kernel,keys,manifest,serialize,view}.py` (conflicts) | main's bytes (`git checkout origin/main -- packages/microcosm-graph`) | +| `packages/microcosm-graph/src/microcosm/graph/{store,__init__}.py` (auto-merged) | main's bytes, same checkout; the branch's hunks return as pieces A and B | +| `packages/microcosm-graph/src/microcosm/graph/{attachments,availability,schema}.py` (branch-only) | removed in the merge (`git rm`); `availability.py` returns as piece C, the other two are dropped | +| `packages/microcosm-graph/src/microcosm/graph/randomness.py` (branch-only until amendment 20) | kept as the branch had it; replaced by `23ba24770`'s in merge 2 | +| `packages/microcosm-graph/tests/*` | main's bytes; the branch's copies were strict subsets of main's (numstat showed only removals), so this also dropped nothing of the branch's; `test_frame_metadata_store.py` and `test_lazy_snapshot_metadata_integration.py` (branch-only) removed here, the first returns with piece B, the second is dropped | +| `packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json` | main's (consistent with fit/qrf.py being main's in this merge) | +| `packages/microcosm-fit/src/microcosm/fit/qrf.py` (auto-merged to the branch's) | main's bytes; the branch's `predict_from_uniforms` returns with amendment 20 in merge 2 (the branch's qrf.py was byte-identical to `23ba24770`'s) | +| `packages/microcosm-fit/src/microcosm/fit/{_graph_legacy_apply,_graph_legacy_qrf,graph_legacy_apply,graph_legacy_apply_matrix,graph_legacy_qrf,graph_legacy_train,model_input,qrf_target}.py` (branch-only) | kept: not amended content, consumed by the US runtime (layer 01) | +| `docs/graph-interface.lock`, `docs/graph-acceptance.md` | main's bytes | +| `PROGRESS.md` | both sides kept: this lane's section, then main's Amendment 19 and #907 sections, then the branch's history | +| everything else | ordinary auto-merge | + +### Merge 2: `git merge 23ba24770` (commit `051357909`) + +Six conflicts: + +| Path | Resolution | +| --- | --- | +| `packages/microcosm-graph/src/microcosm/graph/randomness.py` (add/add) | `23ba24770`'s (adds the signed-zero normalisation the branch lacked) | +| `packages/microcosm-fit/tests/test_qrf_stateless.py` (add/add) | `23ba24770`'s (182 lines; the branch's 159-line copy was a strict subset) | +| `docs/evidence/spec-engine/us-f0-coverage.json` | `23ba24770`'s, then recomputed (section 4) | +| `packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py` (seed digests) | `23ba24770`'s, then recomputed (section 4) | +| `packages/microcosm-build/tests/test_spec_engine_country_bundles.py` (AM/BE/UK digests) | `23ba24770`'s; see section 5 for the merged-tree check | +| `packages/microcosm-build/tests/test_us_multispine_pool_tool.py` (pool tool `spec_sha256`) | `23ba24770`'s; see section 5 | + +After merge 2 the graph package, fit sources, lock and charter were +byte-identical to `23ba24770` except the eight branch-only fit modules above; +`docs/graph-interface.lock` matched `shasum -a 256` of the two frozen files. + +## 3. The not-yet-amended graph pieces + +Consumers were found by an AST scan of every `.py` under +`packages/microcosm-{build,calibrate,fit,frame,data}` and `tools/` for names +imported from `microcosm.graph.*` that `23ba24770`'s graph package does not +define, plus greps for the keyword-level uses (`_population_observer=`, +`population_retention=`, `emit_validation_artifact=`). + +### Kept (each re-applied on main's file as its own commit, with tests) + +| Piece | Commit | Consumers outside the graph package | Graph-level tests | +| --- | --- | --- | --- | +| A. `codecs.py` raw-byte mode: `SourceBytesCodec`, `register_bytes`/`load_bytes`, `raw-bytes-v1`, `RAW_BYTES_MAX_BYTES`, `load_source_bytes` (+ two `__init__` exports) | `752ab840f` | 13 modules: `graph_atomic_geography`, `graph_geography`, `graph_sources`, `graph_atomic_survey_population`, `graph_puf55_canonical_donor`, `puf_raw_source`, `puf_monetary_source`, `puf_monetary_agi_projection`, `puf_full_source_graph`, `atomic_block_sources`, `atomic_block_api_sources`, `survey_atomic_geography`, `puf55_survey_recipients`; 8 US test files | the branch had none; 4 added to `test_graph_codecs.py` (8 passed) | +| B. `store.py` Frame-metadata storage (`microcosm-graph-frame-v2`, `_encode/_decode_frame_metadata`, `frame_metadata_sha256`, `_put(validate_existing=…)`) and the non-finite JSON decode hooks | `db1b7821a` | `survey_population_replay.same_replayed_frame`, `population_input_coverage`, `graph_full_puf_enrichment` (all call `_encode_frame_metadata`), and every US stage relying on `Frame.metadata` surviving a store round trip | `test_frame_metadata_store.py` (the branch's file, 19 collected tests) | +| C. `availability.py` + executor/manifest execution states: `gate_exception`, `unreached`, `blocked_by`, cache-record schema 3, manifest schema 4, refusal of kernel-authored execution metadata; replaces amendment 19's refusal of a gate declaring a typed output | `d1019762b` | `graph_atomic_survey_clone` (`atomic_geography_nodes(..., emit_validation_artifact=True)` makes the geography gate declare a typed output, which main's executor refuses outright), its consumers `survey_age_calibration`, `graph_current_survey_predictors`, `survey_origin_budget`; `population_input_coverage` (`execution_state` on every producer receipt); build test `test_atomic_geography.py` parametrises the declaration | the branch had none; main's `test_a_gate_kernel_may_not_declare_a_typed_artifact_output` replaced by 6 tests in `test_graph_executor.py` (five in `d1019762b`, the structural-propagation one in `c704a1be6`) | +| D. `run_graph(_population_observer=…)` | `a16eeacf8` | `survey_age_calibration`, `graph_atomic_survey_financial`, `graph_survey_population`, `graph_atomic_survey_population`; 8 US test files | the branch had none; 1 added | + +What was deliberately not re-applied inside those files: the branch's +docstring deletions on the amended surface; its `_opaque_artifact_key` +inline hash (main's `graph_keys.opaque_artifact_key` is the same +derivation); its `KernelContext` argument order (frozen file); its eager +artifact payload reads on cache hits (main authenticates edges without +reading, `9018c4420`); its `StoreMiss`-on-hit for a missing declared artifact +(main's `5c4a8efde` decision, `StoreCorrupt`); the removal of main's #907 +datetime64/timedelta64 object-leaf refusal in `store.py`; and main's +`_validate_typed_ancestry` gate-ancestry shape check and the +`NodeRejectedError`→`ValueError` conversion (`402d9a631`, `7d45c32e5`), which +the branch's older copy lacked. + +### Dropped (no consumer left) + +| Piece | Why dropped | +| --- | --- | +| `attachments.py`, `_PopulationRetention`, `_LazyPopulations`, `run_graph(population_retention="lazy")`, `PopulationView.__slots__ = ("__weakref__",)` (staged as layer 08 GRAPH-ATTACHMENT-METADATA) | no call site outside the graph package passes `population_retention`; its only consumer was the branch's own `test_lazy_snapshot_metadata_integration.py`, dropped with it. It is a memory optimisation for long graphs (the native pilot's 11.5 GB peak is in the PR body); if it is wanted it is a self-contained later PR on top of piece B, which it imports. | +| `schema.py` (`graph_schema`, 316 lines) | imported by nothing in the repository (the `graph_schema_version` hits in the spec bundle are unrelated). | +| `keys.py` `_stream_file` chunked source hashing | no consumer and no test; it changes no identity (same digest sequence, it only bounds memory and refuses a file that changes while hashed). Main's `keys.py` is kept byte-identical. Candidate for a small standalone PR if large-source memory matters. | +| the `_write_node` per-coordinate memory refactor and the `del series` | no consumer; behaviour-preserving; main's amended `_write_node` kept. | + +## 4. Identities re-pinned on the merged tree, and why + +- **Seed protocol and compiled seed map digests** (`inventory_coverage.py` + `EXPECTED_HASHES`, `docs/evidence/spec-engine/us-f0-coverage.json`; + commit `dc621c14c`). Both hash the exact source bytes of the attested kernel + modules in `spec_engine/seeds.py` (no `microcosm.graph` module is on that + list, so the graph pieces do not move them). The merged tree carries the + branch's `acs_transfer` and `housing_inputs` changes and, through amendment + 20, the same `fit.qrf` bytes the branch already carried, so the actual + digests are the values the branch pinned in `1734b9e90` + (`a775cccd…` / `c15bff65…`), not `23ba24770`'s (computed on a tree without + the two US kernel changes). Computed directly (`spec.seed_protocol.implementation_sha256`, + `sha256_json(compiled.seed_stream_map.to_wire())`), then the report was + regenerated with `tools/spec_engine_coverage.py` and `--check` exits 0. +- **H1 parity pins for `calibrate` and `simulate`** (commit `cff8fbf32`). + Six graph tests were red on the merged tree, all on these two cases. + `CalibrateKernel.implementation_hash` binds `microcosm.calibrate`'s solve and + diagnostics modules, which the branch's grouped-bound solver path changes + (layer 05); `SimulateRulesKernel.implementation_hash` binds + `microcosm.frame.bundle`, to which the branch adds `Frame.__reduce__` (layer + 01). Re-recorded with `tools/graph_parity_repin.py calibrate` and + `… simulate`, which keep every pinned platform (both cases pin only + `arm64/darwin/py3.14`) and leave `direct.csv` untouched. The `fit.qrf` pin + came from `23ba24770` and needed no move. +- **The US implementation inventory** (`us_runtime/graph_implementation_inventory.json`; + commit `07566eb17`). This reviewed dependency fence is checked by every US + source stage before it runs (`graph_implementation.implementation_manifest`), + and its refusal surfaces as `PREPARATION_ISSUANCE_REFUSED` (the preparation + wraps every exception; the true cause was recovered from the exception's + `__context__`). Three entries no longer held: the `microcosm.graph` roster + listed the dropped `attachments.py` and `schema.py` (39 survey tests red on + that alone); the contracts for `executor.py`, `manifest.py`, `keys.py` and + `population.py` reflected the branch's versions rather than main's plus the + kept pieces, and `attachments.py` was a listed stage module in ten stages; + and `congressional_district_vintage.py`'s contract predated main's + `05ff48f05` (merged into the branch on 2026-09-11), which added the + `microcosm.calibrate.geography_constants` import — an inherited drift the + branch already carried at `069d5ed9a`, verified by recomputing the + pre-merge tree's contracts against its own inventory. Recomputed with the + module's own `_dependency_contract` / `_covered_imports`; the import + classification set is the exact union again. Seven of the ten stages now + build their manifest; three cannot (section 8.5). +- **Country-bundle digests and the pool tool's `spec_sha256`**: resolved to + `23ba24770`'s values in merge 2; their merged-tree check is in section 5. + +## 5. Commands, exit codes, counts + +All from the worktree root with the locked engine environment +(`uv sync --all-packages --locked --extra us --extra uk` → exit 0 before any +change). Tests ran with `uv run --no-sync pytest … -p no:cacheprovider`. + +| Stage | Command | Result | +| --- | --- | --- | +| baseline (`069d5ed9a`) | `pytest packages/microcosm-graph/tests` | 5 failed, 362 passed, exit 1 | +| baseline | `python tools/ci_test_groups.py --verify` | exit 0 | +| after both merges (`051357909`) | `pytest packages/microcosm-graph/tests` | 6 failed, 485 passed, exit 1 (calibrate/simulate parity pins; section 4) | +| after both merges | `pytest packages/microcosm-fit/tests` | 191 passed, exit 0 | +| after both merges | `python tools/spec_engine_coverage.py --check` | exit 1 (`seed protocol content digest differs`, `compiled seed map digest differs`) | +| after both merges | `python tools/ci_test_groups.py --verify` | `verification=ok`, exit 0 | +| after both merges | `ruff check .` | All checks passed, exit 0 | +| seed re-pin (`dc621c14c`) | `python tools/spec_engine_coverage.py` then `--check` | exit 0 / exit 0 | +| parity re-pin (`cff8fbf32`) | `python tools/graph_parity_repin.py calibrate`; `… simulate` | exit 0 / exit 0 | +| parity re-pin | `pytest test_acceptance_h_parity.py test_graph_parity_pins.py test_graph_parity_repin.py test_graph_serialize.py` | 27 passed, exit 0 | +| piece A (`752ab840f`) | `pytest packages/microcosm-graph/tests/test_graph_codecs.py` | 8 passed, exit 0 | +| piece B (`db1b7821a`) | `pytest test_frame_metadata_store.py test_graph_store.py test_graph_population.py test_graph_executor.py test_graph_manifest.py` | 274 passed, exit 0 | +| piece C (`d1019762b`) | `pytest packages/microcosm-graph/tests` (whole package, includes the interface-lock test) | **518 passed, exit 0** | +| piece C | `ruff check .`; `ruff format --check packages/microcosm-graph` | exit 0 / exit 0 | +| piece D (`a16eeacf8`) | `pytest test_graph_executor.py test_acceptance_b_ownership.py test_acceptance_f_gates.py` | 94 passed, exit 0 | +| structural test (`c704a1be6`) | `pytest packages/microcosm-graph/tests/test_graph_executor.py` | 82 passed, exit 0 | +| inventory re-pin (`07566eb17`) | `pytest test_us_survey_origin_budget.py::test_actual_initial_budget_and_zero test_us_survey_age_calibration_run.py::test_actual_seven_node_calibration_preserves_full_population` (two of the 39 formerly refused) | 2 passed, exit 0 | +| final tree | `python tools/ci_test_groups.py --verify` | exit 0 | +| final tree | `python tools/spec_engine_coverage.py --check` | exit 0 | + +Build-side and shared consumers, final tree (`07566eb17`). A first battery at `a16eeacf8` found 39 survey tests refused (`PREPARATION_ISSUANCE_REFUSED`) — all traced to the implementation inventory above, re-pinned in `07566eb17` — and one non-existent file name in my own raw-bytes group list (`test_us_puf_raw_source.py`, removed); the battery was then rerun in full: + +_(Historical draft marker resolved: task `bt0nx8fcc` was killed during availability consumers. No final battery counts or exit code exist; later groups did not start.)_ + +## 6. Layer map: landing #893 as sequential PRs + +Basis: `git diff --name-status origin/main HEAD` = 407 files (340 added, 67 +modified, 0 deleted), each attributed to the first branch commit that touched +it (`git log --name-only --no-merges 3094bfe84..HEAD`), with the eleven +`Stage integration layer NN` commits mapped to their layer names, the later +staging commits to layers 12–14, the sixty post-staging commits of 10–12 +September grouped by subject (P1–P6), and the graph pieces of section 3 as +their own layer. Import dependencies come from an AST scan of every changed +`.py` file resolved against that attribution (`microcosm.*` imports only). +The scripts and their JSON output are in this lane's scratchpad +(`layers.py`, `layers2.py`, `layers.json`, `per-file-deps.json`). + +Two corrections to the staged layer names as they sit in git history: + +- Layer 08 GRAPH-ATTACHMENT-METADATA is empty in the map: its only file, + `attachments.py`, is dropped (section 3). +- Layer 09 F-JOINT-GEOGRAPHY-GATE has no file of its own: its 27-line change + lives in `graph_geography.py`, which layer 01 created, so it lands with + layer 01. +- Nineteen files differ from `origin/main` only because `origin/main` does not + yet carry amendment 20 (`kernel.py`, `randomness.py`, `fit/qrf.py`, + `test_graph_kernel_contract.py`, `test_graph_parity_*`, + `test_graph_randomness.py`, `tools/graph_parity_repin.py`, the three + `pins.json`, the lock, the charter, the amendment-20 journal/receipts/ + changelog fragment) or were re-pinned by this lane. They vanish from the diff + when #912 lands and are not #893 content; they are listed under + "already on main's way" below and belong to no PR. + +### Dependency facts the order rests on + +Three kinds of link were measured, strongest first: + +1. **Hard**: a file imports a module that does not exist on `origin/main` + (a file the branch adds). These force order. +2. **Name-hard**: a file imports, from a module main already has, a *name* + that main's version of that module does not define (checked by parsing + `git show origin/main:`). Only three survive after discarding + submodule imports that merely look like names: `GroupedUpperBounds` + (layer 03's two-line re-export in `calibrate/__init__.py` of layer 01's + `group_bounds.py`; imported by layer 12's and 13's tests), + `decode_us_puma_ladder` (layer 06's `puma_ladder.py`; imported by layer + 13's `test_us_geography_integration.py`), and the graph pieces' names + (`load_source_bytes`, `load_raw_bytes`, `RAW_BYTES_MAX_BYTES`, + `_encode_frame_metadata`, `execution_state`). +3. **Soft**: a file imports a module main already has that the branch also + modifies (`frame_checkpoint`, `asec_checkpoint`, `operator_boundary`, + `us_runtime/__init__`, `calibrate/__init__`, `solve.py`, `bundle.py`, + `variable_labels.py`, and amendment 20's `qrf.py`/`kernel.py`). These do + not force order by themselves; each PR's own test run is what proves the + modified behaviour is or is not needed. + +Hard links between layers (source and tests): + +| Layer | hard on | of which tests only | +| --- | --- | --- | +| 01 SAFE-ADDITIVE | nothing | — | +| 03 ACCEPTED-SHARED-RESTORE | 01 (`group_bounds`, `reported_coverage_source`, `_person_signal_summary`, `operator_column_contracts`, `table_identity`), P3 (`asec_raw_stage_v4`) | P3 link is `test_us_asec_checkpoint.py` only | +| 04, 06 | nothing hard (soft on 01/03) | — | +| 05 SOLVE-MERGE-PROPOSAL | 01 (`group_bounds`) | — | +| 07 F-CATALOGUE-OPTIMIZATION | 01 (`survey_population_preparation`) | all three files are tests | +| 10 SOURCE-CLOSURE | G1 (soft) | — | +| 11 PLACEMENT-ADDITIONS | 01 (the eight fit modules, `full_puf_enrichment`, `graph_sources`, `survey_population_replay`), G1, G2 | — | +| 12 ORDINARY-CLOSURE | 01 (fit modules), 03 (`GroupedUpperBounds`) | all files are tests/fixtures | +| 13 INTEGRATION-REGRESSIONS | 01 (`graph_geography`), 03 (`GroupedUpperBounds`), 06 (`decode_us_puma_ladder`) | all three are tests | +| P1 POST-CLONE-ATOMIC-GEOGRAPHY | 01 (`randomness` = amendment 20, fit modules, `graph_combined_clone`, `graph_sources`, `graph_survey_population`, `survey_*`), G1, G3 (the gate's declared artifact), P2 | P2 link is two tests (`test_us_current_survey_predictor_demographics.py`, `test_us_survey_origin_budget_predictor_compatibility.py`) | +| P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR | 01 (`model_input`, `_graph_legacy_qrf`, `graph_legacy_*`, and eleven test-side modules), 11 (`graph_full_puf_enrichment`, two tests), P1 (`graph_atomic_survey_financial`, one test), G1 | 11 and P1 links are tests only | +| P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET | 01 (`reported_coverage_source` from `asec_raw_stage_v4`; six test-side modules), G2, G3 (`population_input_coverage`), P2 (`asec_demographic_source`) | P2 link is `test_us_asec_demographic_source.py` only | +| P4 | nothing hard (soft on 03) | — | +| P5 | nothing hard (soft on 01/03/04/06) | — | +| P6 UK-ADAPTER-DOCS-AND-RECORDS | P1 (`atomic_geography`) | — | + +Two cycles exist and both are test-only: P1↔P2 (three tests) and 03↔P3 +(one test). Every source-level dependency is acyclic. + +### Landing order + +1. **PR-G1 raw-byte codec** — piece A (`codecs.py`, `__init__.py`, + `test_graph_codecs.py`). No dependency. Charter entry 7.1. +2. **PR-G2 Frame metadata storage** — piece B (`store.py`, + `test_frame_metadata_store.py`, + `changelog.d/20260906-frame-metadata-storage.fixed.md`). No dependency. + Charter entry 7.2. +3. **PR-G3 execution states** — piece C (`availability.py`, `executor.py`, + `manifest.py`, `test_graph_executor.py`). No code dependency, but it + supersedes an amendment-19 ruling: **needs Max's decision (section 8) + before it is opened.** Charter entry 7.3. +4. **PR-G4 population observer** — piece D (`executor.py`, + `test_graph_executor.py`). Stacks on G3 only because both edit + `executor.py`; G3+G4 can be one PR. Charter entry 7.4. +5. **PR-1 SAFE-ADDITIVE** — all of layer 01: 101 non-test files (100 modules + plus `graph_implementation_inventory.json`; `randomness.py` is excluded, it + is amendment 20) and its 26 test files, plus layer 03's two-line + `calibrate/__init__.py` re-export of `GroupedUpperBounds` (it belongs with + `group_bounds.py`) and layer 09's 27 lines (already inside + `graph_geography.py`), and `graph_implementation_inventory.json`, which + pins the `microcosm.graph` roster and module contracts and therefore has + to be re-pinned against whatever main's graph package is when PR-1 opens + (`07566eb17` shows the procedure). Hard on G1, G2 and G4 only (`_population_observer` + in `survey_age_calibration` and `graph_survey_population`). Its 21 soft + links are listed in the scratchpad `pr1-refined.json`; the PR's gate is + its own 26 test files on main + G1–G4 + PR-1. +6. **PR-2 US runtime restore** — layers 03 (without the `__init__` re-export + moved to PR-1, and with `asec_raw_stage_v4.py` moved in from P3 so + `test_us_asec_checkpoint.py` runs), 04 (`puf_support.py`), 05 + (`calibrate/solve.py`), 06 (`us_runtime/__init__.py`, `puma_ladder*.py`, + `tools/build_us_puma_ladder_artifact.py`, six tests), 12's calibrate + tests and fixture, 13's three regressions and `docs/us-launch-review.md`, + 14 (`dfa7f872c`), and the two re-pins that these files move: the + calibrate H1 pin (`solve.py`) and the seed digests (`acs_transfer.py`, + `housing_inputs.py`). 03/04/05/06 touch disjoint files and could be four + PRs (05 first, since only it is hard on 01); they are one here because + their tests import across all four. +7. **PR-3 catalogue, source closure, placement** — layers 07 (three digest + tests + fragment), 10 (`graph_acs_housing_universe.py`, eight JSON source + definitions), 11 (`graph_full_puf_enrichment.py` + test), and 12's fit + tests (`test_graph_legacy_*`, `test_qrf_target.py`, + `legacy-slice-apply-v1.json`). Hard on PR-1 and G1/G2. +8. **PR-4 calibration diagnostics** — P4 (`variable_labels.py`, its test, + fragments, experiments). Soft on PR-2. +9. **PR-5 survey population, catalogue and budget** — P3 minus + `asec_raw_stage_v4.py`: `common_frame_export_contract`, + `input_coverage_profile`, `population_input_coverage`, + `frame/bundle.py`'s `__reduce__` (carries the simulate H1 re-pin), ten + tests including `test_us_spine_blindness.py`; + `test_us_asec_demographic_source.py` rides with PR-7 instead. Hard on + PR-1, G2, G3. +10. **PR-6 post-clone atomic geography** — P1 (`atomic_geography.py`, + `graph_atomic_geography.py`, `atomic_block_*`, `graph_atomic_survey_*`, + `survey_atomic_geography.py`, `current_survey_geography.py`, + `tools/ci_test_groups.py`, fourteen tests); its two P2-importing tests + ride with PR-7. Hard on PR-1, G1, G3. +11. **PR-7 PUF55 two-route and canonical donor** — P2 (nineteen modules, + twenty-one tests plus the three deferred from PR-5/PR-6, PUF docs and + growth evidence, `pyproject.toml`'s `pythonpath` line). Hard on PR-1, + PR-3 (11), PR-5, PR-6. +12. **PR-8 CI, seed-identity diagnostics and spine blindness** — P5 + (`.github/workflows/test.yml`, `tools/spec_seed_identity_diagnostics.py`, + the seven `test_spec_seed_identity_*` files, `CLAUDE.md`, + `test_spec_engine_loader.py`, `test_us_puf_support.py`, + `test_us_stacked_spine.py`, the AM/BE/UK bundle digests). Soft on PR-2 + and PR-5; last because the diagnostic's roster names modules from every + earlier PR. +13. **PR-9 UK adapter, docs and records** — P6 (`uk_runtime/atomic_area_support.py`, + `atomic_household_identity.py`, two tests, `.gitattributes`, records). + Hard on PR-6. + +Every `experiments/` record and `changelog.d/` fragment rides with the PR +whose code it describes (listed per layer below). The "already on main's +way" files (amendment 20, listed at the top of this section) belong to no PR. + +### Per-layer file lists + +#### 00 JOURNAL (1 non-test files, 0 test files) + +- `./`: `PROGRESS.md` + +#### 01 SAFE-ADDITIVE (101 non-test files, 26 test files) + +- `packages/microcosm-build/src/microcosm/build/`: `survey_allocation.py`, `survey_domain_sample.py`, `table_identity.py` +- `packages/microcosm-build/src/microcosm/build/cd_benchmark/`: `__init__.py`, `canonical.py`, `origin.py`, `protocol.py`, `reasons.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `_asec_current_money_codec.py`, `_person_signal_summary.py`, `acs_housing_universe.py`, `acs_housing_universe_source.py`, `acs_native_coverage_binding.py`, `acs_person_coverage_authentication.py`, `acs_person_coverage_columns.py`, `acs_population_catalogue.py`, `asec_2024_native_population.py`, `asec_coverage_authentication.py`, `asec_current_money.py`, `asec_current_money_graph_resources.py`, `asec_current_money_resources.py`, `asec_current_money_selection.py`, `asec_current_money_source.py`, `asec_current_money_units.py`, `asec_engine_evaluation.py`, `asec_household_coverage_fields.py`, `asec_household_observations.py`, `asec_housing_status.py`, `asec_housing_status_source.py`, `asec_housing_universe.py`, `asec_housing_universe_source.py`, `asec_income_observations.py`, `asec_original_household_weights.py`, `asec_person_coverage_source.py`, `asec_person_income_source.py`, `asec_population_catalogue.py`, `asec_prepared_source.py`, `asec_student_controls.py`, `cd_reference.py`, `cd_reference_sources.py`, `cps_carried_current.py`, `demographic_calibration_graph.py`, `full_puf_enrichment.py`, `graph_asec_income.py`, `graph_asec_prepared.py`, `graph_combined_clone.py`, `graph_composed_asec_binding.py`, `graph_composed_asec_measures.py`, `graph_composed_contracts.py`, `graph_composed_population.py`, `graph_context.py`, `graph_current_survey_puf_transfer.py`, `graph_geography.py`, `graph_housing_universe.py`, `graph_implementation.py`, `graph_implementation_inventory.json`, `graph_national_age_counts.py`, `graph_native_household_origin.py`, `graph_native_origin_implementation.py`, `graph_puf_detail_transfer.py`, `graph_puf_diagnostic_consumer.py`, `graph_sources.py`, `graph_survey_age_artifact.py`, `graph_survey_budget.py`, `graph_survey_calibration.py`, `graph_survey_population.py`, `national_age_activation.py`, `native_household_origin.py`, `operator_column_contracts.py`, `puf_detail_transfer.py`, `puf_diagnostic_consumer.py`, `puf_growth.py`, `puf_growth_graph.py`, `puf_monetary_agi_projection.py`, `puf_monetary_source.py`, `puf_price_baseline.py`, `puf_raw_source.py`, `reported_coverage_source.py`, `source_csv_builtin.py`, `survey_age_activation.py`, `survey_age_calibration.py`, `survey_age_sources.py`, `survey_calibration_diagnostics.py`, `survey_catalogue_selection.py`, `survey_observed_age.py`, `survey_origin_budget.py`, `survey_population_domains.py`, `survey_population_preparation.py`, `survey_population_replay.py` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `group_bounds.py` +- `packages/microcosm-fit/src/microcosm/fit/`: `_graph_legacy_apply.py`, `_graph_legacy_qrf.py`, `graph_legacy_apply.py`, `graph_legacy_apply_matrix.py`, `graph_legacy_qrf.py`, `graph_legacy_train.py`, `model_input.py`, `qrf_target.py` +- `packages/microcosm-frame/src/microcosm/frame/adapters/`: `_policyengine_us_source_index.py`, `policyengine_us.py` +- `packages/microcosm-graph/src/microcosm/graph/`: `randomness.py` +- tests: `test_us_acs_housing_source.py`, `test_us_acs_person_coverage_authentication.py`, `test_us_acs_person_coverage_columns.py`, `test_us_acs_population_catalogue.py`, `test_us_asec_2024_native_population.py`, `test_us_asec_coverage_authentication.py`, `test_us_asec_current_money_source.py`, `test_us_asec_person_income_source.py`, `test_us_current_survey_puf_host.py`, `test_us_current_survey_puf_transfer.py`, `test_us_full_puf_enrichment.py`, `test_us_graph_survey_population.py`, `test_us_national_age_counts.py`, `test_us_puf_detail_transfer.py`, `test_us_puf_price_baseline.py`, `test_us_survey_age_activation.py`, `test_us_survey_age_artifact.py`, `test_us_survey_age_calibration_run.py`, `test_us_survey_age_development.py`, `test_us_survey_age_sources.py`, `test_us_survey_calibration.py`, `test_us_survey_observed_age.py`, `test_us_survey_origin_budget.py`, `test_us_survey_population_preparation.py`, `test_us_survey_population_replay.py`, `test_policyengine_us_ownership_index.py` +- source imports from: 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece), P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET, P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS +- tests import from: 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) + +#### 02 GRAPH-RESTORE (1 non-test files, 0 test files) + +- `packages/microcosm-graph/src/microcosm/graph/`: `kernel.py` + +#### 03 ACCEPTED-SHARED-RESTORE (20 non-test files, 2 test files) + +- `packages/microcosm-build/src/microcosm/build/`: `frame_checkpoint.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `acs_inputs.py`, `acs_pums.py`, `acs_transfer.py`, `asec_checkpoint.py`, `congressional_district_vintage.py`, `cps_carried.py`, `eligibility_inputs.py`, `hours_worked.py`, `housing_inputs.py`, `multispine_pool.py`, `operator_boundary.py`, `prior_year_income.py`, `puf_capital_gains_tail.py`, `qbi_inputs.py`, `relationship_inputs.py`, `spine_assembly.py`, `stacked_spine.py` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `__init__.py` +- `packages/microcosm-fit/src/microcosm/fit/`: `qrf.py` +- tests: `test_us_acs_pums.py`, `test_us_asec_checkpoint.py` +- source imports from: 01 SAFE-ADDITIVE, 04 PUF-SUPPORT-MERGE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS +- tests import from: 01 SAFE-ADDITIVE, 06 J-GRAPH-COMPATIBILITY, P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET + +#### 04 PUF-SUPPORT-MERGE (1 non-test files, 0 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `puf_support.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE + +#### 05 SOLVE-MERGE-PROPOSAL (1 non-test files, 0 test files) + +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `solve.py` +- source imports from: 01 SAFE-ADDITIVE + +#### 06 J-GRAPH-COMPATIBILITY (4 non-test files, 6 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `__init__.py`, `puma_ladder.py`, `puma_ladder_sources.py` +- `tools/`: `build_us_puma_ladder_artifact.py` +- tests: `test_us_acs_multispine.py`, `test_us_acs_multispine_legacy_builder.py`, `test_us_base_pool.py`, `test_us_multispine_pool_tool.py`, `test_us_puma_ladder.py`, `test_us_puma_ladder_sources.py` +- tests import from: 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE + +#### 07 F-CATALOGUE-OPTIMIZATION (1 non-test files, 3 test files) + +- `changelog.d/`: `us-survey-catalogue-digest.changed.md` +- tests: `test_us_survey_catalogue_digest.py`, `test_us_survey_catalogue_digest_authority.py`, `test_us_survey_catalogue_digest_fast_path.py` +- tests import from: 01 SAFE-ADDITIVE, 06 J-GRAPH-COMPATIBILITY + +#### 10 SOURCE-CLOSURE (9 non-test files, 0 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `acs_2024_housing_universe.json`, `asec_current_money_consumers_v1.json`, `asec_current_money_domains_v1.json`, `asec_current_money_price_basis_v1.json`, `graph_acs_housing_universe.py`, `native_origin_graph_inventory.json`, `puf_2015_monetary_agi_source_projection.json`, `puf_2015_monetary_source_projection.json`, `puf_2015_raw_source_definition.json` +- source imports from: G RAW-BYTES-CODEC (graph piece) + +#### 11 PLACEMENT-ADDITIONS (1 non-test files, 1 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `graph_full_puf_enrichment.py` +- tests: `test_us_graph_full_puf_enrichment.py` +- source imports from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### 12 ORDINARY-CLOSURE (0 non-test files, 8 test files) + +- tests: `legacy_prepatch.json`, `test_group_bounds.py`, `test_grouped_fixed_support.py`, `legacy-slice-apply-v1.json`, `test_graph_legacy_matrix.py`, `test_graph_legacy_qrf.py`, `test_qrf_stateless.py`, `test_qrf_target.py` +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, G RAW-BYTES-CODEC (graph piece) + +#### 13 INTEGRATION-REGRESSIONS (2 non-test files, 3 test files) + +- `changelog.d/`: `us-launch-integration.fixed.md` +- `docs/`: `us-launch-review.md` +- tests: `test_us_geography_integration.py`, `test_us_runtime_facade_union.py`, `test_solve_us_launch_integration.py` +- tests import from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### Already on main's way (amendment 20 / this lane's re-pins; not #893 content) (10 non-test files, 9 test files) + +- `./`: `PROGRESS-amendment-20-keyed-draws.md` +- `changelog.d/`: `amend-keyed-seed-and-uniform-draws.added.md` +- `docs/`: `graph-acceptance.md`, `graph-interface.lock` +- `experiments/`: `amendment-20-keyed-draws-receipts.md`, `us-native-atomic-financial-required-replay-20260910.json`, `us-puf-donor-boundaries-20260910.json`, `us-puf55-public-recipient-controls-20260910.json`, `us-puf55-two-route-numerical-20260910.json` +- `tools/`: `graph_parity_repin.py` +- tests: `test_us_puf55_model_boundaries.py`, `test_kernels.py`, `pins.json`, `pins.json`, `pins.json`, `test_graph_kernel_contract.py`, `test_graph_parity_pins.py`, `test_graph_parity_repin.py`, `test_graph_randomness.py` +- source imports from: G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### G RAW-BYTES-CODEC (graph piece) (2 non-test files, 1 test files) + +- `packages/microcosm-graph/src/microcosm/graph/`: `__init__.py`, `codecs.py` +- tests: `test_graph_codecs.py` +- tests import from: G STORE-FRAME-METADATA (graph piece) + +#### G STORE-FRAME-METADATA (graph piece) (2 non-test files, 1 test files) + +- `changelog.d/`: `20260906-frame-metadata-storage.fixed.md` +- `packages/microcosm-graph/src/microcosm/graph/`: `store.py` +- tests: `test_frame_metadata_store.py` + +#### P1 POST-CLONE-ATOMIC-GEOGRAPHY (38 non-test files, 16 test files) + +- `changelog.d/`: `893-population-only-block-source.added.md`, `893-source-demographic-conditioning.added.md`, `893-survey-atomic-prefix.added.md`, `shared-atomic-geography.added.md`, `us-atomic-financial-composition.added.md`, `us-national-atomic-support.added.md` +- `experiments/`: `us-atomic-age-v2-1-controls-20260909.json`, `us-atomic-block-adapter-20260909.json`, `us-atomic-block-api-sources-35-controls-20260909.json`, `us-atomic-block-sources-36-controls-20260909.json`, `us-atomic-budget-semantic-8-controls-20260909.json`, `us-atomic-clone-graph-20260909.json`, `us-atomic-financial-4-controls-20260909.json`, `us-atomic-financial-composition-acceptance-20260909.json`, `us-atomic-financial-corrected-2-controls-20260909.json`, `us-atomic-national-acquisition-20260909.json`, `us-atomic-native-de-1-control-20260909.json`, `us-atomic-native-de-corrected-1-control-20260909.json`, `us-atomic-native-national-1-control-20260909.json`, `us-atomic-survey-population-controls-20260909.json`, `us-budget-predictor-compatibility-1-controls-20260909.json`, `us-financial-demographics-3-controls-20260909.json`, `us-postclone-geography-70-controls-20260910.json`, `us-puf55-native-donor-20260909.json`, `us-survey-geography-graph-controls-20260909.json`, `us-survey-geography-source-controls-20260909.json` +- `packages/microcosm-build/src/microcosm/build/`: `atomic_geography.py`, `graph_atomic_geography.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `atomic_block_api_sources.py`, `atomic_block_sources.py`, `atomic_block_support.py`, `current_survey_geography.py`, `graph_atomic_survey_clone.py`, `graph_atomic_survey_financial.py`, `graph_atomic_survey_population.py`, `graph_current_survey_geography.py`, `survey_atomic_geography.py` +- `tools/`: `ci_test_groups.py` +- tests: `test_atomic_geography.py`, `test_us_atomic_block_api_sources.py`, `test_us_atomic_block_api_sources_native_de.py`, `test_us_atomic_block_api_sources_native_national.py`, `test_us_atomic_block_sources.py`, `test_us_atomic_block_support.py`, `test_us_atomic_survey_clone_graph.py`, `test_us_current_survey_geography.py`, `test_us_current_survey_predictor_demographics.py`, `test_us_graph_atomic_survey_financial.py`, `test_us_graph_atomic_survey_population.py`, `test_us_graph_current_survey_geography.py`, `test_us_postclone_geography_identity.py`, `test_us_survey_age_calibration_atomic_geography.py`, `test_us_survey_origin_budget_atomic_geography.py`, `test_us_survey_origin_budget_predictor_compatibility.py` +- source imports from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR + +#### P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR (51 non-test files, 21 test files) + +- `./`: `pyproject.toml` +- `changelog.d/`: `893-age-count-fixture.internal.md`, `893-diagnostic-refusal-context.internal.md`, `893-historical-zero-provenance.internal.md`, `puf55-canonical-donor.added.md`, `puf55-survey-social-security.added.md`, `survey-social-security-source.added.md`, `us-puf55-canonical-and-output-seals.added.md`, `us-puf55-survey-ss-measurement.added.md` +- `docs/`: `current-survey-puf59-progress.md`, `geography-assignment.md`, `puf2015-canonical59-and-growth.md`, `puf55-survey-ss-measurement-decision.md`, `survey-social-security-source.md`, `us-uk-release-path.md` +- `docs/evidence/puf2015-target2024/`: `GROWTH-RECIPE.json`, `INDEX-VALUES.json`, `NATIONAL-GROWTH-EXTRACT.json`, `PUBLIC-WORKBOOK-SOURCES.json` +- `experiments/`: `us-acs-compilation-cache-adoption-20260910.json`, `us-budget-detached-view-correction-20260910.json`, `us-financial-fixture-profile-20260910.json`, `us-financial-successor-positive-1-limit-stop-20260909.json`, `us-financial-successor-positive-20260910.json`, `us-financial-successor-remaining-controls-20260910.json`, `us-puf55-canonical-create-47-controls-20260910.json`, `us-puf55-numerical-output-seal-46-controls-20260910.json`, `us-puf55-population-controls-20260909.json`, `us-puf55-public-recipient-positives-20260910.json`, `us-puf55-survey-ss-measurement-31-controls-20260909.json`, `us-puf55-two-route-values-controls-20260910.json`, `us-survey-age-development-20260909.json` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `asec_demographic_source.py`, `current_asec_demographics.py`, `current_social_security_source.py`, `current_survey_predictors.py`, `graph_current_survey_predictors.py`, `graph_puf55_canonical_donor.py`, `graph_puf55_survey_recipients.py`, `puf55_canonical_donor.py`, `puf55_route_finalization.py`, `puf55_survey_recipients.py`, `puf55_survey_ss_measurement.py`, `puf59_canonical.py`, `puf59_canonical_artifact.py`, `puf_full_source.py`, `puf_full_source_graph.py`, `puf_qbi_model.py`, `puf_target2024_growth.py`, `survey_financial_successor.py`, `survey_social_security.py` +- tests: `test_frame_checkpoint.py`, `test_spec_seed_diagnostic_refusal_context.py`, `test_us_acs_source_compile_cache.py`, `test_us_current_asec_demographics.py`, `test_us_current_survey_predictors.py`, `test_us_full_puf_output_profiles.py`, `test_us_graph_puf55_canonical_donor.py`, `test_us_graph_puf55_survey_recipients.py`, `test_us_graph_puf55_survey_ss.py`, `test_us_plan.py`, `test_us_puf55_route_finalization.py`, `test_us_puf55_route_numerical_finalization.py`, `test_us_puf55_survey_recipients.py`, `test_us_puf55_survey_ss_measurement.py`, `test_us_puf55_survey_ss_profile.py`, `test_us_puf59_canonical.py`, `test_us_puf_target2024_growth.py`, `test_us_runtime_import_order.py`, `test_us_survey_financial_successor.py`, `test_us_survey_origin_budget_detached_views.py`, `test_us_survey_social_security.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, 11 PLACEMENT-ADDITIONS, G RAW-BYTES-CODEC (graph piece), P1 POST-CLONE-ATOMIC-GEOGRAPHY, P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET + +#### P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET (16 non-test files, 10 test files) + +- `changelog.d/`: `common-frame-export-contract.added.md`, `us-survey-copy-and-identity.fixed.md` +- `docs/`: `us-input-coverage-diagnostic.md` +- `experiments/`: `us-acs-code-memo-comparison-20260910.json`, `us-acs-code-memo-not-adopted-20260910.patch`, `us-acs-loaded-code-controls-20260910.json`, `us-common-frame-export36-20260910.json`, `us-input-coverage25-20260910.json`, `us-postclone-budget-replay-fence4-20260910.json`, `us-survey-catalogue-memo20-20260910.json`, `us-survey-numeric-diagnostics27-20260910.json` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `asec_raw_stage_v4.py`, `common_frame_export_contract.py`, `input_coverage_profile.py`, `population_input_coverage.py` +- `packages/microcosm-frame/src/microcosm/frame/`: `bundle.py` +- tests: `test_us_acs_transfer.py`, `test_us_asec_catalogue_records_memo.py`, `test_us_asec_demographic_source.py`, `test_us_asec_household_coverage_fields.py`, `test_us_common_frame_export_contract.py`, `test_us_population_input_coverage.py`, `test_us_spine_blindness.py`, `test_us_survey_catalogue_immutable_memo.py`, `test_us_survey_frame_identity_encoding.py`, `test_us_survey_origin_budget_replay_producer.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR + +#### P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS (6 non-test files, 1 test files) + +- `changelog.d/`: `us-launch-store-grouped-validation.fixed.md`, `us-survey-diagnostic-options.fixed.md` +- `experiments/`: `us-calibration-consolidation-149-20260910.json`, `us-puf55-canonical-create-failed-20260910.json`, `us-puf55-canonical-cycle39-failed-20260910.json` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `variable_labels.py` +- tests: `test_variable_labels.py` +- tests import from: 03 ACCEPTED-SHARED-RESTORE + +#### P5 CI-SEED-IDENTITY-AND-SPINE-BLINDNESS (10 non-test files, 10 test files) + +- `./`: `CLAUDE.md` +- `.github/workflows/`: `test.yml` +- `changelog.d/`: `893-diagnostic-current-lock.fixed.md`, `893-spec-identity-diagnostics.internal.md`, `893-worker-resource-trace.fixed.md` +- `docs/evidence/spec-engine/`: `us-f0-coverage.json` +- `experiments/`: `spec-seed-current-lock-20260910.json`, `us-acs-producer-profile-20260910.json` +- `packages/microcosm-build/src/microcosm/build/spec_engine/`: `inventory_coverage.py` +- `tools/`: `spec_seed_identity_diagnostics.py` +- tests: `test_spec_engine_country_bundles.py`, `test_spec_engine_loader.py`, `test_spec_seed_identity_cpu_bootstrap.py`, `test_spec_seed_identity_engine_parameter_files.py`, `test_spec_seed_identity_outer_failure.py`, `test_spec_seed_identity_owned_temp.py`, `test_spec_seed_identity_per_code_context.py`, `test_spec_seed_identity_system_metadata.py`, `test_us_puf_support.py`, `test_us_stacked_spine.py` +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY + +#### P6 UK-ADAPTER-DOCS-AND-RECORDS (10 non-test files, 2 test files) + +- `./`: `.gitattributes` +- `changelog.d/`: `893-uk-atomic-area-and-lazy-exports.added.md` +- `docs/`: `validation-cost-and-borrow-boundaries.md` +- `experiments/`: `uk-atomic-area-lazy52-20260910.json`, `us-financial-default-2-controls-20260909.json`, `us-native-atomic-financial-cold-20260910.json`, `us-native-postclone-financial-incomplete-20260910.json` +- `packages/microcosm-build/src/microcosm/build/uk_runtime/`: `__init__.py`, `atomic_area_support.py`, `atomic_household_identity.py` +- tests: `test_uk_atomic_area_support.py`, `test_uk_runtime_lazy_exports.py` +- source imports from: P1 POST-CLONE-ATOMIC-GEOGRAPHY +- tests import from: P1 POST-CLONE-ATOMIC-GEOGRAPHY + +**Layer-01 modules with no cross-layer import at all (53; `randomness.py` among them is amendment 20, not #893 content):** `__init__.py`, `canonical.py`, `origin.py`, `protocol.py`, `reasons.py`, `survey_allocation.py`, `survey_domain_sample.py`, `table_identity.py`, `_asec_current_money_codec.py`, `_person_signal_summary.py`, `acs_housing_universe.py`, `acs_native_coverage_binding.py`, `acs_person_coverage_authentication.py`, `acs_person_coverage_columns.py`, `acs_population_catalogue.py`, `asec_coverage_authentication.py`, `asec_current_money.py`, `asec_current_money_graph_resources.py`, `asec_current_money_resources.py`, `asec_current_money_units.py`, `asec_household_coverage_fields.py`, `asec_housing_status.py`, `asec_housing_status_source.py`, `asec_housing_universe.py`, `asec_housing_universe_source.py`, `asec_income_observations.py`, `asec_original_household_weights.py`, `asec_person_coverage_source.py`, `asec_population_catalogue.py`, `asec_prepared_source.py`, `asec_student_controls.py`, `cd_reference.py`, `cd_reference_sources.py`, `cps_carried_current.py`, `graph_implementation.py`, `graph_implementation_inventory.json`, `graph_native_origin_implementation.py`, `native_household_origin.py`, `operator_column_contracts.py`, `puf_detail_transfer.py`, `puf_diagnostic_consumer.py`, `puf_growth.py`, `puf_price_baseline.py`, `reported_coverage_source.py`, `source_csv_builtin.py`, `survey_age_activation.py`, `survey_age_sources.py`, `survey_catalogue_selection.py`, `survey_observed_age.py`, `survey_population_domains.py`, `group_bounds.py`, `_policyengine_us_source_index.py`, `randomness.py` + + +**Layer-01 modules with a cross-layer import (48; 27 of them only on the graph pieces G1/G2, the other 21 soft on modified modules of 03/05/06/P3/P4 — see `pr1-refined.json`):** `acs_housing_universe_source.py`, `asec_2024_native_population.py`, `asec_current_money_selection.py`, `asec_current_money_source.py`, `asec_engine_evaluation.py`, `asec_household_observations.py`, `asec_person_income_source.py`, `demographic_calibration_graph.py`, `full_puf_enrichment.py`, `graph_asec_income.py`, `graph_asec_prepared.py`, `graph_combined_clone.py`, `graph_composed_asec_binding.py`, `graph_composed_asec_measures.py`, `graph_composed_contracts.py`, `graph_composed_population.py`, `graph_context.py`, `graph_current_survey_puf_transfer.py`, `graph_geography.py`, `graph_housing_universe.py`, `graph_national_age_counts.py`, `graph_native_household_origin.py`, `graph_puf_detail_transfer.py`, `graph_puf_diagnostic_consumer.py`, `graph_sources.py`, `graph_survey_age_artifact.py`, `graph_survey_budget.py`, `graph_survey_calibration.py`, `graph_survey_population.py`, `national_age_activation.py`, `puf_growth_graph.py`, `puf_monetary_agi_projection.py`, `puf_monetary_source.py`, `puf_raw_source.py`, `survey_age_calibration.py`, `survey_calibration_diagnostics.py`, `survey_origin_budget.py`, `survey_population_preparation.py`, `survey_population_replay.py`, `_graph_legacy_apply.py`, `_graph_legacy_qrf.py`, `graph_legacy_apply.py`, `graph_legacy_apply_matrix.py`, `graph_legacy_qrf.py`, `graph_legacy_train.py`, `model_input.py`, `qrf_target.py`, `policyengine_us.py` + +**Layer-01 tests (all ride with PR-1; 26):** `test_us_acs_housing_source.py`, `test_us_acs_person_coverage_authentication.py`, `test_us_acs_person_coverage_columns.py`, `test_us_acs_population_catalogue.py`, `test_us_asec_2024_native_population.py`, `test_us_asec_coverage_authentication.py`, `test_us_asec_current_money_source.py`, `test_us_asec_person_income_source.py`, `test_us_current_survey_puf_host.py`, `test_us_current_survey_puf_transfer.py`, `test_us_full_puf_enrichment.py`, `test_us_graph_survey_population.py`, `test_us_national_age_counts.py`, `test_us_puf_detail_transfer.py`, `test_us_puf_price_baseline.py`, `test_us_survey_age_activation.py`, `test_us_survey_age_artifact.py`, `test_us_survey_age_calibration_run.py`, `test_us_survey_age_development.py`, `test_us_survey_age_sources.py`, `test_us_survey_calibration.py`, `test_us_survey_observed_age.py`, `test_us_survey_origin_budget.py`, `test_us_survey_population_preparation.py`, `test_us_survey_population_replay.py`, `test_policyengine_us_ownership_index.py` + +## 7. Draft charter amendment entries (one per kept graph piece) + +Written in the style of `docs/graph-acceptance.md` "Interface freeze". None of +the four touches `decl.py` or `kernel.py`, so none re-records the lock; they +are offered as numbered entries because each changes what a receipt, a store +object or an executor outcome means, which the charter records. + +**7.1 Raw-byte sources.** A source codec is registered in exactly one of two +modes and a name belongs to at most one: *Frame mode* (`SourceCodec`, +`SourceCodecRegistry.register` / `load`) decodes a source into a population +`Frame`, and *raw-byte mode* (`SourceBytesCodec`, `register_bytes` / +`load_bytes`, `load_source_bytes`) decodes it into immutable `bytes` that the +consuming kernel alone interprets. Neither mode can be loaded through the +other: the mismatch is a `TypeError` naming the mode the codec has, so no +caller receives bytes where it declared a Frame. The shipped `raw-bytes-v1` +codec reads one regular file, bounded at `RAW_BYTES_MAX_BYTES` (64 MiB) and +opened non-blocking so a directory, FIFO, device or socket is refused from +the descriptor's own mode. Identity, the pre-run content key and the post-run +mutation check stay in the executor (E1, E3 unchanged; C3's "declared +predecessors only" is untouched because a source is not a node). Why: a +lookup NPZ or a crosswalk CSV that an import kernel turns into a typed +`ArtifactOutput` (amendment 19) is not a population, and the only way to read +it before this was a Frame codec registered as a pretence. Raised by the US +launch integration branch (thirteen consumers); no key moves, because +`SourceRef.codec` was already normative and `raw-bytes-v1` is a new name. + +**7.2 Frames keep their metadata across the store.** A frame object is +written as `microcosm-graph-frame-v2`: `Frame.metadata` is encoded +losslessly (mappings, tuples, frozensets, `float64` bit patterns, scalars) +into the frame manifest, its SHA-256 is recorded in the object header, and +`_read_frame` refuses a mismatch as `StoreCorruptError`. A v1 object is +`StoreUnavailableError` on load rather than a Frame with its metadata silently +dropped, and writing a frame under a key that already holds one with different +metadata is `StoreCorruptError` (E1: a key identifies exactly one content). +The JSON decode boundary refuses `NaN`/`Infinity` and overflowing numerals, +because every store JSON is written with `allow_nan=False`, so a corrupt +manifest is `StoreCorruptError` at the boundary rather than a `TypeError` +from the canonical re-encoder (E2). Why: the US survey stages carry source +provenance in `Frame.metadata` and replay verifiers compare it +(`same_replayed_frame`); until now a population that went through the store +came back without it. No node key moves: the frame key is the node's, and +metadata was never part of a key. Raised by the US launch integration +branch (`8d9f62c40`); acceptance items E1 and E2. + +**7.3 Execution states: a gate exception leaves its consumers unreached.** +Amendment 19 refused a gate kernel that declares a typed artifact output +because it had no regime for an output a node was unable to produce. This +entry supplies the regime and lifts the refusal. The executor owns an +execution state (`"microcosm.graph.execution.v1"`) that a kernel may not +author: a gate that declares typed outputs and raises still becomes a `fail` +verdict and the run continues (amendment 7, now literally true for every +node shape), and its receipt records `gate_exception` with the outputs it +could not produce; a node whose predecessor is `unreached`, or whose declared +byte input its producer recorded as unavailable, is itself recorded +`unreached` with its blockers named by node key — it runs no kernel and +invents no column, frame, weight or byte (F1: nothing is fabricated); a +release behind such an edge derives its tier from the same ancestry and stays +`evidence` (F2). Cache records that carry a state use schema 3; a cached +unreached record is a hit only while the same inputs are unavailable for the +same reason and is `StoreCorruptError` once they exist (E3); `require` +preflight derives blockers from the cached parents. A run manifest carrying +any state serializes at schema 4 and authenticates every blocker on load +(F5, E4). No node key moves: states live in receipts and records, never in a +key. Why: the US post-clone geography gate emits a typed validation artifact +that the financial stages consume, so a geography failure must leave those +stages unreached rather than abort the run or launder the gate into a compute +node. Raised by the US launch integration branch; supersedes the interim +ruling recorded in amendment 19. + +**7.4 A private population observer.** `run_graph` accepts +`_population_observer`, a callable handed each node's admitted population +(design anchors included) after the result is applied and before the node is +persisted, on cold execution and on cache hits alike; it must not mutate the +population, an exception it raises refuses the run, and an unreached node +has no population to observe. It is an integration seam for verifiers (the +US survey stages assert design-weight provenance through it), not a kernel +capability: it enters no key, receipt or record (A1, B4 unchanged). Raised by +the US launch integration branch. + +## 8. Decisions needed from Max + +1. **Piece C supersedes an amendment-19 ruling.** The Amendment 19 lane + recorded "the gate-artifact-output refusal is the one interim ruling this + lane made on his behalf". This branch's US runtime needs the opposite (the + post-clone geography gate declares its validation artifact and three + stages consume it), so piece C re-applies the branch's execution-state + regime and replaces main's test that pins the refusal with five tests of + the new regime. Both the frozen files and every `test_acceptance_*` file + are untouched, but the executor's behaviour on that one node shape now + differs from what the charter's amendment 19 text says. Landing PR-G3 + requires adopting entry 7.3 (or an edited form) in the charter; until then + this branch's executor and the charter disagree on that sentence. +2. **Lazy population retention** (`attachments.py`, layer 08) is dropped for + lack of a consumer. If the native pilot's memory ceiling is a reason to + keep it, it is a self-contained PR on top of PR-G2. +3. **`_stream_file` and the `_write_node` refactor** are dropped for lack of a + consumer. Both are memory-only; say if either should come back as its own + small PR. +4. **The report's home.** `out.md` at the repo root is a tracked file holding + the F1 Sol gate round-1 report (2026-09-04). This report is written there + for the dispatcher but is **not committed** there; the committed copy is + `experiments/893-reconciliation-amendments-19-20-20260912.md`. +5. **Three US stages name a resource that does not exist anywhere.** + `asec_prepared_v3`, `composed_asec_binding_v1` and `composed_population_v1` + list `us_runtime/asec_current_money_engine_defaults_v1.json` in their + resources; the file is on neither this branch (at `069d5ed9a` or now) nor + `origin/main` (it appears only in two commits of another branch, + `b0e7f54b2` / `b6cfc84b0`). `implementation_manifest` for those stages has + therefore been failing since before this lane. Not invented here; the + owner of layer 10 SOURCE-CLOSURE has to admit the file or drop the entry, + and the inventory's "scope review" rule applies to `07566eb17` as well. +6. **Two build tests fail locally for reasons outside this lane** if they are + run: `test_release_target_parity.py`'s two `_feed_or_skip` tests need a 131 + MB feed outside the repository (already recorded by the Amendment 19 lane). + They were not part of this lane's runs. + +## 9. Where things are + +- Journal: `PROGRESS.md` (top section, cumulative file). +- This report: `out.md` (uncommitted, the `-o` path) and + `experiments/893-reconciliation-amendments-19-20-20260912.md` (committed). +- Scratch evidence (session-local, not committed): the AST consumer scan, + the layer scripts and JSON, and every gate log named in section 5. diff --git a/experiments/908-review-findings-recheck.json b/experiments/908-review-findings-recheck.json new file mode 100644 index 000000000..44ea5058a --- /dev/null +++ b/experiments/908-review-findings-recheck.json @@ -0,0 +1,33 @@ +{ + "code_1_aggregate_only_metadata": { + "candidate_mapping_with_denylisted_person_id_refused": true, + "candidate_mapping_with_record_vectors_refused": true, + "closed": true, + "context_record_vectors_refused": true, + "nested_context_refused": true + }, + "code_2_nested_alias": { + "caller_context_unchanged": true, + "closed": true, + "next_snapshot_unchanged": true + }, + "code_3_history_publication": { + "a_complete_write_still_publishes": true, + "closed": true, + "no_leftover_temporary": true, + "reader_sees_nothing_after_failure": true, + "retained_after_failure": 0, + "write_failed": true + }, + "code_4_identity_validation": { + "closed": true, + "invalid_cases_now_refused": [ + "epoch_exceeds_epochs", + "invalid_timestamp", + "invalid_best_metadata", + "candidate_id_not_string" + ] + }, + "reviewed_head": "bae1887ffc5f6e32edefa9774ec6428568286247", + "scope": "The review's four counterexamples replayed against this branch. Invented data only; no optimizer, engine, network or native input." +} diff --git a/experiments/908-target-snapshot-bench-receipt.json b/experiments/908-target-snapshot-bench-receipt.json new file mode 100644 index 000000000..f57535753 --- /dev/null +++ b/experiments/908-target-snapshot-bench-receipt.json @@ -0,0 +1,84 @@ +{ + "label": "SYNTHETIC \u2014 invented matrices and frames, not US or UK calibration", + "dimension_points": { + "codec_target_counts": [ + 500, + 2000, + 5000, + 10000 + ], + "solver_records": 10000, + "solver_targets": 100, + "solver_epochs": 300 + }, + "bounds": { + "numeric_threads": 1, + "cpu_seconds_limit_applied": 120, + "memory_bytes_limit_applied": null, + "peak_rss_bytes": 289800192, + "native_inputs_read": 0 + }, + "codec_costs": [ + { + "n_targets": 500, + "build_and_validate_ms": 2.099, + "serialize_ms": 0.603, + "store_write_ms": 3.913, + "serialized_bytes": 88625, + "bytes_per_target": 177.2 + }, + { + "n_targets": 2000, + "build_and_validate_ms": 9.087, + "serialize_ms": 2.321, + "store_write_ms": 13.491, + "serialized_bytes": 353895, + "bytes_per_target": 176.9 + }, + { + "n_targets": 5000, + "build_and_validate_ms": 18.943, + "serialize_ms": 5.723, + "store_write_ms": 29.589, + "serialized_bytes": 885599, + "bytes_per_target": 177.1 + }, + { + "n_targets": 10000, + "build_and_validate_ms": 38.339, + "serialize_ms": 11.357, + "store_write_ms": 60.114, + "serialized_bytes": 1771933, + "bytes_per_target": 177.2 + } + ], + "solver_overhead": [ + { + "cadence": "observer_off", + "best_of_3_seconds": 0.036, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": null, + "chunks_retained_on_disk": 0, + "retained_bytes_on_disk": 0 + }, + { + "cadence": "bounded_every_25", + "best_of_3_seconds": 0.0597, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": 14, + "chunks_retained_on_disk": 14, + "retained_bytes_on_disk": 231241 + }, + { + "cadence": "every_epoch", + "best_of_3_seconds": 0.594, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": 301, + "chunks_retained_on_disk": 256, + "retained_bytes_on_disk": 4220712 + } + ] +} diff --git a/experiments/908-target-snapshot-bench-receipts.md b/experiments/908-target-snapshot-bench-receipts.md new file mode 100644 index 000000000..faf4d8816 --- /dev/null +++ b/experiments/908-target-snapshot-bench-receipts.md @@ -0,0 +1,63 @@ +# microcosm#908 — per-target snapshot cost, SYNTHETIC measurement + +**Everything below is synthetic.** The numbers come from invented target +matrices and an invented seeded frame produced by +[`908_target_snapshot_bench.py`](908_target_snapshot_bench.py). No native +microdata was read, no country engine ran, and nothing was uploaded. These are +**not** US or UK calibration runtimes and **not** upload times. The dimension +points were chosen to bracket an order of magnitude; they are not measured from +the real US or UK target registries. + +Run 2026-09-12 on macOS 26.6.2 / arm64, one numeric thread, CPU rlimit 120 s, +peak RSS 0.29 GiB (re-measured after the adversarial-review fixes) (well inside the 2 GiB cap; macOS refused `RLIMIT_AS`, so the +script records the limit it actually applied and the peak RSS it reached). +Machine-readable copy: [`908-target-snapshot-bench-receipt.json`](908-target-snapshot-bench-receipt.json). + +## Codec and local-store cost per snapshot, by target count + +Build+validate is the full emission cost (`snapshot()` validates what it +builds). Store write is one immutable history chunk plus the atomic `latest.json` +replacement plus the index rewrite. + +| targets | build+validate (ms) | serialize (ms) | store write (ms) | serialized bytes | bytes/target | +|--------:|--------------------:|---------------:|-----------------:|-----------------:|-------------:| +| 500 | 2.10 | 0.60 | 3.91 | 88,625 | 177.2 | +| 2,000 | 9.09 | 2.32 | 13.49 | 353,895 | 176.9 | +| 5,000 | 18.94 | 5.72 | 29.59 | 885,599 | 177.1 | +| 10,000 | 38.34 | 11.36 | 60.11 | 1,771,933 | 177.2 | + +Both cost and size are linear in target count at ~177 bytes per target row +(indent=1, sorted keys). A single 10,000-target snapshot is ~1.7 MB. + +## Solver overhead at three cadences + +Invented frame: 10,000 households, 100 targets, 300 epochs, seed 0. One +discarded warm-up run, then best of three timed runs per cadence. + +| cadence | best-of-3 (s) | snapshots emitted | chunks retained | retained bytes | weights bitwise equal to observer-off | +|---|---:|---:|---:|---:|---| +| observer off | 0.0360 | — | 0 | 0 | yes (reference) | +| bounded, every 25 | 0.0597 | 14 | 14 | 231,241 | yes | +| every epoch | 0.5940 | 301 | 256 | 4,220,712 | yes | + +The closing loss is identical to all printed digits across all three +(`2.7968459435242565e-07`), and the returned weight vectors are bitwise equal. + +## What this does and does not license + +It does establish that emission is linear, that ~177 bytes/target is the wire +cost, and that turning the observer on does not change the optimizer's result. + +It does **not** license a production cadence default. This solve costs 36 ms +total, so per-snapshot cost dominates it by construction and the 16x +every-epoch ratio above is an artifact of a trivially cheap solve — a real +calibration's epoch is far more expensive, so the same absolute per-snapshot +cost would be a much smaller fraction of it. Choosing the production default +needs the native UK and US measurements #908 asks for, which this host is not +permitted to run. The shipped default is therefore *off*; a caller that opts in +picks its own cadence. + +The 5 MiB per-remote-file ceiling in the version-2 staging content policy is a +separate, unmeasured constraint here: at ~177 bytes/target a single snapshot +stays well inside it, but an accumulated history does not, which is why the +store chunks history rather than growing one document. diff --git a/experiments/908_review_findings_recheck.py b/experiments/908_review_findings_recheck.py new file mode 100644 index 000000000..01b72e10b --- /dev/null +++ b/experiments/908_review_findings_recheck.py @@ -0,0 +1,180 @@ +"""Re-run the #908 independent review's four counterexamples against the fix. + +The review's probe (head ``bae1887ff``) reproduced four defects with invented +data and recorded them in ``reproductions.json``. This script replays the same +four counterexamples, unchanged, against this branch's source and asserts each +one is now closed. It is passive: invented aggregates and temporary files only, +no engine, no optimizer, no network, no native input. + +Run with the lane's isolated interpreter, e.g.:: + + python -I -B -S experiments/908_review_findings_recheck.py +""" + +from __future__ import annotations + +import copy +import json +import pathlib +import sys +import tempfile +from datetime import UTC, datetime + +import numpy as np + +from microcosm.calibrate.target_snapshots import ( + TargetSnapshotError, + TargetSnapshotObserver, + TargetSnapshotWriter, + iter_history, + validate_target_snapshot, +) + +#: The review's own record-level vectors, copied verbatim from its probe. +VECTORS = { + "household_weights": [10.0, 20.0], + "tax_unit_id": [101, 102], + "spm_unit_id": [201, 202], +} + + +def _observer(sink=lambda payload: None, context=None): + return TargetSnapshotObserver( + sink=sink, + run_id="invented", + context={} if context is None else context, + clock=lambda: datetime(2026, 9, 12, tzinfo=UTC), + ).bind(names=("aggregate",), targets=[30.0]) + + +def _payload(): + return _observer().snapshot(np.array([30.0]), epoch=1, epochs=2, iterate="current") + + +def _accepted(payload) -> bool: + try: + validate_target_snapshot(payload) + except TargetSnapshotError: + return False + return True + + +def _refused(call) -> bool: + try: + call() + except TargetSnapshotError: + return True + return False + + +def finding_1_aggregate_only_metadata() -> dict[str, object]: + """Record vectors and mapping identifiers no longer have a shape to ride in.""" + candidate = _payload() + candidate["candidate_id"] = copy.deepcopy(VECTORS) + person = _payload() + person["candidate_id"] = {"person_id": [1, 2]} + return { + "context_record_vectors_refused": _refused(lambda: _observer(context=VECTORS)), + "candidate_mapping_with_record_vectors_refused": not _accepted(candidate), + "candidate_mapping_with_denylisted_person_id_refused": not _accepted(person), + "nested_context_refused": _refused( + lambda: _observer(context={"display": {"unit": "USD"}}) + ), + "closed": True, + } + + +def finding_2_nested_alias() -> dict[str, object]: + """A sink cannot reach the caller's metadata or the next snapshot.""" + context = {"display_unit": "USD"} + + def mutate(payload): + payload["context"]["display_unit"] = "changed-by-sink" + + bound = _observer(sink=mutate, context=context) + bound.emit(np.array([30.0]), epoch=1, epochs=2, iterate="current") + following = bound.snapshot(np.array([30.0]), epoch=2, epochs=2, iterate="current") + return { + "caller_context_unchanged": context == {"display_unit": "USD"}, + "next_snapshot_unchanged": following["context"] == {"display_unit": "USD"}, + "closed": True, + } + + +def finding_3_history_publication() -> dict[str, object]: + """A partial chunk is never visible, and a failed write leaves nothing.""" + import os + + observed: dict[str, object] = {} + with tempfile.TemporaryDirectory(prefix="microcosm-908-fix-recheck-") as tmp: + writer = TargetSnapshotWriter(pathlib.Path(tmp)) + real_fsync = os.fsync + calls: list[int] = [] + + def failing_fsync(descriptor): + calls.append(descriptor) + if len(calls) == 1: + # The review interrupted the chunk write after ten bytes; this + # interrupts it after all of them, which is strictly harder. + raise OSError("invented disk failure before publication") + return real_fsync(descriptor) + + os.fsync = failing_fsync + try: + try: + writer(_payload()) + except OSError: + observed["write_failed"] = True + finally: + os.fsync = real_fsync + observed["retained_after_failure"] = len(writer.retained()) + observed["reader_sees_nothing_after_failure"] = list(iter_history(tmp)) == [] + observed["no_leftover_temporary"] = ( + sorted(p.name for p in (pathlib.Path(tmp) / "history").iterdir()) == [] + ) + writer(_payload()) + observed["a_complete_write_still_publishes"] = len(list(iter_history(tmp))) == 1 + return {**observed, "closed": True} + + +def finding_4_identity_validation() -> dict[str, object]: + """The codec refuses the four impossible payloads it used to admit.""" + cases = { + "epoch_exceeds_epochs": {"epoch": 99, "epochs": 1}, + "invalid_timestamp": {"created_at": "not-a-time"}, + "invalid_best_metadata": { + "iterate": "best_retained", + "best_retained": {"available": True, "epoch": -7, "loss": "not-a-number"}, + }, + "candidate_id_not_string": {"candidate_id": {"unexpected": [1, 2]}}, + } + refused = [] + for name, changes in cases.items(): + payload = _payload() + payload.update(changes) + if not _accepted(payload): + refused.append(name) + return {"invalid_cases_now_refused": refused, "closed": len(refused) == len(cases)} + + +def main() -> int: + report = { + "scope": ( + "The review's four counterexamples replayed against this branch. " + "Invented data only; no optimizer, engine, network or native input." + ), + "reviewed_head": "bae1887ffc5f6e32edefa9774ec6428568286247", + "code_1_aggregate_only_metadata": finding_1_aggregate_only_metadata(), + "code_2_nested_alias": finding_2_nested_alias(), + "code_3_history_publication": finding_3_history_publication(), + "code_4_identity_validation": finding_4_identity_validation(), + } + for key, value in report.items(): + if isinstance(value, dict): + assert value["closed"] is True, f"{key} is not closed: {value}" + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/experiments/908_target_snapshot_bench.py b/experiments/908_target_snapshot_bench.py new file mode 100644 index 000000000..c48d02785 --- /dev/null +++ b/experiments/908_target_snapshot_bench.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""SYNTHETIC benchmark for the #908 per-target snapshot path. + +Every number this script produces comes from **invented** target matrices and +an **invented** frame built from a seeded RNG. Nothing here reads native +microdata, runs a country engine, or touches a network. The dimension points +below were chosen to bracket an order of magnitude; they are NOT measured from +the real US or UK calibration target registries, and the timings are NOT US or +UK calibration times or upload times. Read them as "what does the codec and the +local store cost per target row and per epoch", nothing more. + +Run (from the repo root, inside the isolated lane interpreter):: + + python experiments/908_target_snapshot_bench.py + +Bounded on purpose: one numeric thread, a 2 GiB address-space cap and a 120 s +CPU-time cap are set before torch loads, so the script cannot quietly grow into +a machine-sized job. +""" + +from __future__ import annotations + +import json +import os +import resource +import sys +import tempfile +import time +from pathlib import Path + +_MEMORY_BYTES = 2 * 1024**3 + + +def _bound_resource(which: int, value: int) -> int | None: + """Apply a soft rlimit where the platform allows it; report what stuck.""" + soft, hard = resource.getrlimit(which) + if hard != resource.RLIM_INFINITY and value > hard: + return None + try: + resource.setrlimit(which, (value, hard)) + except (ValueError, OSError): + return None + return value + + +_CPU_LIMIT = _bound_resource(resource.RLIMIT_CPU, 120) +_MEMORY_LIMIT = _bound_resource(resource.RLIMIT_AS, _MEMORY_BYTES) +if _MEMORY_LIMIT is None: + # macOS refuses RLIMIT_AS here; RLIMIT_DATA is the portable fallback. + _MEMORY_LIMIT = _bound_resource(resource.RLIMIT_DATA, _MEMORY_BYTES) +for _variable in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS"): + os.environ.setdefault(_variable, "1") + +import numpy as np # noqa: E402 - after the resource and thread bounds +import pandas as pd # noqa: E402 - after the resource and thread bounds +import torch # noqa: E402 - after the resource and thread bounds + +from microcosm.calibrate import ( # noqa: E402 - after the resource bounds + EVERY_EPOCH, + Target, + TargetSet, + TargetSnapshotCadence, + TargetSnapshotObserver, + TargetSnapshotWriter, + calibrate, +) +from microcosm.frame import ( # noqa: E402 - after the resource bounds + EntitySchema, + Frame, + WeightKind, + Weights, +) + +torch.set_num_threads(1) + +#: Invented target-count dimension points for the codec measurement. +TARGET_COUNTS = (500, 2_000, 5_000, 10_000) + +#: Invented solver dimension point: households x targets x epochs. +SOLVER_RECORDS = 10_000 +SOLVER_TARGETS = 100 +SOLVER_EPOCHS = 300 + +#: Timed repeats per cadence; the table reports the best (least noisy) run. +REPEATS = 3 + +_SCHEMA = EntitySchema(group_entities=("household",)) + + +def _bound(n_targets: int): + """A bound observer over ``n_targets`` invented targets, sinking to nowhere.""" + rng = np.random.default_rng(0) + names = tuple(f"synthetic_target_{index:06d}@2024" for index in range(n_targets)) + targets = rng.lognormal(12.0, 1.0, n_targets) + observer = TargetSnapshotObserver(sink=lambda payload: None, run_id="synthetic") + return observer.bind(names=names, targets=targets), rng.lognormal( + 12.0, 1.0, n_targets + ) + + +def codec_costs() -> list[dict[str, object]]: + """Build/validate/serialize/write cost per snapshot, by target count.""" + rows: list[dict[str, object]] = [] + for n_targets in TARGET_COUNTS: + bound, estimates = _bound(n_targets) + repeats = max(3, 30_000 // n_targets) + start = time.perf_counter() + for _ in range(repeats): + # snapshot() validates the payload it builds, so this is the full + # build + validate cost a real emission pays. + payload = bound.snapshot( + estimates, epoch=1, epochs=1, iterate="current", loss=0.1 + ) + build_seconds = (time.perf_counter() - start) / repeats + + start = time.perf_counter() + for _ in range(repeats): + serialized = json.dumps(payload, indent=1, sort_keys=True, allow_nan=False) + serialize_seconds = (time.perf_counter() - start) / repeats + + with tempfile.TemporaryDirectory() as directory: + writer = TargetSnapshotWriter(Path(directory), history_limit=8) + start = time.perf_counter() + for index in range(repeats): + copy = dict(payload) + copy["sequence"] = index + 1 + writer(copy) + write_seconds = (time.perf_counter() - start) / repeats + + rows.append( + { + "n_targets": n_targets, + "build_and_validate_ms": round(build_seconds * 1e3, 3), + "serialize_ms": round(serialize_seconds * 1e3, 3), + "store_write_ms": round(write_seconds * 1e3, 3), + "serialized_bytes": len(serialized.encode("utf-8")), + "bytes_per_target": round( + len(serialized.encode("utf-8")) / n_targets, 1 + ), + } + ) + return rows + + +def _synthetic_frame_and_targets() -> tuple[Frame, TargetSet]: + rng = np.random.default_rng(1) + household_ids = np.arange(SOLVER_RECORDS, dtype="int64") + columns = { + f"measure_{index:03d}": rng.lognormal(3.0, 0.5, SOLVER_RECORDS) + for index in range(SOLVER_TARGETS) + } + weights = np.full(SOLVER_RECORDS, 100.0) + household = pd.DataFrame({"household_id": household_ids, **columns}) + person = pd.DataFrame( + {"person_id": household_ids, "person_household_id": household_ids} + ) + frame = Frame( + {"person": person, "household": household}, + _SCHEMA, + {"household": Weights(values=weights, kind=WeightKind.DESIGN)}, + ) + targets = TargetSet( + [ + Target( + name=name, + period=2024, + entity="household", + measure=name, + # An invented 8% miss, so the optimizer has somewhere to go. + value=float((values * weights).sum()) * 1.08, + ) + for name, values in columns.items() + ] + ) + return frame, targets + + +def solver_overhead() -> list[dict[str, object]]: + """Added wall-clock from snapshot emission, at three cadences. + + One discarded warm-up run first (the first calibrate of a process pays + matrix-build and torch warm-up that would otherwise be charged to whichever + cadence happened to run first), then the best of ``REPEATS`` timed runs per + cadence. Each run's returned weights are compared bitwise against the + observer-off run, so the table also evidences that emission changed nothing. + """ + frame, targets = _synthetic_frame_and_targets() + cadences = ( + ("observer_off", None), + ("bounded_every_25", TargetSnapshotCadence(every=25)), + ("every_epoch", TargetSnapshotCadence(every=EVERY_EPOCH)), + ) + calibrate(frame, targets, epochs=SOLVER_EPOCHS, seed=0) # warm-up, discarded + + rows: list[dict[str, object]] = [] + reference: np.ndarray | None = None + for label, cadence in cadences: + best_seconds = float("inf") + emitted = retained_chunks = retained_bytes = 0 + for _ in range(REPEATS): + with tempfile.TemporaryDirectory() as directory: + observer = ( + None + if cadence is None + else TargetSnapshotObserver( + sink=TargetSnapshotWriter(Path(directory)), + run_id="synthetic", + cadence=cadence, + ) + ) + start = time.perf_counter() + result = calibrate( + frame, + targets, + epochs=SOLVER_EPOCHS, + seed=0, + target_snapshots=observer, + ) + best_seconds = min(best_seconds, time.perf_counter() - start) + history = Path(directory) / "history" + if observer is not None: + retained_chunks = len(list(history.glob("*.json"))) + retained_bytes = sum( + path.stat().st_size for path in history.glob("*.json") + ) + emitted = int( + json.loads( + (Path(directory) / "history_index.json").read_text( + encoding="utf-8" + ) + )["last_sequence"] + ) + if reference is None: + reference = result.weights.copy() + identical = True + else: + identical = bool(np.array_equal(reference, result.weights)) + rows.append( + { + "cadence": label, + f"best_of_{REPEATS}_seconds": round(best_seconds, 4), + "closing_loss": float(result.closing_loss), + "weights_bitwise_equal_to_observer_off": identical, + "snapshots_emitted": emitted or None, + "chunks_retained_on_disk": retained_chunks, + "retained_bytes_on_disk": retained_bytes, + } + ) + return rows + + +def main() -> int: + report = { + "label": "SYNTHETIC — invented matrices and frames, not US or UK calibration", + "dimension_points": { + "codec_target_counts": list(TARGET_COUNTS), + "solver_records": SOLVER_RECORDS, + "solver_targets": SOLVER_TARGETS, + "solver_epochs": SOLVER_EPOCHS, + }, + "bounds": { + "numeric_threads": torch.get_num_threads(), + "cpu_seconds_limit_applied": _CPU_LIMIT, + "memory_bytes_limit_applied": _MEMORY_LIMIT, + "peak_rss_bytes": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + "native_inputs_read": 0, + }, + "codec_costs": codec_costs(), + "solver_overhead": solver_overhead(), + } + json.dump(report, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/acs-serial-membership/README.md b/experiments/acs-serial-membership/README.md new file mode 100644 index 000000000..c375634da --- /dev/null +++ b/experiments/acs-serial-membership/README.md @@ -0,0 +1,63 @@ +# Reuse ACS serial membership lookups + +This native-lane continuation starts at `9af56aa8a`. It prepares each exact +string-key lookup once per person-archive read, then checks every incoming +chunk against that lookup. The pandas Arrow path previously rebuilt its +complete comparison set for each chunk. The completed native attribution +sample assigned about 108 cold / 119 replay CPU seconds to that operation; +these are statistical estimates, not function timers or promised savings. + +An object-dtype pandas Index retains its hash table for the duration of the +archive call. Missing chunk values remain unmatched because the optimized +keys are exact strings. Any mixed key set or other Series dtype uses the +original pandas `isin` path. Age validation, complete orphan checking, +household roster validation and selection order are unchanged. No source row, +archive parse, validation result or issued authority is cached. The native +coverage owner explicitly pins the reviewed reader hash; altered source bytes +still refuse with `UNREVIEWED_PREPARATION`. + +## Evidence + +- `tests-with-owner.txt`: 185 passes, covering the new differential tests, + existing ACS PUMS behavior, coverage authentication and source compilation. +- Differential cases include both pandas string storage backends, both missing + policies, mixed/non-string fallback keys, duplicate indices, names, Unicode, + literal leading-zero IDs, chunk changes and exact orphan/age error messages. +- Actual native `_producer()` admits the reviewed implementation; a separate + changed-source-byte test confirms the existing refusal remains active. +- `ci-inventory.txt`: the new flat test file is included in the CI inventory. +- Ruff check/format and `git diff --check` pass. + +`benchmark.py` uses invented arrays only: 200,000 comparison keys and twenty +20,000-row chunks on pandas 3.0.3 / PyArrow 25.0.0. Separate processes produced +`original.json` and `prepared.json`, and every result matched literal pandas +`isin`. Including preparation, timed CPU was 6.691459 seconds original and +0.052461 seconds prepared. The prepared first lookup took 0.012154 CPU seconds +and reused the same index engine for the remaining nineteen calls. + +This is a focused microbenchmark, not a native build measurement. The prepared +Index's deep estimate is 17,627,112 bytes including Python strings shared with +the existing key set; it is not incremental RSS. The process RSS observations +include imports and the untimed pandas oracle. The optimization adds a local +O(key-count) index and O(rows-per-chunk) temporary lookup buffers, and does not +establish lower memory use or full-source capacity. Key preparation also occurs +before archive/member validation, so malformed archives can incur that bounded +extra allocation before the same validation refusal. + +## Reproduce + +The recorded environment is the existing native-lane Python 3.14.4 virtual +environment, reused without installs or lock changes. All six Microcosm source +shards are explicitly loaded from this checkout; `benchmark.py` checks its +reader import origin. Run it with isolated Python (`-I -B`), then choose +`original` or `prepared`. No real archive or population data is read. + +The first test-launch attempt lacked `-I` and encountered an unrelated +`/private/tmp/dis.py` shadowing the standard library before test collection. +Isolated Python corrected the harness. The subsequent expected red test failed +because the lookup helpers did not yet exist; the final suite above passes. + +This branch changes producer-bound provenance and may change graph keys. +Compare all/compact retention at the same combined source revision, and record +cross-revision semantic equivalence separately. No actual native replay, +certification, default change, merge or publication is claimed by this change. diff --git a/experiments/acs-serial-membership/benchmark.py b/experiments/acs-serial-membership/benchmark.py new file mode 100644 index 000000000..b6a2a6700 --- /dev/null +++ b/experiments/acs-serial-membership/benchmark.py @@ -0,0 +1,101 @@ +"""Bounded invented membership timing; never reads Census records or archives.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import resource +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(path) for path in sorted((ROOT / "packages").glob("*/src"))] + +import pandas as pd # noqa: E402 +import psutil # noqa: E402 +import pyarrow # noqa: E402 + +from microcosm.build.us_runtime import acs_pums # noqa: E402 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("original", "prepared")) + args = parser.parse_args() + assert Path(acs_pums.__file__).resolve().is_relative_to(ROOT) + process = psutil.Process() + # Key count matters: the old Arrow path rebuilds this set for every chunk. + keys = frozenset(f"invented-{i:09d}" for i in range(200_000)) + values = pd.Series( + [f"invented-{i:09d}" for i in range(15_000)] + ["absent", None] * 2_500, + dtype=pd.StringDtype(storage="pyarrow"), + name="SERIALNO", + ) + expected = values.isin(keys) + rss_before = process.memory_info().rss + started = time.perf_counter() + cpu = time.process_time() + lookup = acs_pums._serial_lookup(keys) if args.mode == "prepared" else None + preparation = { + "wall_seconds": time.perf_counter() - started, + "cpu_seconds": time.process_time() - cpu, + } + timings = [] + engine = None + for i in range(20): + started, cpu = time.perf_counter(), time.process_time() + actual = ( + values.isin(keys) + if args.mode == "original" + else acs_pums._serial_isin(values, keys, lookup) + ) + timings.append( + { + "wall_seconds": time.perf_counter() - started, + "cpu_seconds": time.process_time() - cpu, + } + ) + pd.testing.assert_series_equal(actual, expected) + if lookup is not None: + if i == 0: + engine = lookup._engine + assert lookup._engine is engine + result = { + "scope": "invented_arrays_only; not native preparation performance", + "mode": args.mode, + "keys": len(keys), + "rows_per_chunk": len(values), + "chunks": len(timings), + "all_masks_match_pandas": True, + "prepared_index_engine_reused": lookup is not None, + "preparation": preparation, + "first_lookup": timings[0], + "warm_lookups": timings[1:], + "total_cpu_seconds": preparation["cpu_seconds"] + + sum(t["cpu_seconds"] for t in timings), + "total_wall_seconds": preparation["wall_seconds"] + + sum(t["wall_seconds"] for t in timings), + "rss_before_lookup_bytes": rss_before, + "rss_after_lookups_bytes": process.memory_info().rss, + "whole_process_peak_rss_bytes_macos": resource.getrusage( + resource.RUSAGE_SELF + ).ru_maxrss, + # Includes shared Python strings: this is not net incremental RSS. + "index_deep_bytes_including_shared_strings": None + if lookup is None + else lookup.memory_usage(deep=True), + "pandas": pd.__version__, + "pyarrow": pyarrow.__version__, + "python": sys.version, + "module_sha256": hashlib.sha256( + Path(acs_pums.__file__).read_bytes() + ).hexdigest(), + "benchmark_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + } + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/experiments/acs-serial-membership/ci-inventory.txt b/experiments/acs-serial-membership/ci-inventory.txt new file mode 100644 index 000000000..4364f99e7 --- /dev/null +++ b/experiments/acs-serial-membership/ci-inventory.txt @@ -0,0 +1,1740 @@ +tracked_test_files=557 +[fast] +trade: files=7 procs=main shards_per_process=1 + packages/microcosm-build/tests/test_us_trade_census_imports.py + packages/microcosm-build/tests/test_us_trade_crosscheck.py + packages/microcosm-build/tests/test_us_trade_entries_cli.py + packages/microcosm-build/tests/test_us_trade_entry_generator.py + packages/microcosm-build/tests/test_us_trade_facts.py + packages/microcosm-build/tests/test_us_trade_imdb_bulk.py + packages/microcosm-build/tests/test_us_trade_imdb_goldens.py +spine-uk: files=138 procs=main shards_per_process=1 + packages/microcosm-build/tests/test_uk_age_tail.py + packages/microcosm-build/tests/test_uk_atomic_area_support.py + packages/microcosm-build/tests/test_uk_atomic_household_lineage.py + packages/microcosm-build/tests/test_uk_battery_bindings.py + packages/microcosm-build/tests/test_uk_brma_resource.py + packages/microcosm-build/tests/test_uk_bus_targets.py + packages/microcosm-build/tests/test_uk_bus_use_incidence.py + packages/microcosm-build/tests/test_uk_calibration_run.py + packages/microcosm-build/tests/test_uk_calibration_seam_driver.py + packages/microcosm-build/tests/test_uk_cgt_calibration.py + packages/microcosm-build/tests/test_uk_cgt_imputation.py + packages/microcosm-build/tests/test_uk_cgt_observation_period.py + packages/microcosm-build/tests/test_uk_cgt_source_manifest.py + packages/microcosm-build/tests/test_uk_cgt_structure.py + packages/microcosm-build/tests/test_uk_cgt_targets.py + packages/microcosm-build/tests/test_uk_childcare_targets.py + packages/microcosm-build/tests/test_uk_chronicle_feed.py + packages/microcosm-build/tests/test_uk_consumption_resources.py + packages/microcosm-build/tests/test_uk_content_identity.py + packages/microcosm-build/tests/test_uk_corrected_measurement_receipt.py + packages/microcosm-build/tests/test_uk_council_tax_targets.py + packages/microcosm-build/tests/test_uk_data_target_parity.py + packages/microcosm-build/tests/test_uk_dense_release_assembler.py + packages/microcosm-build/tests/test_uk_diagnostics.py + packages/microcosm-build/tests/test_uk_donor_uprating.py + packages/microcosm-build/tests/test_uk_e5_identity.py + packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py + packages/microcosm-build/tests/test_uk_energy_pricing.py + packages/microcosm-build/tests/test_uk_etb_services.py + packages/microcosm-build/tests/test_uk_etb_vat.py + packages/microcosm-build/tests/test_uk_evaluate_dataset_size_tool.py + packages/microcosm-build/tests/test_uk_fact_raking.py + packages/microcosm-build/tests/test_uk_firm_generation.py + packages/microcosm-build/tests/test_uk_firms_targets.py + packages/microcosm-build/tests/test_uk_frame_linkage_preflight.py + packages/microcosm-build/tests/test_uk_frs_council_tax.py + packages/microcosm-build/tests/test_uk_frs_disability.py + packages/microcosm-build/tests/test_uk_frs_education.py + packages/microcosm-build/tests/test_uk_frs_education_grants.py + packages/microcosm-build/tests/test_uk_frs_employment.py + packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py + packages/microcosm-build/tests/test_uk_frs_legacy_proxies.py + packages/microcosm-build/tests/test_uk_frs_relationships.py + packages/microcosm-build/tests/test_uk_frs_release.py + packages/microcosm-build/tests/test_uk_frs_spine.py + packages/microcosm-build/tests/test_uk_frs_stochastic.py + packages/microcosm-build/tests/test_uk_geography_ladder.py + packages/microcosm-build/tests/test_uk_geography_sources.py + packages/microcosm-build/tests/test_uk_graph.py + packages/microcosm-build/tests/test_uk_h5_payload_compare.py + packages/microcosm-build/tests/test_uk_hmrc_calibration.py + packages/microcosm-build/tests/test_uk_hmrc_capital_gains.py + packages/microcosm-build/tests/test_uk_hmrc_income.py + packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py + packages/microcosm-build/tests/test_uk_hmrc_replay.py + packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py + packages/microcosm-build/tests/test_uk_identity_stability_receipts.py + packages/microcosm-build/tests/test_uk_incumbent_surface_evaluation.py + packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py + packages/microcosm-build/tests/test_uk_ladder_sni_sources.py + packages/microcosm-build/tests/test_uk_ladder_targets.py + packages/microcosm-build/tests/test_uk_lcfs_consumption.py + packages/microcosm-build/tests/test_uk_ledger_fact_vendoring.py + packages/microcosm-build/tests/test_uk_ledger_targets.py + packages/microcosm-build/tests/test_uk_local_area_crosswalk.py + packages/microcosm-build/tests/test_uk_local_candidate_scorer.py + packages/microcosm-build/tests/test_uk_local_doctrine.py + packages/microcosm-build/tests/test_uk_local_gate_battery.py + packages/microcosm-build/tests/test_uk_local_geography.py + packages/microcosm-build/tests/test_uk_local_holdout.py + packages/microcosm-build/tests/test_uk_local_incumbent_extractor.py + packages/microcosm-build/tests/test_uk_local_release_preflight.py + packages/microcosm-build/tests/test_uk_local_rowwise.py + packages/microcosm-build/tests/test_uk_local_target_census.py + packages/microcosm-build/tests/test_uk_local_targets.py + packages/microcosm-build/tests/test_uk_local_validation_levels.py + packages/microcosm-build/tests/test_uk_measure_simulation.py + packages/microcosm-build/tests/test_uk_national_calibration.py + packages/microcosm-build/tests/test_uk_national_frame.py + packages/microcosm-build/tests/test_uk_national_sampling.py + packages/microcosm-build/tests/test_uk_nhs_allocation.py + packages/microcosm-build/tests/test_uk_ods_tables.py + packages/microcosm-build/tests/test_uk_parity_reference.py + packages/microcosm-build/tests/test_uk_payload_diff_classifier.py + packages/microcosm-build/tests/test_uk_pipr_deferrals.py + packages/microcosm-build/tests/test_uk_population_targets.py + packages/microcosm-build/tests/test_uk_provenance_flags.py + packages/microcosm-build/tests/test_uk_raking.py + packages/microcosm-build/tests/test_uk_regional_uprating.py + packages/microcosm-build/tests/test_uk_release_assembler.py + packages/microcosm-build/tests/test_uk_release_certification.py + packages/microcosm-build/tests/test_uk_release_identity.py + packages/microcosm-build/tests/test_uk_release_input_coverage.py + packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py + packages/microcosm-build/tests/test_uk_rowwise_build_driver.py + packages/microcosm-build/tests/test_uk_rowwise_candidate.py + packages/microcosm-build/tests/test_uk_rowwise_dataset.py + packages/microcosm-build/tests/test_uk_rowwise_dry_run.py + packages/microcosm-build/tests/test_uk_rowwise_geography.py + packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py + packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py + packages/microcosm-build/tests/test_uk_salary_sacrifice.py + packages/microcosm-build/tests/test_uk_signed_differences.py + packages/microcosm-build/tests/test_uk_size_evaluation.py + packages/microcosm-build/tests/test_uk_source_runtime.py + packages/microcosm-build/tests/test_uk_source_stages.py + packages/microcosm-build/tests/test_uk_spi_income.py + packages/microcosm-build/tests/test_uk_spi_spine.py + packages/microcosm-build/tests/test_uk_spi_support.py + packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py + packages/microcosm-build/tests/test_uk_spine_parity_instrument.py + packages/microcosm-build/tests/test_uk_stage_checkpoints.py + packages/microcosm-build/tests/test_uk_stage_health.py + packages/microcosm-build/tests/test_uk_staging_integration.py + packages/microcosm-build/tests/test_uk_staging_provisioning.py + packages/microcosm-build/tests/test_uk_stochastic_tools.py + packages/microcosm-build/tests/test_uk_student_loans.py + packages/microcosm-build/tests/test_uk_take_up_contract.py + packages/microcosm-build/tests/test_uk_take_up_gate.py + packages/microcosm-build/tests/test_uk_target_references.py + packages/microcosm-build/tests/test_uk_terminal_gates.py + packages/microcosm-build/tests/test_uk_tfc_routed_share.py + packages/microcosm-build/tests/test_uk_uc_capital_coherence.py + packages/microcosm-build/tests/test_uk_uc_deduction_attributes.py + packages/microcosm-build/tests/test_uk_uc_family.py + packages/microcosm-build/tests/test_uk_uc_lp_evidence.py + packages/microcosm-build/tests/test_uk_uc_matrix_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_relationships.py + packages/microcosm-build/tests/test_uk_uc_reporter_redraw.py + packages/microcosm-build/tests/test_uk_uc_source_periods.py + packages/microcosm-build/tests/test_uk_uc_support_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_target_measurements.py + packages/microcosm-build/tests/test_uk_was_wealth.py + packages/microcosm-build/tests/test_uk_wealth_resources.py + packages/microcosm-build/tests/test_uk_weighted_integrity.py + packages/microcosm-build/tests/test_uk_weighted_integrity_measurement.py + packages/microcosm-build/tests/test_us_stacked_spine.py + packages/microcosm-frame/tests/test_policyengine_uk_adapter.py +rest: files=410 procs=main shards_per_process=6 + packages/microcosm-build/tests/test_atomic_geography.py + packages/microcosm-build/tests/test_chronicle_epoch.py + packages/microcosm-build/tests/test_ci_test_groups.py + packages/microcosm-build/tests/test_compare_us_fiscal_diagnostics.py + packages/microcosm-build/tests/test_country_spec.py + packages/microcosm-build/tests/test_cross_grain.py + packages/microcosm-build/tests/test_demographics.py + packages/microcosm-build/tests/test_diagnose_us_target_support.py + packages/microcosm-build/tests/test_frame_checkpoint.py + packages/microcosm-build/tests/test_frame_sampling.py + packages/microcosm-build/tests/test_frame_serializer_registry.py + packages/microcosm-build/tests/test_gate_battery.py + packages/microcosm-build/tests/test_gate_battery_contract_example.py + packages/microcosm-build/tests/test_gate_battery_contract_pins.py + packages/microcosm-build/tests/test_gates.py + packages/microcosm-build/tests/test_holdout.py + packages/microcosm-build/tests/test_imputation_lineage_spec.py + packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py + packages/microcosm-build/tests/test_ledger_artifact.py + packages/microcosm-build/tests/test_ledger_targets.py + packages/microcosm-build/tests/test_logbook.py + packages/microcosm-build/tests/test_logbook_adoption.py + packages/microcosm-build/tests/test_logbook_archive.py + packages/microcosm-build/tests/test_logbook_backfill.py + packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py + packages/microcosm-build/tests/test_logbook_cli.py + packages/microcosm-build/tests/test_logbook_env.py + packages/microcosm-build/tests/test_logbook_isolation.py + packages/microcosm-build/tests/test_logbook_prediction_seed.py + packages/microcosm-build/tests/test_monetary_profile.py + packages/microcosm-build/tests/test_monetary_targets.py + packages/microcosm-build/tests/test_outer_stage_runtime.py + packages/microcosm-build/tests/test_plan.py + packages/microcosm-build/tests/test_puf_qrf_chain.py + packages/microcosm-build/tests/test_reform_validation.py + packages/microcosm-build/tests/test_release_input_coverage.py + packages/microcosm-build/tests/test_release_target_parity.py + packages/microcosm-build/tests/test_score_uk_national_candidate.py + packages/microcosm-build/tests/test_source_runtime.py + packages/microcosm-build/tests/test_spec_engine_battery_semantics.py + packages/microcosm-build/tests/test_spec_engine_calibration_semantics.py + packages/microcosm-build/tests/test_spec_engine_compiler_ir.py + packages/microcosm-build/tests/test_spec_engine_country_bundles.py + packages/microcosm-build/tests/test_spec_engine_coverage_tool.py + packages/microcosm-build/tests/test_spec_engine_engine_abi.py + packages/microcosm-build/tests/test_spec_engine_field_usage.py + packages/microcosm-build/tests/test_spec_engine_identity_contracts.py + packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py + packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py + packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py + packages/microcosm-build/tests/test_spec_engine_loader.py + packages/microcosm-build/tests/test_spec_engine_plan_lock.py + packages/microcosm-build/tests/test_spec_engine_publication_semantics.py + packages/microcosm-build/tests/test_spec_engine_reemission.py + packages/microcosm-build/tests/test_spec_engine_resolver.py + packages/microcosm-build/tests/test_spec_engine_schema.py + packages/microcosm-build/tests/test_spec_engine_seeds.py + packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py + packages/microcosm-build/tests/test_spec_engine_take_up_semantics.py + packages/microcosm-build/tests/test_spec_engine_typed_closure.py + packages/microcosm-build/tests/test_spec_engine_yaml.py + packages/microcosm-build/tests/test_spec_only_country_packages.py + packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py + packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py + packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py + packages/microcosm-build/tests/test_spec_seed_identity_lock.py + packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py + packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py + packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py + packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py + packages/microcosm-build/tests/test_spec_worker_identity_lock.py + packages/microcosm-build/tests/test_stage_profile.py + packages/microcosm-build/tests/test_staging.py + packages/microcosm-build/tests/test_staging_v2.py + packages/microcosm-build/tests/test_stochastic_assignment.py + packages/microcosm-build/tests/test_target_materialization.py + packages/microcosm-build/tests/test_target_reference_authoring.py + packages/microcosm-build/tests/test_trace.py + packages/microcosm-build/tests/test_us_acs_housing_source.py + packages/microcosm-build/tests/test_us_acs_inputs.py + packages/microcosm-build/tests/test_us_acs_local_release_tool.py + packages/microcosm-build/tests/test_us_acs_multispine.py + packages/microcosm-build/tests/test_us_acs_multispine_base_builder.py + packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py + packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py + packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py + packages/microcosm-build/tests/test_us_acs_population_catalogue.py + packages/microcosm-build/tests/test_us_acs_pums.py + packages/microcosm-build/tests/test_us_acs_record_fence_scan.py + packages/microcosm-build/tests/test_us_acs_release_predictors.py + packages/microcosm-build/tests/test_us_acs_source_compile_cache.py + packages/microcosm-build/tests/test_us_acs_sources.py + packages/microcosm-build/tests/test_us_acs_transfer.py + packages/microcosm-build/tests/test_us_adult_care.py + packages/microcosm-build/tests/test_us_alimony.py + packages/microcosm-build/tests/test_us_asec_2024_native_population.py + packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py + packages/microcosm-build/tests/test_us_asec_checkpoint.py + packages/microcosm-build/tests/test_us_asec_coverage_authentication.py + packages/microcosm-build/tests/test_us_asec_current_money_source.py + packages/microcosm-build/tests/test_us_asec_demographic_source.py + packages/microcosm-build/tests/test_us_asec_household_coverage_fields.py + packages/microcosm-build/tests/test_us_asec_person_income_source.py + packages/microcosm-build/tests/test_us_asec_pool.py + packages/microcosm-build/tests/test_us_asec_prepared_resources.py + packages/microcosm-build/tests/test_us_asec_source_domain_hardening.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_de.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_national.py + packages/microcosm-build/tests/test_us_atomic_block_sources.py + packages/microcosm-build/tests/test_us_atomic_block_support.py + packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py + packages/microcosm-build/tests/test_us_base_pool.py + packages/microcosm-build/tests/test_us_block_ladder_sources.py + packages/microcosm-build/tests/test_us_buildj_base_script.py + packages/microcosm-build/tests/test_us_bundle_core_contracts.py + packages/microcosm-build/tests/test_us_capital_gain_details.py + packages/microcosm-build/tests/test_us_capital_gain_distributions.py + packages/microcosm-build/tests/test_us_casualty_losses.py + packages/microcosm-build/tests/test_us_child_property_income_graph.py + packages/microcosm-build/tests/test_us_child_property_income_graph_owner.py + packages/microcosm-build/tests/test_us_child_property_income_source.py + packages/microcosm-build/tests/test_us_child_property_income_source_owner.py + packages/microcosm-build/tests/test_us_child_support.py + packages/microcosm-build/tests/test_us_childcare.py + packages/microcosm-build/tests/test_us_common_frame_export_contract.py + packages/microcosm-build/tests/test_us_completion_support_custody.py + packages/microcosm-build/tests/test_us_congressional_district_geography.py + packages/microcosm-build/tests/test_us_congressional_district_vintage.py + packages/microcosm-build/tests/test_us_congressional_district_vintage_crosswalk.py + packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py + packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py + packages/microcosm-build/tests/test_us_current_asec_child_support_source.py + packages/microcosm-build/tests/test_us_current_asec_demographics.py + packages/microcosm-build/tests/test_us_current_asec_dividend_source.py + packages/microcosm-build/tests/test_us_current_asec_income_routing.py + packages/microcosm-build/tests/test_us_current_asec_interest_source.py + packages/microcosm-build/tests/test_us_current_asec_property_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py + packages/microcosm-build/tests/test_us_current_property_income_sources.py + packages/microcosm-build/tests/test_us_current_survey_amounts.py + packages/microcosm-build/tests/test_us_current_survey_geography.py + packages/microcosm-build/tests/test_us_current_survey_health_coverage.py + packages/microcosm-build/tests/test_us_current_survey_household_roles.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_graph.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_source.py + packages/microcosm-build/tests/test_us_current_survey_housing.py + packages/microcosm-build/tests/test_us_current_survey_person_status.py + packages/microcosm-build/tests/test_us_current_survey_person_status_graph.py + packages/microcosm-build/tests/test_us_current_survey_person_status_source.py + packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py + packages/microcosm-build/tests/test_us_current_survey_predictors.py + packages/microcosm-build/tests/test_us_current_survey_property_graph.py + packages/microcosm-build/tests/test_us_current_survey_puf_host.py + packages/microcosm-build/tests/test_us_current_survey_puf_transfer.py + packages/microcosm-build/tests/test_us_dc_ptc_take_up_exclusion.py + packages/microcosm-build/tests/test_us_disability_benefits.py + packages/microcosm-build/tests/test_us_documented_absent_inputs.py + packages/microcosm-build/tests/test_us_domestic_production.py + packages/microcosm-build/tests/test_us_early_head_start_exclusion.py + packages/microcosm-build/tests/test_us_education_assistance_source.py + packages/microcosm-build/tests/test_us_education_inputs.py + packages/microcosm-build/tests/test_us_educator_expenses.py + packages/microcosm-build/tests/test_us_eligibility_inputs.py + packages/microcosm-build/tests/test_us_energy_subsidy.py + packages/microcosm-build/tests/test_us_exact_k_ladder.py + packages/microcosm-build/tests/test_us_exact_k_ladder_e2e.py + packages/microcosm-build/tests/test_us_exact_k_ladder_launcher.py + packages/microcosm-build/tests/test_us_farm_business_income.py + packages/microcosm-build/tests/test_us_financial_assistance_exclusion.py + packages/microcosm-build/tests/test_us_fiscal_leaf_policy.py + packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py + packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py + packages/microcosm-build/tests/test_us_fiscal_targets.py + packages/microcosm-build/tests/test_us_form_4952.py + packages/microcosm-build/tests/test_us_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_full_puf_output_profiles.py + packages/microcosm-build/tests/test_us_geography_integration.py + packages/microcosm-build/tests/test_us_geography_ladder.py + packages/microcosm-build/tests/test_us_graph.py + packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py + packages/microcosm-build/tests/test_us_graph_atomic_person_status.py + packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py + packages/microcosm-build/tests/test_us_graph_current_survey_geography.py + packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py + packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py + packages/microcosm-build/tests/test_us_graph_fiscal_target_snapshots.py + packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_selection.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_stream.py + packages/microcosm-build/tests/test_us_graph_property_income.py + packages/microcosm-build/tests/test_us_graph_property_income_receipts.py + packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py + packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py + packages/microcosm-build/tests/test_us_graph_survey_completion.py + packages/microcosm-build/tests/test_us_graph_survey_population.py + packages/microcosm-build/tests/test_us_graph_survey_puf55.py + packages/microcosm-build/tests/test_us_graph_us_survey_enrichment.py + packages/microcosm-build/tests/test_us_hours_worked.py + packages/microcosm-build/tests/test_us_housing_inputs.py + packages/microcosm-build/tests/test_us_housing_participation.py + packages/microcosm-build/tests/test_us_immigration.py + packages/microcosm-build/tests/test_us_input_mass.py + packages/microcosm-build/tests/test_us_l0_refit_export.py + packages/microcosm-build/tests/test_us_late_producer_dag.py + packages/microcosm-build/tests/test_us_medicaid_take_up.py + packages/microcosm-build/tests/test_us_medicare_take_up.py + packages/microcosm-build/tests/test_us_misc_itemized.py + packages/microcosm-build/tests/test_us_multispine_pool.py + packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py + packages/microcosm-build/tests/test_us_multispine_puf_clone.py + packages/microcosm-build/tests/test_us_national_age_counts.py + packages/microcosm-build/tests/test_us_native_row_ceilings.py + packages/microcosm-build/tests/test_us_native_verify_once_epoch.py + packages/microcosm-build/tests/test_us_nonzero_shares.py + packages/microcosm-build/tests/test_us_org_wages.py + packages/microcosm-build/tests/test_us_other_health_insurance.py + packages/microcosm-build/tests/test_us_parity_reference.py + packages/microcosm-build/tests/test_us_plan.py + packages/microcosm-build/tests/test_us_pool_input_consumers.py + packages/microcosm-build/tests/test_us_population_input_coverage.py + packages/microcosm-build/tests/test_us_post_transfer_calibration.py + packages/microcosm-build/tests/test_us_postclone_geography_identity.py + packages/microcosm-build/tests/test_us_pregnancy.py + packages/microcosm-build/tests/test_us_prior_year_income.py + packages/microcosm-build/tests/test_us_property_completion_graph.py + packages/microcosm-build/tests/test_us_property_completion_routing.py + packages/microcosm-build/tests/test_us_puf55_heterogeneous_manifest_seals.py + packages/microcosm-build/tests/test_us_puf55_model_boundaries.py + packages/microcosm-build/tests/test_us_puf55_route_finalization.py + packages/microcosm-build/tests/test_us_puf55_route_numerical_finalization.py + packages/microcosm-build/tests/test_us_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_profile.py + packages/microcosm-build/tests/test_us_puf59_canonical.py + packages/microcosm-build/tests/test_us_puf_aggregate_records.py + packages/microcosm-build/tests/test_us_puf_capital_gains_tail.py + packages/microcosm-build/tests/test_us_puf_detail_transfer.py + packages/microcosm-build/tests/test_us_puf_donor_io.py + packages/microcosm-build/tests/test_us_puf_e01000_reconciliation.py + packages/microcosm-build/tests/test_us_puf_e19200_decomposition.py + packages/microcosm-build/tests/test_us_puf_price_baseline.py + packages/microcosm-build/tests/test_us_puf_support.py + packages/microcosm-build/tests/test_us_puf_support_base_builder.py + packages/microcosm-build/tests/test_us_puf_support_base_equivalence.py + packages/microcosm-build/tests/test_us_puf_tail_bound.py + packages/microcosm-build/tests/test_us_puf_target2024_growth.py + packages/microcosm-build/tests/test_us_puma_ladder.py + packages/microcosm-build/tests/test_us_puma_ladder_sources.py + packages/microcosm-build/tests/test_us_qbi_inputs.py + packages/microcosm-build/tests/test_us_relationship_inputs.py + packages/microcosm-build/tests/test_us_release_gate_preflight.py + packages/microcosm-build/tests/test_us_reported_coverage_vintage_gate.py + packages/microcosm-build/tests/test_us_retirement_contributions.py + packages/microcosm-build/tests/test_us_retirement_distributions.py + packages/microcosm-build/tests/test_us_runtime_facade_union.py + packages/microcosm-build/tests/test_us_runtime_import_order.py + packages/microcosm-build/tests/test_us_salt_refund_income.py + packages/microcosm-build/tests/test_us_scf_auto_loans.py + packages/microcosm-build/tests/test_us_scf_wealth.py + packages/microcosm-build/tests/test_us_sipp_financial_assets.py + packages/microcosm-build/tests/test_us_sipp_head_start.py + packages/microcosm-build/tests/test_us_sipp_tips.py + packages/microcosm-build/tests/test_us_sipp_vehicles.py + packages/microcosm-build/tests/test_us_snap_discretionary_exemption.py + packages/microcosm-build/tests/test_us_snap_state_take_up.py + packages/microcosm-build/tests/test_us_snap_take_up.py + packages/microcosm-build/tests/test_us_source_coverage.py + packages/microcosm-build/tests/test_us_source_runtime.py + packages/microcosm-build/tests/test_us_spec_bundle.py + packages/microcosm-build/tests/test_us_spine_agreement.py + packages/microcosm-build/tests/test_us_spine_assembly.py + packages/microcosm-build/tests/test_us_spine_blindness.py + packages/microcosm-build/tests/test_us_spm_resources.py + packages/microcosm-build/tests/test_us_spm_role_enrichment_builder.py + packages/microcosm-build/tests/test_us_spm_role_source.py + packages/microcosm-build/tests/test_us_ssi_disability_criteria.py + packages/microcosm-build/tests/test_us_ssi_take_up.py + packages/microcosm-build/tests/test_us_state_files_scorer.py + packages/microcosm-build/tests/test_us_survey_age_activation.py + packages/microcosm-build/tests/test_us_survey_age_artifact.py + packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_age_calibration_run.py + packages/microcosm-build/tests/test_us_survey_age_development.py + packages/microcosm-build/tests/test_us_survey_age_sources.py + packages/microcosm-build/tests/test_us_survey_calibration.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py + packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py + packages/microcosm-build/tests/test_us_survey_financial_successor.py + packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py + packages/microcosm-build/tests/test_us_survey_observed_age.py + packages/microcosm-build/tests/test_us_survey_origin_budget.py + packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_origin_budget_detached_views.py + packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py + packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py + packages/microcosm-build/tests/test_us_survey_population_preparation.py + packages/microcosm-build/tests/test_us_survey_population_replay.py + packages/microcosm-build/tests/test_us_survey_social_security.py + packages/microcosm-build/tests/test_us_survivor_benefits_exclusion.py + packages/microcosm-build/tests/test_us_take_up.py + packages/microcosm-build/tests/test_us_take_up_contract.py + packages/microcosm-build/tests/test_us_voluntary_filing.py + packages/microcosm-build/tests/test_us_warm_start_selection.py + packages/microcosm-build/tests/test_us_weeks_unemployed.py + packages/microcosm-build/tests/test_us_whole_series_selection_sites.py + packages/microcosm-build/tests/test_us_wic_claim.py + packages/microcosm-build/tests/test_us_wic_nutritional_risk_exclusion.py + packages/microcosm-build/tests/test_us_workers_compensation.py + packages/microcosm-build/tests/test_validation_input_coverage.py + packages/microcosm-calibrate/tests/test_compat.py + packages/microcosm-calibrate/tests/test_diagnostics.py + packages/microcosm-calibrate/tests/test_exact_k.py + packages/microcosm-calibrate/tests/test_group_bounds.py + packages/microcosm-calibrate/tests/test_grouped_fixed_support.py + packages/microcosm-calibrate/tests/test_hierarchy.py + packages/microcosm-calibrate/tests/test_informed_gates.py + packages/microcosm-calibrate/tests/test_kernels.py + packages/microcosm-calibrate/tests/test_matrix.py + packages/microcosm-calibrate/tests/test_monetary_binding.py + packages/microcosm-calibrate/tests/test_provider_labels.py + packages/microcosm-calibrate/tests/test_registry.py + packages/microcosm-calibrate/tests/test_score.py + packages/microcosm-calibrate/tests/test_solve.py + packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py + packages/microcosm-calibrate/tests/test_sparse_solve.py + packages/microcosm-calibrate/tests/test_target.py + packages/microcosm-calibrate/tests/test_target_snapshots.py + packages/microcosm-calibrate/tests/test_variable_labels.py + packages/microcosm-data/tests/test_compat.py + packages/microcosm-data/tests/test_contract.py + packages/microcosm-data/tests/test_h5_enrichment.py + packages/microcosm-data/tests/test_loader.py + packages/microcosm-data/tests/test_local_area_contract.py + packages/microcosm-data/tests/test_publish_guard.py + packages/microcosm-data/tests/test_registry.py + packages/microcosm-data/tests/test_release.py + packages/microcosm-data/tests/test_slack.py + packages/microcosm-data/tests/test_source_enrichment.py + packages/microcosm-data/tests/test_uk_dense_release_contract.py + packages/microcosm-fit/tests/test_cache.py + packages/microcosm-fit/tests/test_compat.py + packages/microcosm-fit/tests/test_dataframe_fit.py + packages/microcosm-fit/tests/test_graph_joint_empirical.py + packages/microcosm-fit/tests/test_graph_legacy_matrix.py + packages/microcosm-fit/tests/test_graph_legacy_qrf.py + packages/microcosm-fit/tests/test_graph_signed_reconciliation.py + packages/microcosm-fit/tests/test_joint_empirical.py + packages/microcosm-fit/tests/test_kernels.py + packages/microcosm-fit/tests/test_model.py + packages/microcosm-fit/tests/test_qrf.py + packages/microcosm-fit/tests/test_qrf_stateless.py + packages/microcosm-fit/tests/test_qrf_target.py + packages/microcosm-fit/tests/test_signed_reconciliation.py + packages/microcosm-fit/tests/test_weighted_fit_contract.py + packages/microcosm-frame/tests/test_accounting.py + packages/microcosm-frame/tests/test_adapters.py + packages/microcosm-frame/tests/test_axiom_adapter.py + packages/microcosm-frame/tests/test_bundle.py + packages/microcosm-frame/tests/test_contracts.py + packages/microcosm-frame/tests/test_kernels.py + packages/microcosm-frame/tests/test_materialize.py + packages/microcosm-frame/tests/test_place.py + packages/microcosm-frame/tests/test_policyengine_us_adapter.py + packages/microcosm-frame/tests/test_policyengine_us_consumer_index.py + packages/microcosm-frame/tests/test_policyengine_us_metadata_index.py + packages/microcosm-frame/tests/test_policyengine_us_ownership_index.py + packages/microcosm-frame/tests/test_rules_engine_contract.py + packages/microcosm-frame/tests/test_units.py + packages/microcosm-frame/tests/test_weights_schema.py + packages/microcosm-graph/tests/test_acceptance_a_identity.py + packages/microcosm-graph/tests/test_acceptance_b_ownership.py + packages/microcosm-graph/tests/test_acceptance_c_seeds.py + packages/microcosm-graph/tests/test_acceptance_d_weights.py + packages/microcosm-graph/tests/test_acceptance_e_store.py + packages/microcosm-graph/tests/test_acceptance_f_gates.py + packages/microcosm-graph/tests/test_acceptance_g_legibility.py + packages/microcosm-graph/tests/test_acceptance_h_parity.py + packages/microcosm-graph/tests/test_acceptance_replays.py + packages/microcosm-graph/tests/test_frame_metadata_store.py + packages/microcosm-graph/tests/test_graph_acceptance_burndown.py + packages/microcosm-graph/tests/test_graph_canonical.py + packages/microcosm-graph/tests/test_graph_codecs.py + packages/microcosm-graph/tests/test_graph_decl.py + packages/microcosm-graph/tests/test_graph_executor.py + packages/microcosm-graph/tests/test_graph_executor_scalar_float.py + packages/microcosm-graph/tests/test_graph_executor_series_stream.py + packages/microcosm-graph/tests/test_graph_executor_source_identity.py + packages/microcosm-graph/tests/test_graph_explain.py + packages/microcosm-graph/tests/test_graph_interface_lock.py + packages/microcosm-graph/tests/test_graph_kernel_contract.py + packages/microcosm-graph/tests/test_graph_keys.py + packages/microcosm-graph/tests/test_graph_manifest.py + packages/microcosm-graph/tests/test_graph_parity_pins.py + packages/microcosm-graph/tests/test_graph_parity_repin.py + packages/microcosm-graph/tests/test_graph_population.py + packages/microcosm-graph/tests/test_graph_randomness.py + packages/microcosm-graph/tests/test_graph_serialize.py + packages/microcosm-graph/tests/test_graph_storage_parts_whole_series.py + packages/microcosm-graph/tests/test_graph_store.py + packages/microcosm-graph/tests/test_graph_verification_epoch.py + packages/microcosm-graph/tests/test_graph_view.py +[engine] +shared-spec: files=87 procs=main shards_per_process=1 + packages/microcosm-build/tests/test_atomic_geography.py + packages/microcosm-build/tests/test_chronicle_epoch.py + packages/microcosm-build/tests/test_ci_test_groups.py + packages/microcosm-build/tests/test_compare_us_fiscal_diagnostics.py + packages/microcosm-build/tests/test_country_spec.py + packages/microcosm-build/tests/test_cross_grain.py + packages/microcosm-build/tests/test_demographics.py + packages/microcosm-build/tests/test_diagnose_us_target_support.py + packages/microcosm-build/tests/test_frame_checkpoint.py + packages/microcosm-build/tests/test_frame_sampling.py + packages/microcosm-build/tests/test_frame_serializer_registry.py + packages/microcosm-build/tests/test_gate_battery.py + packages/microcosm-build/tests/test_gate_battery_contract_example.py + packages/microcosm-build/tests/test_gate_battery_contract_pins.py + packages/microcosm-build/tests/test_gates.py + packages/microcosm-build/tests/test_holdout.py + packages/microcosm-build/tests/test_imputation_lineage_spec.py + packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py + packages/microcosm-build/tests/test_ledger_artifact.py + packages/microcosm-build/tests/test_ledger_targets.py + packages/microcosm-build/tests/test_logbook.py + packages/microcosm-build/tests/test_logbook_adoption.py + packages/microcosm-build/tests/test_logbook_archive.py + packages/microcosm-build/tests/test_logbook_backfill.py + packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py + packages/microcosm-build/tests/test_logbook_cli.py + packages/microcosm-build/tests/test_logbook_env.py + packages/microcosm-build/tests/test_logbook_isolation.py + packages/microcosm-build/tests/test_logbook_prediction_seed.py + packages/microcosm-build/tests/test_monetary_profile.py + packages/microcosm-build/tests/test_monetary_targets.py + packages/microcosm-build/tests/test_outer_stage_runtime.py + packages/microcosm-build/tests/test_plan.py + packages/microcosm-build/tests/test_reform_validation.py + packages/microcosm-build/tests/test_release_input_coverage.py + packages/microcosm-build/tests/test_release_target_parity.py + packages/microcosm-build/tests/test_score_uk_national_candidate.py + packages/microcosm-build/tests/test_source_runtime.py + packages/microcosm-build/tests/test_spec_engine_battery_semantics.py + packages/microcosm-build/tests/test_spec_engine_calibration_semantics.py + packages/microcosm-build/tests/test_spec_engine_compiler_ir.py + packages/microcosm-build/tests/test_spec_engine_country_bundles.py + packages/microcosm-build/tests/test_spec_engine_coverage_tool.py + packages/microcosm-build/tests/test_spec_engine_engine_abi.py + packages/microcosm-build/tests/test_spec_engine_field_usage.py + packages/microcosm-build/tests/test_spec_engine_identity_contracts.py + packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py + packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py + packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py + packages/microcosm-build/tests/test_spec_engine_loader.py + packages/microcosm-build/tests/test_spec_engine_plan_lock.py + packages/microcosm-build/tests/test_spec_engine_publication_semantics.py + packages/microcosm-build/tests/test_spec_engine_reemission.py + packages/microcosm-build/tests/test_spec_engine_resolver.py + packages/microcosm-build/tests/test_spec_engine_schema.py + packages/microcosm-build/tests/test_spec_engine_seeds.py + packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py + packages/microcosm-build/tests/test_spec_engine_take_up_semantics.py + packages/microcosm-build/tests/test_spec_engine_typed_closure.py + packages/microcosm-build/tests/test_spec_engine_yaml.py + packages/microcosm-build/tests/test_spec_only_country_packages.py + packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py + packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py + packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py + packages/microcosm-build/tests/test_spec_seed_identity_lock.py + packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py + packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py + packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py + packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py + packages/microcosm-build/tests/test_spec_worker_identity_lock.py + packages/microcosm-build/tests/test_stage_profile.py + packages/microcosm-build/tests/test_staging.py + packages/microcosm-build/tests/test_staging_v2.py + packages/microcosm-build/tests/test_stochastic_assignment.py + packages/microcosm-build/tests/test_target_materialization.py + packages/microcosm-build/tests/test_target_reference_authoring.py + packages/microcosm-build/tests/test_trace.py + packages/microcosm-build/tests/test_validation_input_coverage.py + packages/microcosm-frame/tests/test_accounting.py + packages/microcosm-frame/tests/test_adapters.py + packages/microcosm-frame/tests/test_axiom_adapter.py + packages/microcosm-frame/tests/test_bundle.py + packages/microcosm-frame/tests/test_contracts.py + packages/microcosm-frame/tests/test_materialize.py + packages/microcosm-frame/tests/test_place.py + packages/microcosm-frame/tests/test_units.py + packages/microcosm-frame/tests/test_weights_schema.py +us-p: files=33 procs=main shards_per_process=1 + packages/microcosm-build/tests/test_puf_qrf_chain.py + packages/microcosm-build/tests/test_us_parity_reference.py + packages/microcosm-build/tests/test_us_plan.py + packages/microcosm-build/tests/test_us_pool_input_consumers.py + packages/microcosm-build/tests/test_us_population_input_coverage.py + packages/microcosm-build/tests/test_us_post_transfer_calibration.py + packages/microcosm-build/tests/test_us_postclone_geography_identity.py + packages/microcosm-build/tests/test_us_pregnancy.py + packages/microcosm-build/tests/test_us_prior_year_income.py + packages/microcosm-build/tests/test_us_property_completion_graph.py + packages/microcosm-build/tests/test_us_property_completion_routing.py + packages/microcosm-build/tests/test_us_puf55_heterogeneous_manifest_seals.py + packages/microcosm-build/tests/test_us_puf55_model_boundaries.py + packages/microcosm-build/tests/test_us_puf55_route_finalization.py + packages/microcosm-build/tests/test_us_puf55_route_numerical_finalization.py + packages/microcosm-build/tests/test_us_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_profile.py + packages/microcosm-build/tests/test_us_puf59_canonical.py + packages/microcosm-build/tests/test_us_puf_aggregate_records.py + packages/microcosm-build/tests/test_us_puf_capital_gains_tail.py + packages/microcosm-build/tests/test_us_puf_detail_transfer.py + packages/microcosm-build/tests/test_us_puf_donor_io.py + packages/microcosm-build/tests/test_us_puf_e01000_reconciliation.py + packages/microcosm-build/tests/test_us_puf_e19200_decomposition.py + packages/microcosm-build/tests/test_us_puf_price_baseline.py + packages/microcosm-build/tests/test_us_puf_support.py + packages/microcosm-build/tests/test_us_puf_support_base_builder.py + packages/microcosm-build/tests/test_us_puf_support_base_equivalence.py + packages/microcosm-build/tests/test_us_puf_tail_bound.py + packages/microcosm-build/tests/test_us_puf_target2024_growth.py + packages/microcosm-build/tests/test_us_puma_ladder.py + packages/microcosm-build/tests/test_us_puma_ladder_sources.py +us-qs: files=61 procs=build, frame shards_per_process=1 + packages/microcosm-build/tests/test_us_qbi_inputs.py + packages/microcosm-build/tests/test_us_relationship_inputs.py + packages/microcosm-build/tests/test_us_release_gate_preflight.py + packages/microcosm-build/tests/test_us_release_head_to_head_scorer.py + packages/microcosm-build/tests/test_us_reported_coverage_vintage_gate.py + packages/microcosm-build/tests/test_us_retirement_contributions.py + packages/microcosm-build/tests/test_us_retirement_distributions.py + packages/microcosm-build/tests/test_us_runtime_facade_union.py + packages/microcosm-build/tests/test_us_runtime_import_order.py + packages/microcosm-build/tests/test_us_salt_refund_income.py + packages/microcosm-build/tests/test_us_scf_auto_loans.py + packages/microcosm-build/tests/test_us_scf_wealth.py + packages/microcosm-build/tests/test_us_sipp_financial_assets.py + packages/microcosm-build/tests/test_us_sipp_head_start.py + packages/microcosm-build/tests/test_us_sipp_tips.py + packages/microcosm-build/tests/test_us_sipp_vehicles.py + packages/microcosm-build/tests/test_us_snap_discretionary_exemption.py + packages/microcosm-build/tests/test_us_snap_state_take_up.py + packages/microcosm-build/tests/test_us_snap_take_up.py + packages/microcosm-build/tests/test_us_source_coverage.py + packages/microcosm-build/tests/test_us_source_runtime.py + packages/microcosm-build/tests/test_us_spec_bundle.py + packages/microcosm-build/tests/test_us_spine_agreement.py + packages/microcosm-build/tests/test_us_spine_assembly.py + packages/microcosm-build/tests/test_us_spine_blindness.py + packages/microcosm-build/tests/test_us_spm_resources.py + packages/microcosm-build/tests/test_us_spm_role_enrichment_builder.py + packages/microcosm-build/tests/test_us_spm_role_source.py + packages/microcosm-build/tests/test_us_ssi_disability_criteria.py + packages/microcosm-build/tests/test_us_ssi_take_up.py + packages/microcosm-build/tests/test_us_stacked_spine.py + packages/microcosm-build/tests/test_us_state_files_scorer.py + packages/microcosm-build/tests/test_us_survey_age_activation.py + packages/microcosm-build/tests/test_us_survey_age_artifact.py + packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_age_calibration_run.py + packages/microcosm-build/tests/test_us_survey_age_development.py + packages/microcosm-build/tests/test_us_survey_age_sources.py + packages/microcosm-build/tests/test_us_survey_calibration.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py + packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py + packages/microcosm-build/tests/test_us_survey_financial_successor.py + packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py + packages/microcosm-build/tests/test_us_survey_observed_age.py + packages/microcosm-build/tests/test_us_survey_origin_budget.py + packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_origin_budget_detached_views.py + packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py + packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py + packages/microcosm-build/tests/test_us_survey_population_preparation.py + packages/microcosm-build/tests/test_us_survey_population_replay.py + packages/microcosm-build/tests/test_us_survey_social_security.py + packages/microcosm-build/tests/test_us_survivor_benefits_exclusion.py + packages/microcosm-frame/tests/test_kernels.py + packages/microcosm-frame/tests/test_policyengine_us_adapter.py + packages/microcosm-frame/tests/test_policyengine_us_consumer_index.py + packages/microcosm-frame/tests/test_policyengine_us_metadata_index.py + packages/microcosm-frame/tests/test_policyengine_us_ownership_index.py + packages/microcosm-frame/tests/test_rules_engine_contract.py +us-not: files=22 procs=main shards_per_process=1 + packages/microcosm-build/tests/test_us_national_age_counts.py + packages/microcosm-build/tests/test_us_native_row_ceilings.py + packages/microcosm-build/tests/test_us_native_verify_once_epoch.py + packages/microcosm-build/tests/test_us_nonzero_shares.py + packages/microcosm-build/tests/test_us_org_wages.py + packages/microcosm-build/tests/test_us_other_health_insurance.py + packages/microcosm-build/tests/test_us_take_up.py + packages/microcosm-build/tests/test_us_take_up_contract.py + packages/microcosm-build/tests/test_us_trade_census_imports.py + packages/microcosm-build/tests/test_us_trade_crosscheck.py + packages/microcosm-build/tests/test_us_trade_entries_cli.py + packages/microcosm-build/tests/test_us_trade_entry_generator.py + packages/microcosm-build/tests/test_us_trade_facts.py + packages/microcosm-build/tests/test_us_trade_imdb_bulk.py + packages/microcosm-build/tests/test_us_trade_imdb_goldens.py + packages/microcosm-build/tests/test_us_voluntary_filing.py + packages/microcosm-build/tests/test_us_warm_start_selection.py + packages/microcosm-build/tests/test_us_weeks_unemployed.py + packages/microcosm-build/tests/test_us_whole_series_selection_sites.py + packages/microcosm-build/tests/test_us_wic_claim.py + packages/microcosm-build/tests/test_us_wic_nutritional_risk_exclusion.py + packages/microcosm-build/tests/test_us_workers_compensation.py +us-am: files=217 procs=build, other-shards shards_per_process=4 + packages/microcosm-build/tests/test_us_acs_housing_source.py + packages/microcosm-build/tests/test_us_acs_inputs.py + packages/microcosm-build/tests/test_us_acs_local_release_tool.py + packages/microcosm-build/tests/test_us_acs_multispine.py + packages/microcosm-build/tests/test_us_acs_multispine_base_builder.py + packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py + packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py + packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py + packages/microcosm-build/tests/test_us_acs_population_catalogue.py + packages/microcosm-build/tests/test_us_acs_pums.py + packages/microcosm-build/tests/test_us_acs_record_fence_scan.py + packages/microcosm-build/tests/test_us_acs_release_predictors.py + packages/microcosm-build/tests/test_us_acs_source_compile_cache.py + packages/microcosm-build/tests/test_us_acs_sources.py + packages/microcosm-build/tests/test_us_acs_transfer.py + packages/microcosm-build/tests/test_us_adult_care.py + packages/microcosm-build/tests/test_us_alimony.py + packages/microcosm-build/tests/test_us_asec_2024_native_population.py + packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py + packages/microcosm-build/tests/test_us_asec_checkpoint.py + packages/microcosm-build/tests/test_us_asec_coverage_authentication.py + packages/microcosm-build/tests/test_us_asec_current_money_source.py + packages/microcosm-build/tests/test_us_asec_demographic_source.py + packages/microcosm-build/tests/test_us_asec_household_coverage_fields.py + packages/microcosm-build/tests/test_us_asec_person_income_source.py + packages/microcosm-build/tests/test_us_asec_pool.py + packages/microcosm-build/tests/test_us_asec_prepared_resources.py + packages/microcosm-build/tests/test_us_asec_source_domain_hardening.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_de.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_national.py + packages/microcosm-build/tests/test_us_atomic_block_sources.py + packages/microcosm-build/tests/test_us_atomic_block_support.py + packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py + packages/microcosm-build/tests/test_us_base_pool.py + packages/microcosm-build/tests/test_us_block_ladder_sources.py + packages/microcosm-build/tests/test_us_buildj_base_script.py + packages/microcosm-build/tests/test_us_bundle_core_contracts.py + packages/microcosm-build/tests/test_us_capital_gain_details.py + packages/microcosm-build/tests/test_us_capital_gain_distributions.py + packages/microcosm-build/tests/test_us_casualty_losses.py + packages/microcosm-build/tests/test_us_child_property_income_graph.py + packages/microcosm-build/tests/test_us_child_property_income_graph_owner.py + packages/microcosm-build/tests/test_us_child_property_income_source.py + packages/microcosm-build/tests/test_us_child_property_income_source_owner.py + packages/microcosm-build/tests/test_us_child_support.py + packages/microcosm-build/tests/test_us_childcare.py + packages/microcosm-build/tests/test_us_common_frame_export_contract.py + packages/microcosm-build/tests/test_us_completion_support_custody.py + packages/microcosm-build/tests/test_us_congressional_district_geography.py + packages/microcosm-build/tests/test_us_congressional_district_vintage.py + packages/microcosm-build/tests/test_us_congressional_district_vintage_crosswalk.py + packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py + packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py + packages/microcosm-build/tests/test_us_current_asec_child_support_source.py + packages/microcosm-build/tests/test_us_current_asec_demographics.py + packages/microcosm-build/tests/test_us_current_asec_dividend_source.py + packages/microcosm-build/tests/test_us_current_asec_income_routing.py + packages/microcosm-build/tests/test_us_current_asec_interest_source.py + packages/microcosm-build/tests/test_us_current_asec_property_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py + packages/microcosm-build/tests/test_us_current_property_income_sources.py + packages/microcosm-build/tests/test_us_current_survey_amounts.py + packages/microcosm-build/tests/test_us_current_survey_geography.py + packages/microcosm-build/tests/test_us_current_survey_health_coverage.py + packages/microcosm-build/tests/test_us_current_survey_household_roles.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_graph.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_source.py + packages/microcosm-build/tests/test_us_current_survey_housing.py + packages/microcosm-build/tests/test_us_current_survey_person_status.py + packages/microcosm-build/tests/test_us_current_survey_person_status_graph.py + packages/microcosm-build/tests/test_us_current_survey_person_status_source.py + packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py + packages/microcosm-build/tests/test_us_current_survey_predictors.py + packages/microcosm-build/tests/test_us_current_survey_property_graph.py + packages/microcosm-build/tests/test_us_current_survey_puf_host.py + packages/microcosm-build/tests/test_us_current_survey_puf_transfer.py + packages/microcosm-build/tests/test_us_dc_ptc_take_up_exclusion.py + packages/microcosm-build/tests/test_us_disability_benefits.py + packages/microcosm-build/tests/test_us_documented_absent_inputs.py + packages/microcosm-build/tests/test_us_domestic_production.py + packages/microcosm-build/tests/test_us_early_head_start_exclusion.py + packages/microcosm-build/tests/test_us_education_assistance_source.py + packages/microcosm-build/tests/test_us_education_inputs.py + packages/microcosm-build/tests/test_us_educator_expenses.py + packages/microcosm-build/tests/test_us_eligibility_inputs.py + packages/microcosm-build/tests/test_us_energy_subsidy.py + packages/microcosm-build/tests/test_us_exact_k_ladder.py + packages/microcosm-build/tests/test_us_exact_k_ladder_e2e.py + packages/microcosm-build/tests/test_us_exact_k_ladder_launcher.py + packages/microcosm-build/tests/test_us_farm_business_income.py + packages/microcosm-build/tests/test_us_financial_assistance_exclusion.py + packages/microcosm-build/tests/test_us_fiscal_leaf_policy.py + packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py + packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py + packages/microcosm-build/tests/test_us_fiscal_targets.py + packages/microcosm-build/tests/test_us_form_4952.py + packages/microcosm-build/tests/test_us_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_full_puf_output_profiles.py + packages/microcosm-build/tests/test_us_geography_integration.py + packages/microcosm-build/tests/test_us_geography_ladder.py + packages/microcosm-build/tests/test_us_graph.py + packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py + packages/microcosm-build/tests/test_us_graph_atomic_person_status.py + packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py + packages/microcosm-build/tests/test_us_graph_current_survey_geography.py + packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py + packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py + packages/microcosm-build/tests/test_us_graph_fiscal_target_snapshots.py + packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_selection.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_stream.py + packages/microcosm-build/tests/test_us_graph_property_income.py + packages/microcosm-build/tests/test_us_graph_property_income_receipts.py + packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py + packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py + packages/microcosm-build/tests/test_us_graph_survey_completion.py + packages/microcosm-build/tests/test_us_graph_survey_population.py + packages/microcosm-build/tests/test_us_graph_survey_puf55.py + packages/microcosm-build/tests/test_us_graph_us_survey_enrichment.py + packages/microcosm-build/tests/test_us_hours_worked.py + packages/microcosm-build/tests/test_us_housing_inputs.py + packages/microcosm-build/tests/test_us_housing_participation.py + packages/microcosm-build/tests/test_us_immigration.py + packages/microcosm-build/tests/test_us_input_mass.py + packages/microcosm-build/tests/test_us_l0_refit_export.py + packages/microcosm-build/tests/test_us_late_producer_dag.py + packages/microcosm-build/tests/test_us_medicaid_take_up.py + packages/microcosm-build/tests/test_us_medicare_take_up.py + packages/microcosm-build/tests/test_us_misc_itemized.py + packages/microcosm-build/tests/test_us_multispine_pool.py + packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py + packages/microcosm-build/tests/test_us_multispine_pool_tool.py + packages/microcosm-build/tests/test_us_multispine_puf_clone.py + packages/microcosm-calibrate/tests/test_compat.py + packages/microcosm-calibrate/tests/test_diagnostics.py + packages/microcosm-calibrate/tests/test_exact_k.py + packages/microcosm-calibrate/tests/test_group_bounds.py + packages/microcosm-calibrate/tests/test_grouped_fixed_support.py + packages/microcosm-calibrate/tests/test_hierarchy.py + packages/microcosm-calibrate/tests/test_informed_gates.py + packages/microcosm-calibrate/tests/test_kernels.py + packages/microcosm-calibrate/tests/test_matrix.py + packages/microcosm-calibrate/tests/test_monetary_binding.py + packages/microcosm-calibrate/tests/test_provider_labels.py + packages/microcosm-calibrate/tests/test_registry.py + packages/microcosm-calibrate/tests/test_score.py + packages/microcosm-calibrate/tests/test_solve.py + packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py + packages/microcosm-calibrate/tests/test_sparse_solve.py + packages/microcosm-calibrate/tests/test_target.py + packages/microcosm-calibrate/tests/test_target_snapshots.py + packages/microcosm-calibrate/tests/test_variable_labels.py + packages/microcosm-data/tests/test_compat.py + packages/microcosm-data/tests/test_contract.py + packages/microcosm-data/tests/test_h5_enrichment.py + packages/microcosm-data/tests/test_loader.py + packages/microcosm-data/tests/test_local_area_contract.py + packages/microcosm-data/tests/test_publish_guard.py + packages/microcosm-data/tests/test_registry.py + packages/microcosm-data/tests/test_release.py + packages/microcosm-data/tests/test_slack.py + packages/microcosm-data/tests/test_source_enrichment.py + packages/microcosm-data/tests/test_uk_dense_release_contract.py + packages/microcosm-fit/tests/test_cache.py + packages/microcosm-fit/tests/test_compat.py + packages/microcosm-fit/tests/test_dataframe_fit.py + packages/microcosm-fit/tests/test_graph_joint_empirical.py + packages/microcosm-fit/tests/test_graph_legacy_matrix.py + packages/microcosm-fit/tests/test_graph_legacy_qrf.py + packages/microcosm-fit/tests/test_graph_signed_reconciliation.py + packages/microcosm-fit/tests/test_joint_empirical.py + packages/microcosm-fit/tests/test_kernels.py + packages/microcosm-fit/tests/test_model.py + packages/microcosm-fit/tests/test_qrf.py + packages/microcosm-fit/tests/test_qrf_stateless.py + packages/microcosm-fit/tests/test_qrf_target.py + packages/microcosm-fit/tests/test_signed_reconciliation.py + packages/microcosm-fit/tests/test_weighted_fit_contract.py + packages/microcosm-graph/tests/test_acceptance_a_identity.py + packages/microcosm-graph/tests/test_acceptance_b_ownership.py + packages/microcosm-graph/tests/test_acceptance_c_seeds.py + packages/microcosm-graph/tests/test_acceptance_d_weights.py + packages/microcosm-graph/tests/test_acceptance_e_store.py + packages/microcosm-graph/tests/test_acceptance_f_gates.py + packages/microcosm-graph/tests/test_acceptance_g_legibility.py + packages/microcosm-graph/tests/test_acceptance_h_parity.py + packages/microcosm-graph/tests/test_acceptance_replays.py + packages/microcosm-graph/tests/test_frame_metadata_store.py + packages/microcosm-graph/tests/test_graph_acceptance_burndown.py + packages/microcosm-graph/tests/test_graph_canonical.py + packages/microcosm-graph/tests/test_graph_codecs.py + packages/microcosm-graph/tests/test_graph_decl.py + packages/microcosm-graph/tests/test_graph_executor.py + packages/microcosm-graph/tests/test_graph_executor_scalar_float.py + packages/microcosm-graph/tests/test_graph_executor_series_stream.py + packages/microcosm-graph/tests/test_graph_executor_source_identity.py + packages/microcosm-graph/tests/test_graph_explain.py + packages/microcosm-graph/tests/test_graph_interface_lock.py + packages/microcosm-graph/tests/test_graph_kernel_contract.py + packages/microcosm-graph/tests/test_graph_keys.py + packages/microcosm-graph/tests/test_graph_manifest.py + packages/microcosm-graph/tests/test_graph_parity_pins.py + packages/microcosm-graph/tests/test_graph_parity_repin.py + packages/microcosm-graph/tests/test_graph_population.py + packages/microcosm-graph/tests/test_graph_randomness.py + packages/microcosm-graph/tests/test_graph_serialize.py + packages/microcosm-graph/tests/test_graph_storage_parts_whole_series.py + packages/microcosm-graph/tests/test_graph_store.py + packages/microcosm-graph/tests/test_graph_verification_epoch.py + packages/microcosm-graph/tests/test_graph_view.py +uk: files=137 procs=build, frame shards_per_process=1 + packages/microcosm-build/tests/test_uk_age_tail.py + packages/microcosm-build/tests/test_uk_atomic_area_support.py + packages/microcosm-build/tests/test_uk_atomic_household_lineage.py + packages/microcosm-build/tests/test_uk_battery_bindings.py + packages/microcosm-build/tests/test_uk_brma_resource.py + packages/microcosm-build/tests/test_uk_bus_targets.py + packages/microcosm-build/tests/test_uk_bus_use_incidence.py + packages/microcosm-build/tests/test_uk_calibration_run.py + packages/microcosm-build/tests/test_uk_calibration_seam_driver.py + packages/microcosm-build/tests/test_uk_cgt_calibration.py + packages/microcosm-build/tests/test_uk_cgt_imputation.py + packages/microcosm-build/tests/test_uk_cgt_observation_period.py + packages/microcosm-build/tests/test_uk_cgt_source_manifest.py + packages/microcosm-build/tests/test_uk_cgt_structure.py + packages/microcosm-build/tests/test_uk_cgt_targets.py + packages/microcosm-build/tests/test_uk_childcare_targets.py + packages/microcosm-build/tests/test_uk_chronicle_feed.py + packages/microcosm-build/tests/test_uk_consumption_resources.py + packages/microcosm-build/tests/test_uk_content_identity.py + packages/microcosm-build/tests/test_uk_corrected_measurement_receipt.py + packages/microcosm-build/tests/test_uk_council_tax_targets.py + packages/microcosm-build/tests/test_uk_data_target_parity.py + packages/microcosm-build/tests/test_uk_dense_release_assembler.py + packages/microcosm-build/tests/test_uk_diagnostics.py + packages/microcosm-build/tests/test_uk_donor_uprating.py + packages/microcosm-build/tests/test_uk_e5_identity.py + packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py + packages/microcosm-build/tests/test_uk_energy_pricing.py + packages/microcosm-build/tests/test_uk_etb_services.py + packages/microcosm-build/tests/test_uk_etb_vat.py + packages/microcosm-build/tests/test_uk_evaluate_dataset_size_tool.py + packages/microcosm-build/tests/test_uk_fact_raking.py + packages/microcosm-build/tests/test_uk_firm_generation.py + packages/microcosm-build/tests/test_uk_firms_targets.py + packages/microcosm-build/tests/test_uk_frame_linkage_preflight.py + packages/microcosm-build/tests/test_uk_frs_council_tax.py + packages/microcosm-build/tests/test_uk_frs_disability.py + packages/microcosm-build/tests/test_uk_frs_education.py + packages/microcosm-build/tests/test_uk_frs_education_grants.py + packages/microcosm-build/tests/test_uk_frs_employment.py + packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py + packages/microcosm-build/tests/test_uk_frs_legacy_proxies.py + packages/microcosm-build/tests/test_uk_frs_relationships.py + packages/microcosm-build/tests/test_uk_frs_release.py + packages/microcosm-build/tests/test_uk_frs_spine.py + packages/microcosm-build/tests/test_uk_frs_stochastic.py + packages/microcosm-build/tests/test_uk_geography_ladder.py + packages/microcosm-build/tests/test_uk_geography_sources.py + packages/microcosm-build/tests/test_uk_graph.py + packages/microcosm-build/tests/test_uk_h5_payload_compare.py + packages/microcosm-build/tests/test_uk_hmrc_calibration.py + packages/microcosm-build/tests/test_uk_hmrc_capital_gains.py + packages/microcosm-build/tests/test_uk_hmrc_income.py + packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py + packages/microcosm-build/tests/test_uk_hmrc_replay.py + packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py + packages/microcosm-build/tests/test_uk_identity_stability_receipts.py + packages/microcosm-build/tests/test_uk_incumbent_surface_evaluation.py + packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py + packages/microcosm-build/tests/test_uk_ladder_sni_sources.py + packages/microcosm-build/tests/test_uk_ladder_targets.py + packages/microcosm-build/tests/test_uk_lcfs_consumption.py + packages/microcosm-build/tests/test_uk_ledger_fact_vendoring.py + packages/microcosm-build/tests/test_uk_ledger_targets.py + packages/microcosm-build/tests/test_uk_local_area_crosswalk.py + packages/microcosm-build/tests/test_uk_local_candidate_scorer.py + packages/microcosm-build/tests/test_uk_local_doctrine.py + packages/microcosm-build/tests/test_uk_local_gate_battery.py + packages/microcosm-build/tests/test_uk_local_geography.py + packages/microcosm-build/tests/test_uk_local_holdout.py + packages/microcosm-build/tests/test_uk_local_incumbent_extractor.py + packages/microcosm-build/tests/test_uk_local_release_preflight.py + packages/microcosm-build/tests/test_uk_local_rowwise.py + packages/microcosm-build/tests/test_uk_local_target_census.py + packages/microcosm-build/tests/test_uk_local_targets.py + packages/microcosm-build/tests/test_uk_local_validation_levels.py + packages/microcosm-build/tests/test_uk_measure_simulation.py + packages/microcosm-build/tests/test_uk_national_calibration.py + packages/microcosm-build/tests/test_uk_national_frame.py + packages/microcosm-build/tests/test_uk_national_sampling.py + packages/microcosm-build/tests/test_uk_nhs_allocation.py + packages/microcosm-build/tests/test_uk_ods_tables.py + packages/microcosm-build/tests/test_uk_parity_reference.py + packages/microcosm-build/tests/test_uk_payload_diff_classifier.py + packages/microcosm-build/tests/test_uk_pipr_deferrals.py + packages/microcosm-build/tests/test_uk_population_targets.py + packages/microcosm-build/tests/test_uk_provenance_flags.py + packages/microcosm-build/tests/test_uk_raking.py + packages/microcosm-build/tests/test_uk_regional_uprating.py + packages/microcosm-build/tests/test_uk_release_assembler.py + packages/microcosm-build/tests/test_uk_release_certification.py + packages/microcosm-build/tests/test_uk_release_identity.py + packages/microcosm-build/tests/test_uk_release_input_coverage.py + packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py + packages/microcosm-build/tests/test_uk_rowwise_build_driver.py + packages/microcosm-build/tests/test_uk_rowwise_candidate.py + packages/microcosm-build/tests/test_uk_rowwise_dataset.py + packages/microcosm-build/tests/test_uk_rowwise_dry_run.py + packages/microcosm-build/tests/test_uk_rowwise_geography.py + packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py + packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py + packages/microcosm-build/tests/test_uk_salary_sacrifice.py + packages/microcosm-build/tests/test_uk_signed_differences.py + packages/microcosm-build/tests/test_uk_size_evaluation.py + packages/microcosm-build/tests/test_uk_source_runtime.py + packages/microcosm-build/tests/test_uk_source_stages.py + packages/microcosm-build/tests/test_uk_spi_income.py + packages/microcosm-build/tests/test_uk_spi_spine.py + packages/microcosm-build/tests/test_uk_spi_support.py + packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py + packages/microcosm-build/tests/test_uk_spine_parity_instrument.py + packages/microcosm-build/tests/test_uk_stage_checkpoints.py + packages/microcosm-build/tests/test_uk_stage_health.py + packages/microcosm-build/tests/test_uk_staging_integration.py + packages/microcosm-build/tests/test_uk_staging_provisioning.py + packages/microcosm-build/tests/test_uk_stochastic_tools.py + packages/microcosm-build/tests/test_uk_student_loans.py + packages/microcosm-build/tests/test_uk_take_up_contract.py + packages/microcosm-build/tests/test_uk_take_up_gate.py + packages/microcosm-build/tests/test_uk_target_references.py + packages/microcosm-build/tests/test_uk_terminal_gates.py + packages/microcosm-build/tests/test_uk_tfc_routed_share.py + packages/microcosm-build/tests/test_uk_uc_capital_coherence.py + packages/microcosm-build/tests/test_uk_uc_deduction_attributes.py + packages/microcosm-build/tests/test_uk_uc_family.py + packages/microcosm-build/tests/test_uk_uc_lp_evidence.py + packages/microcosm-build/tests/test_uk_uc_matrix_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_relationships.py + packages/microcosm-build/tests/test_uk_uc_reporter_redraw.py + packages/microcosm-build/tests/test_uk_uc_source_periods.py + packages/microcosm-build/tests/test_uk_uc_support_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_target_measurements.py + packages/microcosm-build/tests/test_uk_was_wealth.py + packages/microcosm-build/tests/test_uk_wealth_resources.py + packages/microcosm-build/tests/test_uk_weighted_integrity.py + packages/microcosm-build/tests/test_uk_weighted_integrity_measurement.py + packages/microcosm-frame/tests/test_policyengine_uk_adapter.py +[wheels] +wheels: files=557 procs=main shards_per_process=8 + packages/microcosm-build/tests/test_atomic_geography.py + packages/microcosm-build/tests/test_chronicle_epoch.py + packages/microcosm-build/tests/test_ci_test_groups.py + packages/microcosm-build/tests/test_compare_us_fiscal_diagnostics.py + packages/microcosm-build/tests/test_country_spec.py + packages/microcosm-build/tests/test_cross_grain.py + packages/microcosm-build/tests/test_demographics.py + packages/microcosm-build/tests/test_diagnose_us_target_support.py + packages/microcosm-build/tests/test_frame_checkpoint.py + packages/microcosm-build/tests/test_frame_sampling.py + packages/microcosm-build/tests/test_frame_serializer_registry.py + packages/microcosm-build/tests/test_gate_battery.py + packages/microcosm-build/tests/test_gate_battery_contract_example.py + packages/microcosm-build/tests/test_gate_battery_contract_pins.py + packages/microcosm-build/tests/test_gates.py + packages/microcosm-build/tests/test_holdout.py + packages/microcosm-build/tests/test_imputation_lineage_spec.py + packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py + packages/microcosm-build/tests/test_ledger_artifact.py + packages/microcosm-build/tests/test_ledger_targets.py + packages/microcosm-build/tests/test_logbook.py + packages/microcosm-build/tests/test_logbook_adoption.py + packages/microcosm-build/tests/test_logbook_archive.py + packages/microcosm-build/tests/test_logbook_backfill.py + packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py + packages/microcosm-build/tests/test_logbook_cli.py + packages/microcosm-build/tests/test_logbook_env.py + packages/microcosm-build/tests/test_logbook_isolation.py + packages/microcosm-build/tests/test_logbook_prediction_seed.py + packages/microcosm-build/tests/test_monetary_profile.py + packages/microcosm-build/tests/test_monetary_targets.py + packages/microcosm-build/tests/test_outer_stage_runtime.py + packages/microcosm-build/tests/test_plan.py + packages/microcosm-build/tests/test_puf_qrf_chain.py + packages/microcosm-build/tests/test_reform_validation.py + packages/microcosm-build/tests/test_release_input_coverage.py + packages/microcosm-build/tests/test_release_target_parity.py + packages/microcosm-build/tests/test_score_uk_national_candidate.py + packages/microcosm-build/tests/test_source_runtime.py + packages/microcosm-build/tests/test_spec_engine_battery_semantics.py + packages/microcosm-build/tests/test_spec_engine_calibration_semantics.py + packages/microcosm-build/tests/test_spec_engine_compiler_ir.py + packages/microcosm-build/tests/test_spec_engine_country_bundles.py + packages/microcosm-build/tests/test_spec_engine_coverage_tool.py + packages/microcosm-build/tests/test_spec_engine_engine_abi.py + packages/microcosm-build/tests/test_spec_engine_field_usage.py + packages/microcosm-build/tests/test_spec_engine_identity_contracts.py + packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py + packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py + packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py + packages/microcosm-build/tests/test_spec_engine_loader.py + packages/microcosm-build/tests/test_spec_engine_plan_lock.py + packages/microcosm-build/tests/test_spec_engine_publication_semantics.py + packages/microcosm-build/tests/test_spec_engine_reemission.py + packages/microcosm-build/tests/test_spec_engine_resolver.py + packages/microcosm-build/tests/test_spec_engine_schema.py + packages/microcosm-build/tests/test_spec_engine_seeds.py + packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py + packages/microcosm-build/tests/test_spec_engine_take_up_semantics.py + packages/microcosm-build/tests/test_spec_engine_typed_closure.py + packages/microcosm-build/tests/test_spec_engine_yaml.py + packages/microcosm-build/tests/test_spec_only_country_packages.py + packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py + packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py + packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py + packages/microcosm-build/tests/test_spec_seed_identity_lock.py + packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py + packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py + packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py + packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py + packages/microcosm-build/tests/test_spec_worker_identity_lock.py + packages/microcosm-build/tests/test_stage_profile.py + packages/microcosm-build/tests/test_staging.py + packages/microcosm-build/tests/test_staging_v2.py + packages/microcosm-build/tests/test_stochastic_assignment.py + packages/microcosm-build/tests/test_target_materialization.py + packages/microcosm-build/tests/test_target_reference_authoring.py + packages/microcosm-build/tests/test_trace.py + packages/microcosm-build/tests/test_uk_age_tail.py + packages/microcosm-build/tests/test_uk_atomic_area_support.py + packages/microcosm-build/tests/test_uk_atomic_household_lineage.py + packages/microcosm-build/tests/test_uk_battery_bindings.py + packages/microcosm-build/tests/test_uk_brma_resource.py + packages/microcosm-build/tests/test_uk_bus_targets.py + packages/microcosm-build/tests/test_uk_bus_use_incidence.py + packages/microcosm-build/tests/test_uk_calibration_run.py + packages/microcosm-build/tests/test_uk_calibration_seam_driver.py + packages/microcosm-build/tests/test_uk_cgt_calibration.py + packages/microcosm-build/tests/test_uk_cgt_imputation.py + packages/microcosm-build/tests/test_uk_cgt_observation_period.py + packages/microcosm-build/tests/test_uk_cgt_source_manifest.py + packages/microcosm-build/tests/test_uk_cgt_structure.py + packages/microcosm-build/tests/test_uk_cgt_targets.py + packages/microcosm-build/tests/test_uk_childcare_targets.py + packages/microcosm-build/tests/test_uk_chronicle_feed.py + packages/microcosm-build/tests/test_uk_consumption_resources.py + packages/microcosm-build/tests/test_uk_content_identity.py + packages/microcosm-build/tests/test_uk_corrected_measurement_receipt.py + packages/microcosm-build/tests/test_uk_council_tax_targets.py + packages/microcosm-build/tests/test_uk_data_target_parity.py + packages/microcosm-build/tests/test_uk_dense_release_assembler.py + packages/microcosm-build/tests/test_uk_diagnostics.py + packages/microcosm-build/tests/test_uk_donor_uprating.py + packages/microcosm-build/tests/test_uk_e5_identity.py + packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py + packages/microcosm-build/tests/test_uk_energy_pricing.py + packages/microcosm-build/tests/test_uk_etb_services.py + packages/microcosm-build/tests/test_uk_etb_vat.py + packages/microcosm-build/tests/test_uk_evaluate_dataset_size_tool.py + packages/microcosm-build/tests/test_uk_fact_raking.py + packages/microcosm-build/tests/test_uk_firm_generation.py + packages/microcosm-build/tests/test_uk_firms_targets.py + packages/microcosm-build/tests/test_uk_frame_linkage_preflight.py + packages/microcosm-build/tests/test_uk_frs_council_tax.py + packages/microcosm-build/tests/test_uk_frs_disability.py + packages/microcosm-build/tests/test_uk_frs_education.py + packages/microcosm-build/tests/test_uk_frs_education_grants.py + packages/microcosm-build/tests/test_uk_frs_employment.py + packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py + packages/microcosm-build/tests/test_uk_frs_legacy_proxies.py + packages/microcosm-build/tests/test_uk_frs_relationships.py + packages/microcosm-build/tests/test_uk_frs_release.py + packages/microcosm-build/tests/test_uk_frs_spine.py + packages/microcosm-build/tests/test_uk_frs_stochastic.py + packages/microcosm-build/tests/test_uk_geography_ladder.py + packages/microcosm-build/tests/test_uk_geography_sources.py + packages/microcosm-build/tests/test_uk_graph.py + packages/microcosm-build/tests/test_uk_h5_payload_compare.py + packages/microcosm-build/tests/test_uk_hmrc_calibration.py + packages/microcosm-build/tests/test_uk_hmrc_capital_gains.py + packages/microcosm-build/tests/test_uk_hmrc_income.py + packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py + packages/microcosm-build/tests/test_uk_hmrc_replay.py + packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py + packages/microcosm-build/tests/test_uk_identity_stability_receipts.py + packages/microcosm-build/tests/test_uk_incumbent_surface_evaluation.py + packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py + packages/microcosm-build/tests/test_uk_ladder_sni_sources.py + packages/microcosm-build/tests/test_uk_ladder_targets.py + packages/microcosm-build/tests/test_uk_lcfs_consumption.py + packages/microcosm-build/tests/test_uk_ledger_fact_vendoring.py + packages/microcosm-build/tests/test_uk_ledger_targets.py + packages/microcosm-build/tests/test_uk_local_area_crosswalk.py + packages/microcosm-build/tests/test_uk_local_candidate_scorer.py + packages/microcosm-build/tests/test_uk_local_doctrine.py + packages/microcosm-build/tests/test_uk_local_gate_battery.py + packages/microcosm-build/tests/test_uk_local_geography.py + packages/microcosm-build/tests/test_uk_local_holdout.py + packages/microcosm-build/tests/test_uk_local_incumbent_extractor.py + packages/microcosm-build/tests/test_uk_local_release_preflight.py + packages/microcosm-build/tests/test_uk_local_rowwise.py + packages/microcosm-build/tests/test_uk_local_target_census.py + packages/microcosm-build/tests/test_uk_local_targets.py + packages/microcosm-build/tests/test_uk_local_validation_levels.py + packages/microcosm-build/tests/test_uk_measure_simulation.py + packages/microcosm-build/tests/test_uk_national_calibration.py + packages/microcosm-build/tests/test_uk_national_frame.py + packages/microcosm-build/tests/test_uk_national_sampling.py + packages/microcosm-build/tests/test_uk_nhs_allocation.py + packages/microcosm-build/tests/test_uk_ods_tables.py + packages/microcosm-build/tests/test_uk_parity_reference.py + packages/microcosm-build/tests/test_uk_payload_diff_classifier.py + packages/microcosm-build/tests/test_uk_pipr_deferrals.py + packages/microcosm-build/tests/test_uk_population_targets.py + packages/microcosm-build/tests/test_uk_provenance_flags.py + packages/microcosm-build/tests/test_uk_raking.py + packages/microcosm-build/tests/test_uk_regional_uprating.py + packages/microcosm-build/tests/test_uk_release_assembler.py + packages/microcosm-build/tests/test_uk_release_certification.py + packages/microcosm-build/tests/test_uk_release_identity.py + packages/microcosm-build/tests/test_uk_release_input_coverage.py + packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py + packages/microcosm-build/tests/test_uk_rowwise_build_driver.py + packages/microcosm-build/tests/test_uk_rowwise_candidate.py + packages/microcosm-build/tests/test_uk_rowwise_dataset.py + packages/microcosm-build/tests/test_uk_rowwise_dry_run.py + packages/microcosm-build/tests/test_uk_rowwise_geography.py + packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py + packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py + packages/microcosm-build/tests/test_uk_salary_sacrifice.py + packages/microcosm-build/tests/test_uk_signed_differences.py + packages/microcosm-build/tests/test_uk_size_evaluation.py + packages/microcosm-build/tests/test_uk_source_runtime.py + packages/microcosm-build/tests/test_uk_source_stages.py + packages/microcosm-build/tests/test_uk_spi_income.py + packages/microcosm-build/tests/test_uk_spi_spine.py + packages/microcosm-build/tests/test_uk_spi_support.py + packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py + packages/microcosm-build/tests/test_uk_spine_parity_instrument.py + packages/microcosm-build/tests/test_uk_stage_checkpoints.py + packages/microcosm-build/tests/test_uk_stage_health.py + packages/microcosm-build/tests/test_uk_staging_integration.py + packages/microcosm-build/tests/test_uk_staging_provisioning.py + packages/microcosm-build/tests/test_uk_stochastic_tools.py + packages/microcosm-build/tests/test_uk_student_loans.py + packages/microcosm-build/tests/test_uk_take_up_contract.py + packages/microcosm-build/tests/test_uk_take_up_gate.py + packages/microcosm-build/tests/test_uk_target_references.py + packages/microcosm-build/tests/test_uk_terminal_gates.py + packages/microcosm-build/tests/test_uk_tfc_routed_share.py + packages/microcosm-build/tests/test_uk_uc_capital_coherence.py + packages/microcosm-build/tests/test_uk_uc_deduction_attributes.py + packages/microcosm-build/tests/test_uk_uc_family.py + packages/microcosm-build/tests/test_uk_uc_lp_evidence.py + packages/microcosm-build/tests/test_uk_uc_matrix_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_relationships.py + packages/microcosm-build/tests/test_uk_uc_reporter_redraw.py + packages/microcosm-build/tests/test_uk_uc_source_periods.py + packages/microcosm-build/tests/test_uk_uc_support_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_target_measurements.py + packages/microcosm-build/tests/test_uk_was_wealth.py + packages/microcosm-build/tests/test_uk_wealth_resources.py + packages/microcosm-build/tests/test_uk_weighted_integrity.py + packages/microcosm-build/tests/test_uk_weighted_integrity_measurement.py + packages/microcosm-build/tests/test_us_acs_housing_source.py + packages/microcosm-build/tests/test_us_acs_inputs.py + packages/microcosm-build/tests/test_us_acs_local_release_tool.py + packages/microcosm-build/tests/test_us_acs_multispine.py + packages/microcosm-build/tests/test_us_acs_multispine_base_builder.py + packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py + packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py + packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py + packages/microcosm-build/tests/test_us_acs_population_catalogue.py + packages/microcosm-build/tests/test_us_acs_pums.py + packages/microcosm-build/tests/test_us_acs_record_fence_scan.py + packages/microcosm-build/tests/test_us_acs_release_predictors.py + packages/microcosm-build/tests/test_us_acs_source_compile_cache.py + packages/microcosm-build/tests/test_us_acs_sources.py + packages/microcosm-build/tests/test_us_acs_transfer.py + packages/microcosm-build/tests/test_us_adult_care.py + packages/microcosm-build/tests/test_us_alimony.py + packages/microcosm-build/tests/test_us_asec_2024_native_population.py + packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py + packages/microcosm-build/tests/test_us_asec_checkpoint.py + packages/microcosm-build/tests/test_us_asec_coverage_authentication.py + packages/microcosm-build/tests/test_us_asec_current_money_source.py + packages/microcosm-build/tests/test_us_asec_demographic_source.py + packages/microcosm-build/tests/test_us_asec_household_coverage_fields.py + packages/microcosm-build/tests/test_us_asec_person_income_source.py + packages/microcosm-build/tests/test_us_asec_pool.py + packages/microcosm-build/tests/test_us_asec_prepared_resources.py + packages/microcosm-build/tests/test_us_asec_source_domain_hardening.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_de.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_national.py + packages/microcosm-build/tests/test_us_atomic_block_sources.py + packages/microcosm-build/tests/test_us_atomic_block_support.py + packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py + packages/microcosm-build/tests/test_us_base_pool.py + packages/microcosm-build/tests/test_us_block_ladder_sources.py + packages/microcosm-build/tests/test_us_buildj_base_script.py + packages/microcosm-build/tests/test_us_bundle_core_contracts.py + packages/microcosm-build/tests/test_us_capital_gain_details.py + packages/microcosm-build/tests/test_us_capital_gain_distributions.py + packages/microcosm-build/tests/test_us_casualty_losses.py + packages/microcosm-build/tests/test_us_child_property_income_graph.py + packages/microcosm-build/tests/test_us_child_property_income_graph_owner.py + packages/microcosm-build/tests/test_us_child_property_income_source.py + packages/microcosm-build/tests/test_us_child_property_income_source_owner.py + packages/microcosm-build/tests/test_us_child_support.py + packages/microcosm-build/tests/test_us_childcare.py + packages/microcosm-build/tests/test_us_common_frame_export_contract.py + packages/microcosm-build/tests/test_us_completion_support_custody.py + packages/microcosm-build/tests/test_us_congressional_district_geography.py + packages/microcosm-build/tests/test_us_congressional_district_vintage.py + packages/microcosm-build/tests/test_us_congressional_district_vintage_crosswalk.py + packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py + packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py + packages/microcosm-build/tests/test_us_current_asec_child_support_source.py + packages/microcosm-build/tests/test_us_current_asec_demographics.py + packages/microcosm-build/tests/test_us_current_asec_dividend_source.py + packages/microcosm-build/tests/test_us_current_asec_income_routing.py + packages/microcosm-build/tests/test_us_current_asec_interest_source.py + packages/microcosm-build/tests/test_us_current_asec_property_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py + packages/microcosm-build/tests/test_us_current_property_income_sources.py + packages/microcosm-build/tests/test_us_current_survey_amounts.py + packages/microcosm-build/tests/test_us_current_survey_geography.py + packages/microcosm-build/tests/test_us_current_survey_health_coverage.py + packages/microcosm-build/tests/test_us_current_survey_household_roles.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_graph.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_source.py + packages/microcosm-build/tests/test_us_current_survey_housing.py + packages/microcosm-build/tests/test_us_current_survey_person_status.py + packages/microcosm-build/tests/test_us_current_survey_person_status_graph.py + packages/microcosm-build/tests/test_us_current_survey_person_status_source.py + packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py + packages/microcosm-build/tests/test_us_current_survey_predictors.py + packages/microcosm-build/tests/test_us_current_survey_property_graph.py + packages/microcosm-build/tests/test_us_current_survey_puf_host.py + packages/microcosm-build/tests/test_us_current_survey_puf_transfer.py + packages/microcosm-build/tests/test_us_dc_ptc_take_up_exclusion.py + packages/microcosm-build/tests/test_us_disability_benefits.py + packages/microcosm-build/tests/test_us_documented_absent_inputs.py + packages/microcosm-build/tests/test_us_domestic_production.py + packages/microcosm-build/tests/test_us_early_head_start_exclusion.py + packages/microcosm-build/tests/test_us_education_assistance_source.py + packages/microcosm-build/tests/test_us_education_inputs.py + packages/microcosm-build/tests/test_us_educator_expenses.py + packages/microcosm-build/tests/test_us_eligibility_inputs.py + packages/microcosm-build/tests/test_us_energy_subsidy.py + packages/microcosm-build/tests/test_us_exact_k_ladder.py + packages/microcosm-build/tests/test_us_exact_k_ladder_e2e.py + packages/microcosm-build/tests/test_us_exact_k_ladder_launcher.py + packages/microcosm-build/tests/test_us_farm_business_income.py + packages/microcosm-build/tests/test_us_financial_assistance_exclusion.py + packages/microcosm-build/tests/test_us_fiscal_leaf_policy.py + packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py + packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py + packages/microcosm-build/tests/test_us_fiscal_targets.py + packages/microcosm-build/tests/test_us_form_4952.py + packages/microcosm-build/tests/test_us_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_full_puf_output_profiles.py + packages/microcosm-build/tests/test_us_geography_integration.py + packages/microcosm-build/tests/test_us_geography_ladder.py + packages/microcosm-build/tests/test_us_graph.py + packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py + packages/microcosm-build/tests/test_us_graph_atomic_person_status.py + packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py + packages/microcosm-build/tests/test_us_graph_current_survey_geography.py + packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py + packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py + packages/microcosm-build/tests/test_us_graph_fiscal_target_snapshots.py + packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_selection.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_stream.py + packages/microcosm-build/tests/test_us_graph_property_income.py + packages/microcosm-build/tests/test_us_graph_property_income_receipts.py + packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py + packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py + packages/microcosm-build/tests/test_us_graph_survey_completion.py + packages/microcosm-build/tests/test_us_graph_survey_population.py + packages/microcosm-build/tests/test_us_graph_survey_puf55.py + packages/microcosm-build/tests/test_us_graph_us_survey_enrichment.py + packages/microcosm-build/tests/test_us_hours_worked.py + packages/microcosm-build/tests/test_us_housing_inputs.py + packages/microcosm-build/tests/test_us_housing_participation.py + packages/microcosm-build/tests/test_us_immigration.py + packages/microcosm-build/tests/test_us_input_mass.py + packages/microcosm-build/tests/test_us_l0_refit_export.py + packages/microcosm-build/tests/test_us_late_producer_dag.py + packages/microcosm-build/tests/test_us_medicaid_take_up.py + packages/microcosm-build/tests/test_us_medicare_take_up.py + packages/microcosm-build/tests/test_us_misc_itemized.py + packages/microcosm-build/tests/test_us_multispine_pool.py + packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py + packages/microcosm-build/tests/test_us_multispine_pool_tool.py + packages/microcosm-build/tests/test_us_multispine_puf_clone.py + packages/microcosm-build/tests/test_us_national_age_counts.py + packages/microcosm-build/tests/test_us_native_row_ceilings.py + packages/microcosm-build/tests/test_us_native_verify_once_epoch.py + packages/microcosm-build/tests/test_us_nonzero_shares.py + packages/microcosm-build/tests/test_us_org_wages.py + packages/microcosm-build/tests/test_us_other_health_insurance.py + packages/microcosm-build/tests/test_us_parity_reference.py + packages/microcosm-build/tests/test_us_plan.py + packages/microcosm-build/tests/test_us_pool_input_consumers.py + packages/microcosm-build/tests/test_us_population_input_coverage.py + packages/microcosm-build/tests/test_us_post_transfer_calibration.py + packages/microcosm-build/tests/test_us_postclone_geography_identity.py + packages/microcosm-build/tests/test_us_pregnancy.py + packages/microcosm-build/tests/test_us_prior_year_income.py + packages/microcosm-build/tests/test_us_property_completion_graph.py + packages/microcosm-build/tests/test_us_property_completion_routing.py + packages/microcosm-build/tests/test_us_puf55_heterogeneous_manifest_seals.py + packages/microcosm-build/tests/test_us_puf55_model_boundaries.py + packages/microcosm-build/tests/test_us_puf55_route_finalization.py + packages/microcosm-build/tests/test_us_puf55_route_numerical_finalization.py + packages/microcosm-build/tests/test_us_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_profile.py + packages/microcosm-build/tests/test_us_puf59_canonical.py + packages/microcosm-build/tests/test_us_puf_aggregate_records.py + packages/microcosm-build/tests/test_us_puf_capital_gains_tail.py + packages/microcosm-build/tests/test_us_puf_detail_transfer.py + packages/microcosm-build/tests/test_us_puf_donor_io.py + packages/microcosm-build/tests/test_us_puf_e01000_reconciliation.py + packages/microcosm-build/tests/test_us_puf_e19200_decomposition.py + packages/microcosm-build/tests/test_us_puf_price_baseline.py + packages/microcosm-build/tests/test_us_puf_support.py + packages/microcosm-build/tests/test_us_puf_support_base_builder.py + packages/microcosm-build/tests/test_us_puf_support_base_equivalence.py + packages/microcosm-build/tests/test_us_puf_tail_bound.py + packages/microcosm-build/tests/test_us_puf_target2024_growth.py + packages/microcosm-build/tests/test_us_puma_ladder.py + packages/microcosm-build/tests/test_us_puma_ladder_sources.py + packages/microcosm-build/tests/test_us_qbi_inputs.py + packages/microcosm-build/tests/test_us_relationship_inputs.py + packages/microcosm-build/tests/test_us_release_gate_preflight.py + packages/microcosm-build/tests/test_us_release_head_to_head_scorer.py + packages/microcosm-build/tests/test_us_reported_coverage_vintage_gate.py + packages/microcosm-build/tests/test_us_retirement_contributions.py + packages/microcosm-build/tests/test_us_retirement_distributions.py + packages/microcosm-build/tests/test_us_runtime_facade_union.py + packages/microcosm-build/tests/test_us_runtime_import_order.py + packages/microcosm-build/tests/test_us_salt_refund_income.py + packages/microcosm-build/tests/test_us_scf_auto_loans.py + packages/microcosm-build/tests/test_us_scf_wealth.py + packages/microcosm-build/tests/test_us_sipp_financial_assets.py + packages/microcosm-build/tests/test_us_sipp_head_start.py + packages/microcosm-build/tests/test_us_sipp_tips.py + packages/microcosm-build/tests/test_us_sipp_vehicles.py + packages/microcosm-build/tests/test_us_snap_discretionary_exemption.py + packages/microcosm-build/tests/test_us_snap_state_take_up.py + packages/microcosm-build/tests/test_us_snap_take_up.py + packages/microcosm-build/tests/test_us_source_coverage.py + packages/microcosm-build/tests/test_us_source_runtime.py + packages/microcosm-build/tests/test_us_spec_bundle.py + packages/microcosm-build/tests/test_us_spine_agreement.py + packages/microcosm-build/tests/test_us_spine_assembly.py + packages/microcosm-build/tests/test_us_spine_blindness.py + packages/microcosm-build/tests/test_us_spm_resources.py + packages/microcosm-build/tests/test_us_spm_role_enrichment_builder.py + packages/microcosm-build/tests/test_us_spm_role_source.py + packages/microcosm-build/tests/test_us_ssi_disability_criteria.py + packages/microcosm-build/tests/test_us_ssi_take_up.py + packages/microcosm-build/tests/test_us_stacked_spine.py + packages/microcosm-build/tests/test_us_state_files_scorer.py + packages/microcosm-build/tests/test_us_survey_age_activation.py + packages/microcosm-build/tests/test_us_survey_age_artifact.py + packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_age_calibration_run.py + packages/microcosm-build/tests/test_us_survey_age_development.py + packages/microcosm-build/tests/test_us_survey_age_sources.py + packages/microcosm-build/tests/test_us_survey_calibration.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py + packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py + packages/microcosm-build/tests/test_us_survey_financial_successor.py + packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py + packages/microcosm-build/tests/test_us_survey_observed_age.py + packages/microcosm-build/tests/test_us_survey_origin_budget.py + packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_origin_budget_detached_views.py + packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py + packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py + packages/microcosm-build/tests/test_us_survey_population_preparation.py + packages/microcosm-build/tests/test_us_survey_population_replay.py + packages/microcosm-build/tests/test_us_survey_social_security.py + packages/microcosm-build/tests/test_us_survivor_benefits_exclusion.py + packages/microcosm-build/tests/test_us_take_up.py + packages/microcosm-build/tests/test_us_take_up_contract.py + packages/microcosm-build/tests/test_us_trade_census_imports.py + packages/microcosm-build/tests/test_us_trade_crosscheck.py + packages/microcosm-build/tests/test_us_trade_entries_cli.py + packages/microcosm-build/tests/test_us_trade_entry_generator.py + packages/microcosm-build/tests/test_us_trade_facts.py + packages/microcosm-build/tests/test_us_trade_imdb_bulk.py + packages/microcosm-build/tests/test_us_trade_imdb_goldens.py + packages/microcosm-build/tests/test_us_voluntary_filing.py + packages/microcosm-build/tests/test_us_warm_start_selection.py + packages/microcosm-build/tests/test_us_weeks_unemployed.py + packages/microcosm-build/tests/test_us_whole_series_selection_sites.py + packages/microcosm-build/tests/test_us_wic_claim.py + packages/microcosm-build/tests/test_us_wic_nutritional_risk_exclusion.py + packages/microcosm-build/tests/test_us_workers_compensation.py + packages/microcosm-build/tests/test_validation_input_coverage.py + packages/microcosm-calibrate/tests/test_compat.py + packages/microcosm-calibrate/tests/test_diagnostics.py + packages/microcosm-calibrate/tests/test_exact_k.py + packages/microcosm-calibrate/tests/test_group_bounds.py + packages/microcosm-calibrate/tests/test_grouped_fixed_support.py + packages/microcosm-calibrate/tests/test_hierarchy.py + packages/microcosm-calibrate/tests/test_informed_gates.py + packages/microcosm-calibrate/tests/test_kernels.py + packages/microcosm-calibrate/tests/test_matrix.py + packages/microcosm-calibrate/tests/test_monetary_binding.py + packages/microcosm-calibrate/tests/test_provider_labels.py + packages/microcosm-calibrate/tests/test_registry.py + packages/microcosm-calibrate/tests/test_score.py + packages/microcosm-calibrate/tests/test_solve.py + packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py + packages/microcosm-calibrate/tests/test_sparse_solve.py + packages/microcosm-calibrate/tests/test_target.py + packages/microcosm-calibrate/tests/test_target_snapshots.py + packages/microcosm-calibrate/tests/test_variable_labels.py + packages/microcosm-data/tests/test_compat.py + packages/microcosm-data/tests/test_contract.py + packages/microcosm-data/tests/test_h5_enrichment.py + packages/microcosm-data/tests/test_loader.py + packages/microcosm-data/tests/test_local_area_contract.py + packages/microcosm-data/tests/test_publish_guard.py + packages/microcosm-data/tests/test_registry.py + packages/microcosm-data/tests/test_release.py + packages/microcosm-data/tests/test_slack.py + packages/microcosm-data/tests/test_source_enrichment.py + packages/microcosm-data/tests/test_uk_dense_release_contract.py + packages/microcosm-fit/tests/test_cache.py + packages/microcosm-fit/tests/test_compat.py + packages/microcosm-fit/tests/test_dataframe_fit.py + packages/microcosm-fit/tests/test_graph_joint_empirical.py + packages/microcosm-fit/tests/test_graph_legacy_matrix.py + packages/microcosm-fit/tests/test_graph_legacy_qrf.py + packages/microcosm-fit/tests/test_graph_signed_reconciliation.py + packages/microcosm-fit/tests/test_joint_empirical.py + packages/microcosm-fit/tests/test_kernels.py + packages/microcosm-fit/tests/test_model.py + packages/microcosm-fit/tests/test_qrf.py + packages/microcosm-fit/tests/test_qrf_stateless.py + packages/microcosm-fit/tests/test_qrf_target.py + packages/microcosm-fit/tests/test_signed_reconciliation.py + packages/microcosm-fit/tests/test_weighted_fit_contract.py + packages/microcosm-frame/tests/test_accounting.py + packages/microcosm-frame/tests/test_adapters.py + packages/microcosm-frame/tests/test_axiom_adapter.py + packages/microcosm-frame/tests/test_bundle.py + packages/microcosm-frame/tests/test_contracts.py + packages/microcosm-frame/tests/test_kernels.py + packages/microcosm-frame/tests/test_materialize.py + packages/microcosm-frame/tests/test_place.py + packages/microcosm-frame/tests/test_policyengine_uk_adapter.py + packages/microcosm-frame/tests/test_policyengine_us_adapter.py + packages/microcosm-frame/tests/test_policyengine_us_consumer_index.py + packages/microcosm-frame/tests/test_policyengine_us_metadata_index.py + packages/microcosm-frame/tests/test_policyengine_us_ownership_index.py + packages/microcosm-frame/tests/test_rules_engine_contract.py + packages/microcosm-frame/tests/test_units.py + packages/microcosm-frame/tests/test_weights_schema.py + packages/microcosm-graph/tests/test_acceptance_a_identity.py + packages/microcosm-graph/tests/test_acceptance_b_ownership.py + packages/microcosm-graph/tests/test_acceptance_c_seeds.py + packages/microcosm-graph/tests/test_acceptance_d_weights.py + packages/microcosm-graph/tests/test_acceptance_e_store.py + packages/microcosm-graph/tests/test_acceptance_f_gates.py + packages/microcosm-graph/tests/test_acceptance_g_legibility.py + packages/microcosm-graph/tests/test_acceptance_h_parity.py + packages/microcosm-graph/tests/test_acceptance_replays.py + packages/microcosm-graph/tests/test_frame_metadata_store.py + packages/microcosm-graph/tests/test_graph_acceptance_burndown.py + packages/microcosm-graph/tests/test_graph_canonical.py + packages/microcosm-graph/tests/test_graph_codecs.py + packages/microcosm-graph/tests/test_graph_decl.py + packages/microcosm-graph/tests/test_graph_executor.py + packages/microcosm-graph/tests/test_graph_executor_scalar_float.py + packages/microcosm-graph/tests/test_graph_executor_series_stream.py + packages/microcosm-graph/tests/test_graph_executor_source_identity.py + packages/microcosm-graph/tests/test_graph_explain.py + packages/microcosm-graph/tests/test_graph_interface_lock.py + packages/microcosm-graph/tests/test_graph_kernel_contract.py + packages/microcosm-graph/tests/test_graph_keys.py + packages/microcosm-graph/tests/test_graph_manifest.py + packages/microcosm-graph/tests/test_graph_parity_pins.py + packages/microcosm-graph/tests/test_graph_parity_repin.py + packages/microcosm-graph/tests/test_graph_population.py + packages/microcosm-graph/tests/test_graph_randomness.py + packages/microcosm-graph/tests/test_graph_serialize.py + packages/microcosm-graph/tests/test_graph_storage_parts_whole_series.py + packages/microcosm-graph/tests/test_graph_store.py + packages/microcosm-graph/tests/test_graph_verification_epoch.py + packages/microcosm-graph/tests/test_graph_view.py +[engine-only] excluded from the fast tier, run in the engine tier + packages/microcosm-build/tests/test_us_multispine_pool_tool.py -> us-am + packages/microcosm-build/tests/test_us_release_head_to_head_scorer.py -> us-qs +[defaulted] + packages/microcosm-build/tests/test_chronicle_epoch.py -> shared-spec + packages/microcosm-build/tests/test_compare_us_fiscal_diagnostics.py -> shared-spec + packages/microcosm-build/tests/test_country_spec.py -> shared-spec + packages/microcosm-build/tests/test_demographics.py -> shared-spec + packages/microcosm-build/tests/test_diagnose_us_target_support.py -> shared-spec + packages/microcosm-build/tests/test_frame_checkpoint.py -> shared-spec + packages/microcosm-build/tests/test_frame_sampling.py -> shared-spec + packages/microcosm-build/tests/test_frame_serializer_registry.py -> shared-spec + packages/microcosm-build/tests/test_gate_battery.py -> shared-spec + packages/microcosm-build/tests/test_gate_battery_contract_example.py -> shared-spec + packages/microcosm-build/tests/test_gate_battery_contract_pins.py -> shared-spec + packages/microcosm-build/tests/test_gates.py -> shared-spec + packages/microcosm-build/tests/test_holdout.py -> shared-spec + packages/microcosm-build/tests/test_imputation_lineage_spec.py -> shared-spec + packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py -> shared-spec + packages/microcosm-build/tests/test_ledger_artifact.py -> shared-spec + packages/microcosm-build/tests/test_ledger_targets.py -> shared-spec + packages/microcosm-build/tests/test_logbook.py -> shared-spec + packages/microcosm-build/tests/test_logbook_adoption.py -> shared-spec + packages/microcosm-build/tests/test_logbook_archive.py -> shared-spec + packages/microcosm-build/tests/test_logbook_backfill.py -> shared-spec + packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py -> shared-spec + packages/microcosm-build/tests/test_logbook_cli.py -> shared-spec + packages/microcosm-build/tests/test_logbook_env.py -> shared-spec + packages/microcosm-build/tests/test_logbook_isolation.py -> shared-spec + packages/microcosm-build/tests/test_logbook_prediction_seed.py -> shared-spec + packages/microcosm-build/tests/test_monetary_profile.py -> shared-spec + packages/microcosm-build/tests/test_monetary_targets.py -> shared-spec + packages/microcosm-build/tests/test_outer_stage_runtime.py -> shared-spec + packages/microcosm-build/tests/test_plan.py -> shared-spec + packages/microcosm-build/tests/test_reform_validation.py -> shared-spec + packages/microcosm-build/tests/test_release_input_coverage.py -> shared-spec + packages/microcosm-build/tests/test_release_target_parity.py -> shared-spec + packages/microcosm-build/tests/test_score_uk_national_candidate.py -> shared-spec + packages/microcosm-build/tests/test_source_runtime.py -> shared-spec + packages/microcosm-build/tests/test_stage_profile.py -> shared-spec + packages/microcosm-build/tests/test_staging.py -> shared-spec + packages/microcosm-build/tests/test_staging_v2.py -> shared-spec + packages/microcosm-build/tests/test_stochastic_assignment.py -> shared-spec + packages/microcosm-build/tests/test_target_materialization.py -> shared-spec + packages/microcosm-build/tests/test_target_reference_authoring.py -> shared-spec + packages/microcosm-build/tests/test_trace.py -> shared-spec + packages/microcosm-build/tests/test_validation_input_coverage.py -> shared-spec + packages/microcosm-frame/tests/test_accounting.py -> shared-spec + packages/microcosm-frame/tests/test_adapters.py -> shared-spec + packages/microcosm-frame/tests/test_axiom_adapter.py -> shared-spec + packages/microcosm-frame/tests/test_bundle.py -> shared-spec + packages/microcosm-frame/tests/test_contracts.py -> shared-spec + packages/microcosm-frame/tests/test_materialize.py -> shared-spec + packages/microcosm-frame/tests/test_place.py -> shared-spec + packages/microcosm-frame/tests/test_units.py -> shared-spec + packages/microcosm-frame/tests/test_weights_schema.py -> shared-spec +verification=ok diff --git a/experiments/acs-serial-membership/original.json b/experiments/acs-serial-membership/original.json new file mode 100644 index 000000000..e79b477f5 --- /dev/null +++ b/experiments/acs-serial-membership/original.json @@ -0,0 +1,106 @@ +{ + "all_masks_match_pandas": true, + "benchmark_sha256": "5d50e685ea4ebeb2769bba6cd434eece17e126b436383726c1a6ae62d3beff86", + "chunks": 20, + "first_lookup": { + "cpu_seconds": 0.33849600000000013, + "wall_seconds": 0.339910457842052 + }, + "index_deep_bytes_including_shared_strings": null, + "keys": 200000, + "mode": "original", + "module_sha256": "893fd570cf74bf87c133d98e2a450793ad031ce60a40778fb5ea03b3cb9280c8", + "pandas": "3.0.3", + "preparation": { + "cpu_seconds": 2.0000000000575113e-06, + "wall_seconds": 3.0838418751955032e-06 + }, + "prepared_index_engine_reused": false, + "pyarrow": "25.0.0", + "python": "3.14.4 (main, Apr 14 2026, 14:46:33) [Clang 22.1.3 ]", + "rows_per_chunk": 20000, + "rss_after_lookups_bytes": 409485312, + "rss_before_lookup_bytes": 407896064, + "scope": "invented_arrays_only; not native preparation performance", + "total_cpu_seconds": 6.691459000000003, + "total_wall_seconds": 6.729966166429222, + "warm_lookups": [ + { + "cpu_seconds": 0.3393189999999999, + "wall_seconds": 0.34137666691094637 + }, + { + "cpu_seconds": 0.33743600000000007, + "wall_seconds": 0.3397197499871254 + }, + { + "cpu_seconds": 0.3358890000000003, + "wall_seconds": 0.3384829161223024 + }, + { + "cpu_seconds": 0.3301660000000002, + "wall_seconds": 0.33123829215765 + }, + { + "cpu_seconds": 0.3281459999999998, + "wall_seconds": 0.3287552078254521 + }, + { + "cpu_seconds": 0.328306, + "wall_seconds": 0.3294437499716878 + }, + { + "cpu_seconds": 0.32299299999999986, + "wall_seconds": 0.3233217499218881 + }, + { + "cpu_seconds": 0.3580370000000004, + "wall_seconds": 0.37026783311739564 + }, + { + "cpu_seconds": 0.3380220000000005, + "wall_seconds": 0.33984450018033385 + }, + { + "cpu_seconds": 0.33664000000000005, + "wall_seconds": 0.3371821250766516 + }, + { + "cpu_seconds": 0.3283490000000002, + "wall_seconds": 0.3295408331323415 + }, + { + "cpu_seconds": 0.335947, + "wall_seconds": 0.33823804184794426 + }, + { + "cpu_seconds": 0.3306930000000001, + "wall_seconds": 0.332630000077188 + }, + { + "cpu_seconds": 0.3262930000000006, + "wall_seconds": 0.3279706249013543 + }, + { + "cpu_seconds": 0.3291460000000006, + "wall_seconds": 0.3301870001014322 + }, + { + "cpu_seconds": 0.3377799999999995, + "wall_seconds": 0.33863633312284946 + }, + { + "cpu_seconds": 0.34295900000000046, + "wall_seconds": 0.34419145807623863 + }, + { + "cpu_seconds": 0.3380799999999997, + "wall_seconds": 0.339578292099759 + }, + { + "cpu_seconds": 0.32875999999999994, + "wall_seconds": 0.32944725011475384 + } + ], + "whole_process_peak_rss_bytes_macos": 428949504 +} diff --git a/experiments/acs-serial-membership/prepared.json b/experiments/acs-serial-membership/prepared.json new file mode 100644 index 000000000..ad46e2fec --- /dev/null +++ b/experiments/acs-serial-membership/prepared.json @@ -0,0 +1,106 @@ +{ + "all_masks_match_pandas": true, + "benchmark_sha256": "5d50e685ea4ebeb2769bba6cd434eece17e126b436383726c1a6ae62d3beff86", + "chunks": 20, + "first_lookup": { + "cpu_seconds": 0.012153999999999998, + "wall_seconds": 0.013858875026926398 + }, + "index_deep_bytes_including_shared_strings": 17627112, + "keys": 200000, + "mode": "prepared", + "module_sha256": "893fd570cf74bf87c133d98e2a450793ad031ce60a40778fb5ea03b3cb9280c8", + "pandas": "3.0.3", + "preparation": { + "cpu_seconds": 0.011846000000000023, + "wall_seconds": 0.012051749974489212 + }, + "prepared_index_engine_reused": true, + "pyarrow": "25.0.0", + "python": "3.14.4 (main, Apr 14 2026, 14:46:33) [Clang 22.1.3 ]", + "rows_per_chunk": 20000, + "rss_after_lookups_bytes": 407994368, + "rss_before_lookup_bytes": 407060480, + "scope": "invented_arrays_only; not native preparation performance", + "total_cpu_seconds": 0.05246100000000009, + "total_wall_seconds": 0.05446754000149667, + "warm_lookups": [ + { + "cpu_seconds": 0.0018370000000000886, + "wall_seconds": 0.0018601249903440475 + }, + { + "cpu_seconds": 0.0016259999999999053, + "wall_seconds": 0.0016246249433606863 + }, + { + "cpu_seconds": 0.0015140000000000153, + "wall_seconds": 0.001539166085422039 + }, + { + "cpu_seconds": 0.001682999999999879, + "wall_seconds": 0.0016859162133187056 + }, + { + "cpu_seconds": 0.001527000000000056, + "wall_seconds": 0.0015269580762833357 + }, + { + "cpu_seconds": 0.0015130000000000976, + "wall_seconds": 0.0015121668111532927 + }, + { + "cpu_seconds": 0.00143099999999996, + "wall_seconds": 0.001430625095963478 + }, + { + "cpu_seconds": 0.0013640000000001429, + "wall_seconds": 0.0013634159695357084 + }, + { + "cpu_seconds": 0.0016320000000000778, + "wall_seconds": 0.001649708952754736 + }, + { + "cpu_seconds": 0.001562999999999981, + "wall_seconds": 0.001562874997034669 + }, + { + "cpu_seconds": 0.0015339999999999243, + "wall_seconds": 0.0015337499789893627 + }, + { + "cpu_seconds": 0.0013920000000000599, + "wall_seconds": 0.001392250182107091 + }, + { + "cpu_seconds": 0.0013369999999999216, + "wall_seconds": 0.0013375419657677412 + }, + { + "cpu_seconds": 0.001346999999999987, + "wall_seconds": 0.001363749848678708 + }, + { + "cpu_seconds": 0.0015739999999999643, + "wall_seconds": 0.0015869159251451492 + }, + { + "cpu_seconds": 0.0015049999999998676, + "wall_seconds": 0.0015045409090816975 + }, + { + "cpu_seconds": 0.0014180000000001414, + "wall_seconds": 0.0014184580650180578 + }, + { + "cpu_seconds": 0.0013360000000000039, + "wall_seconds": 0.001335417153313756 + }, + { + "cpu_seconds": 0.0013279999999999959, + "wall_seconds": 0.0013287088368088007 + } + ], + "whole_process_peak_rss_bytes_macos": 420200448 +} diff --git a/experiments/acs-serial-membership/tests-with-owner.txt b/experiments/acs-serial-membership/tests-with-owner.txt new file mode 100644 index 000000000..ea71cf6f3 --- /dev/null +++ b/experiments/acs-serial-membership/tests-with-owner.txt @@ -0,0 +1,4 @@ +........................................................................ [ 38%] +........................................................................ [ 77%] +......................................... [100%] +185 passed in 8.02s diff --git a/experiments/acs-serial-membership/tests.txt b/experiments/acs-serial-membership/tests.txt new file mode 100644 index 000000000..94a80a0a4 --- /dev/null +++ b/experiments/acs-serial-membership/tests.txt @@ -0,0 +1,2 @@ +............................................................... [100%] +63 passed in 0.70s diff --git a/experiments/asec-usual-hours-observations-20260919.json b/experiments/asec-usual-hours-observations-20260919.json new file mode 100644 index 000000000..ef3a0724e --- /dev/null +++ b/experiments/asec-usual-hours-observations-20260919.json @@ -0,0 +1,84 @@ +{ + "scope": "Pure ASEC all-age observations and invented source-owner regressions; no actual data or all-person output", + "base": "41fe226d82a2b41fe911f8e04b4dc1d503d14af3", + "source": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_usual_hours.py": "9660045cd2df2d9a69157e47f626eb77e6ebcfa98a8bb12e66e320307e1476cd", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_hours.py": "bfc79e2c580fa24c0e8afac6f5cfacd643d2cdf7fcbf1d2ffb960eaa5e26ca89", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_hours_source.py": "953c61db93d1c5b85755dc84287a8d5535571ac265f968a1d5c451592eb06b74", + "packages/microcosm-build/tests/test_us_current_asec_usual_hours.py": "4c33fff8f50177a5f5e8755a8a6f60c4c1c793f1de00c34d6d356f7bf725f050" + }, + "tests": { + "all_age_observations": 37, + "source_owner": 24, + "existing_pure_recoder": 54, + "failed": 0, + "skipped": 0 + }, + "result_sha256": "f1e9ac23f774c56d958c3f23bd83b3680d46b6152a21567dad4e20fb4479f4a8", + "xml_sha256": "0de4b0f841c4a18cce845044f64cf51b6ac61208345f6b174c9bd711371202f1", + "wall_seconds": 149.05698354099877, + "cpu_seconds": 115.30500799999999, + "peak_rss_bytes": 494436352, + "primary_dictionary": { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "visually_checked_pdf_pages": [ + 38, + 41 + ] + }, + "primary_api_metadata": [ + { + "name": "HRSWK", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/HRSWK.json", + "sha256": "adf6a9e2034b18d7216eff32bfbf0df6938e76b2032fb3d3d3adb2e27af4d56f" + }, + { + "name": "WKSWORK", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/WKSWORK.json", + "sha256": "4bcff6dbc5569c1341857cebae96cd8acd5bc3ae6fd875448a99b510eb06802a" + }, + { + "name": "WORKYN", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/WORKYN.json", + "sha256": "eba9950f0b21c96ffa4cf1526e5429e5daecbd91000c10f1751b3f9ca7949fb5" + }, + { + "name": "WTEMP", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/WTEMP.json", + "sha256": "48451e221582750fa8b4031d5d8a653ccfeaaccba82345ffdeb55054de3cae18" + }, + { + "name": "WRK_CK", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/WRK_CK.json", + "sha256": "09c41254643089d6bbb776d4332574e709c4215f9a76c6af353d1890de2f0dc5" + }, + { + "name": "A_AGE", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/A_AGE.json", + "sha256": "4b2222c321d110f6b4e7536046ec02bc5727772cb84034af654ee9f14e87a98a" + }, + { + "name": "WSAL_VAL", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/WSAL_VAL.json", + "sha256": "d4cd19311d821b481da5707ea93d9ed03df409e3f2d6288438b509109d9428a7" + }, + { + "name": "SEMP_VAL", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/SEMP_VAL.json", + "sha256": "2f2c969321d776622d807a52550cfe929f33b2dd86392ee4464459efea1861cd" + }, + { + "name": "FRSE_VAL", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/FRSE_VAL.json", + "sha256": "f8d8b57de8e4b55eb2997153622977f5ec6416d716b520a704ff9f5616e97451" + } + ], + "review": "Root source review; no independent peer after quota limitation", + "remaining": [ + "Actual all-age source audit", + "Explicit under15 modeled completion with earnings conflict checks", + "Source owner/graph/clone integration", + "Release no-worse comparison" + ] +} diff --git a/experiments/child-property-empirical-candidate-20260913.md b/experiments/child-property-empirical-candidate-20260913.md new file mode 100644 index 000000000..1e9d8f58a --- /dev/null +++ b/experiments/child-property-empirical-candidate-20260913.md @@ -0,0 +1,23 @@ +# Child property-income empirical candidate + +This source-only candidate implements an explicit transfer from original ASEC persons aged15–17 to explicitly unmeasured children aged0–14. It is opt-in methodology under review, not a released default or a measured child-income distribution. No native fit or test execution has occurred in this commit. + +`microcosm.fit.joint_empirical` fits a pooled weighted joint law for two nonnegative targets. It classifies each donor into neither, first-only, second-only or both-positive, then draws one paired observation within the chosen pattern. It uses Microcosm's typed/explicit weight resolvers. It does not run QRF or independently sample marginal amounts. + +For observed weighted pattern probabilities p, receipt factor r produces q_g=r*p_g for positive patterns and q_00=1-r*sum(p_positive). It refuses r<0, nonfinite values or r*sum(p_positive)>1; no probability is clipped. Two strictly positive amount factors scale the paired donor coordinates. Source-positive amounts lost to underflow or overflow refuse. Reference factors are one. r=0 is a separately labeled all-zero stress, still requiring the same fitted support-qualified donor population. + +Cumulative positive weights are normalized by their own finite positive endpoint. This declared normalization makes the final CDF endpoint one; non-increasing positive intervals refuse. Uniform coordinates use half-open intervals. The numerical operator accepts supplied uniforms; the separately owned graph adapter uses the maintained typed `keyed_uniform`, with exact original source coordinates rather than row positions, clone IDs or graph keys. Model JSON preserves integer coordinate types with decimal-string tags, including IDs above2**53. Descriptive model bytes cannot issue source authority. + +Reference transport retains the fitted zero-pattern mass directly rather than reconstructing it by subtracting rounded positive probabilities from one. This preserves representable leading intervals such as a zero donor with weight `2**-54` beside a positive donor with weight one. Every present positive-mass pattern, including the zero-income pattern when not named as required, must meet the declared pattern minima. Binary64 is the numerical input contract: floating dtypes wider than 64 bits refuse before target, explicit weight or uniform conversion. Where NumPy's `longdouble` aliases binary64, it remains supported; tests do not claim to exercise an unavailable extended-exponent representation. Already constructed Frame weights retain the Frame's own binary64 input contract. + +Support policy is mandatory and has no native default. It records positive-mass row/household counts, person and household effective sizes, separate overall/positive/pattern minima, and required patterns. Zero-weight observations stay visible in raw diagnostics but are never sampled or counted as positive support. Absent nonrequired patterns retain empirical zero and an explicit absence diagnostic; observed rare patterns cannot bypass the declared minima. Invented tests use named test policies. Native adequacy thresholds and sensitivity criteria require separate review. + +The US source qualifier calls the existing actual preparation and complete ASEC catalogue owners. Existing interest/dividend qualifiers already authenticate all current-year member rows before projecting selected `.person` tables; their full `.asec_literals` supply the donor pool. Exact full roster joins retain original household weights using `Fraction(HSUP_WGT,100)`, with no sample or clone multiplier. Ordinary interest is `TRDINT_VAL`, not combined `INT_VAL`; dividends are `DIV_VAL`. Missing retirement/rental information does not disqualify a known pair. Allocation/edit/topcode descriptors, including the existing I_DIVYN codebook ambiguity, remain descriptive provenance. + +The recipient path admits only jointly unknown, explicitly unmeasured under15 O/D observations. Source errors take precedence. It refuses contradictory or positive out-of-universe ASEC values, numeric out-of-universe ACS INTP, invalid ACS adjustments hidden by blank-status precedence, and unsupported known/partially-known incumbents. Source amounts and knownness are never rewritten by this module. Future model zeros must be labeled imputed, not observed nonreceipt. + +The qualifier extends maintained source identity and live function/default/closure fences around initial and final actual owner borrows. Its returned tables remain descriptive: downstream owners must retain and requalify original source owners, verify model/scenario artifacts and independently reconstruct the complete receiving population. A copied receipt or DataFrame cannot grant that authority. The new source tables are bounded to600,000 full source rows; the64MiB encoding ceiling applies to encoded metadata/individual coordinate data here, not a demonstrated resident-memory bound. The existing parent and future runtime guards retain their independent limits. + +The interpretation seal includes the routing domain, receipt, account and allocation constants used by the literal projectors. The unpublished-allocation set is represented by its type and sorted immutable values, so changing its contents cannot silently change provenance. Authored actual-owner controls preserve an allocation flag on the excluded teenager, compare raw original catalogue weights with projected weights, refuse malformed or contradictory excluded-donor literals pinned before issuance, and mutate detached projections during final catalogue/preparation callbacks. These controls remain unrun pending guarded acceptance. + +This commit changes no receiving graph, canonical columns, source-knownness, tax rules, PUF allocator, calibration, dataset, paper/site deployment or release gate. The graph integration lane owns separate private model-support CREATE branches with exact schema Slices; it will leave the receiving survey spine, clone expansion and geography unchanged. Whole-Frame reconstruction, composed cold/required replay, child-containing PUF admission, native donor diagnostics, household holdouts and scenario sensitivity remain pending. diff --git a/experiments/fiscal-target-snapshot-host-20260912.md b/experiments/fiscal-target-snapshot-host-20260912.md new file mode 100644 index 000000000..824fe94e2 --- /dev/null +++ b/experiments/fiscal-target-snapshot-host-20260912.md @@ -0,0 +1,113 @@ +# Target snapshots through the US fiscal graph + +The actual `FiscalDenseCalibrationKernel` now accepts an optional host-owned +`TargetSnapshotObserver` and forwards it to its existing grouped `calibrate` +call. Default construction and explicit `None` preserve the unobserved path. +Observer configuration stays on the kernel instance; it is absent from graph +parameters, implementation identity inputs, cache keys, artifacts and receipts. +The [usage documentation](../docs/calibration-target-snapshots.md#us-fiscal-host) +shows how a caller supplies the sink. + +This integration combines grouped core `2f62bb8448010bcc9b9e1419ea24aafcb85917d1` +with PR #914 head `16fc6cb501ccb58ceb158e9064bf4f39ee23479c`, including reviewed +repair `fbb109987e30f09ccc87fc402c47190d7c3ecfd2`. The grouped loop now reports +completed optimizer updates, starting at zero, while cadence continues to use +the one-based evaluation ordinal. Its selected snapshot remains the final +closing state at `epoch == epochs`, with no retained best. Ordinary Adam, +proximal and winning budget-probe labels preserve their reviewed repairs. +No numeric update, accepted projection, optimizer selection or RNG operation +was changed by the added fiscal observer boundary. + +## Integrated execution evidence + +The full calibration test directory, existing dense fiscal tests and nine new +fiscal snapshot controls completed with **473 passed, one skipped and no +failures or errors**. The run took 12.051 seconds wall time, 12.375 CPU seconds +and 491,569,152 bytes peak RSS. All 478 Python source files under shard source +directories retained their hashes; Torch threads were 1/1. + +The one skip is the saved prepatch exact-byte fixture. It requires its recorded +Torch interop setting of 18, while this bounded run used one. The current +on/off numerical comparisons and all new fiscal controls executed; the skipped +historical comparison is not claimed as a pass. The earlier grouped lane's +443-test evidence remains separate from this integrated run. + +The new controls exercise the real solver and tiny invented fiscal graph: + +- Default, explicit `None`, and observers with different labels/cadences produce + exactly equal weights, graph artifacts and receipts. Kernel implementation + hashes and declared node identities are equal across observer configurations. +- Snapshots use the actual measurement's ordered target names and values. The + emitted in-loop estimates equal the existing float32 loss tensors, and final + estimates equal the final float64 diagnostics. A six-update run at cadence + three emits completed-update labels 0, 2, 5 and selected 6. +- Every delivered dictionary is detached. A sink can damage its own aggregate + metadata and target rows without changing later snapshots or returned results. +- Exceptions from current and selected sinks propagate. Selected sinks that + corrupt live measurement values, household IDs or accepted weights are + refused by the existing final checks after callback completion. +- Actual graph executions with observation off/on have identical node keys, + weights and artifact bytes. Required replay with a fresh failing observer and + forbidden optimizer hook hits every cached node and invokes neither hook. + A cache replay therefore generates no new optimizer snapshots. + +The first attempt is preserved as failed test evidence. Its final-weight +corruption control tried normal assignment to an already read-only array and +stopped before reaching the intended final guard. The corrected test explicitly +injects a corrupted live weight container; final acceptance then refuses it. +This was a test-only correction; production solver and host bytes were unchanged. + +| Local evidence | SHA256 | +| --- | --- | +| Integrated execution receipt | `b4fb9b48a1d1ca4739e6d22d78968eae14a4855789548c9805068b7ffac2d38f` | +| Integrated JUnit result | `68b110709ade099efc3084af2e0adae0944d6a251172bce8496a3c265e20ed00` | +| Fiscal host source | `c26a9484b596015b4ffaf6a6a1e68e45f0ab4e81e63b5b612149777b100d56b9` | +| New fiscal snapshot test source | `f60d71aa9a4ac6bcf897dd937fe787eb21b0866cdd04223dba513af684849fcb` | +| Reconciled solver source | `8f22f325ed06cde31c4c25467efa39a43ba8ccef9598b14e468701f3263c1010` | + +These bounded wrappers apply runtime/thread limits and source hashes. They are +not the strict native pilot audit guard. The reviewed execution path uses +invented sources and no country simulation; flags or table counts alone do not +establish native/source admission. + +## Source-derived identities + +The merge had five derived-identity conflicts. Final values were recomputed +from the combined checkout using the existing spec loader/compiler, complete +coverage verifier and selective calibration parity tool. No parent branch's +hash was accepted as the final value. Fit and simulation parity pins were left +byte-identical. The parity tool also checks that the local direct calibration +output remains equal to its recorded bytes before rewriting its key. + +| Identity | Final value | +| --- | --- | +| Seed protocol | `5f93b3ec98ada30338b06ad978a6e1b47d0f9f916af89418500cea435235ad06` | +| Seed map | `da07a54ab2bc4e297ac7d0693a8559b359de230787757e4b6e26dcb619c8e5a0` | +| US resolved spec | `54aa5d96b062a66207dfe49d4a03817e9658bd0ac11acb37546b1bf2a01f533d` | +| Minimal loader golden | `a866bfe36a9eeb3b9a9888466b4b906faf4d8da57daf380d9bbc8ccf22e1e048` | +| Fiscal kernel implementation | `3d68e0f9b95a6d07f7a61b93f2044157ba4ab925e09a0a9600beebc16b60b8b7` | +| Calibration parity node | `f85d2af2276acc617779567e7ce8c97878febda0062acd7d2ddf326f4629d520` | + +The first identity-helper invocation correctly refused two advertised source +roots from the shared editable environment. Its failure is preserved. The +successful invocation disabled automatic site initialization and supplied the +owned shard roots plus dependency site-packages explicitly. No source-identity +check was relaxed. Its report SHA256 is +`69666f687e025d9a27d9d76b2cc87fefc2622734a56fc4530674b9de019ff23d`. + +Five focused checks of the final minimal/country spec goldens and calibration +parity identities then passed without failures or skips. They took 58.312 +seconds wall time with all 478 source pins unchanged. The receipt SHA256 is +`ee662e419c17acc27b76761b32f2bdbb096f2dc993e1efeb437d3f01006573e8`; +the JUnit result SHA256 is +`a60a4ffcc11b4afde6fe64a332c2bcfe54b8ac298f9aa523d9a4279089ccd116`. +No UK or US country-simulation test was selected. + +## Remaining acceptance + +This is solver and fiscal-host instrumentation, not a dataset release or +dashboard deployment. Complete-population/source authority remains external. +Native calibration quality, observer overhead at production target counts, +cadence choice, staging upload and a dashboard consumer need their own evidence. +Retain prior histories with their actual run identities; do not label cached +replay as new optimizer progress. diff --git a/experiments/grouped-target-snapshot-integration-20260912.json b/experiments/grouped-target-snapshot-integration-20260912.json new file mode 100644 index 000000000..5f7c0e72b --- /dev/null +++ b/experiments/grouped-target-snapshot-integration-20260912.json @@ -0,0 +1,90 @@ +{ + "lane": "grouped-target-snapshot-integration-20260912", + "date": "2026-09-12", + "scope": "shared calibration solver seam only; no host wiring, no native build, no release action, nothing pushed", + "base": { + "origin_main": "116d46ee9dc2aafdc68259b7c06e4c3462522e8b", + "moved_since_pin": false, + "note": "origin/main was fetched and is byte-identical to the shared base both reviewed heads were cut from, so no main drift had to be preserved" + }, + "integrated_heads": { + "grouped_us_solver": "536f1ceefcdafda3cc619c14b4da18e012a7be57", + "target_snapshots_pr914": "b43369dc49e175803f62020cc1e72fa53926aed8", + "merge_base_of_the_two": "116d46ee9dc2aafdc68259b7c06e4c3462522e8b" + }, + "pinned_source_hashes_verified_before_merge": { + "grouped/solve.py": "925a1b6bae1c714de93df3ee44e26afeed7f3912377b93942f3a5173281d851a", + "grouped/calibrate__init__.py": "69ac527f4732cdb67c4807bcb8ee9faec56174621f663ec491140dfc324a705f", + "snapshots/solve.py": "f43be0efecdd21b3f5caa9789edc8463a127423c665523ac7f0c8bc48f4b4eed", + "snapshots/calibrate__init__.py": "6b66552827852dd16a36512bdef76c28e47842d9a6b0b59d176a4a18a71f48ae", + "all_four_matched_the_pin_file": true + }, + "commits": [ + {"sha": "fff20fca56f41060db55742b884a6df2693e139f", "what": "merge of the two reviewed heads; neither solver replaced"}, + {"sha": "e64d5a8850ab301b7e819a6b972f235c51688568", "what": "grouped Adam snapshot emission and retain-best separation"}, + {"sha": "7ac2b864a93977ffa98fa86674c09e04ea6b20b5", "what": "source-derived identity recomputation"}, + {"sha": "2f62bb8448010bcc9b9e1419ea24aafcb85917d1", "what": "grouped x snapshot cross-product controls"} + ], + "final_source_hashes": { + "packages/microcosm-calibrate/src/microcosm/calibrate/solve.py": "e031ab5823d0f7bff43642e6ebb160ad867ec387e88658534ccd1e6ef62b273d", + "packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py": "1841264fe0c09a96ccf6aacf467cfa201de92ee67185b00cd378456fc7631a78", + "packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py": "a86958e3e94ca2454d997567883e2223e832f7edb5d994f689f0a64ddad1a88a", + "packages/microcosm-calibrate/src/microcosm/calibrate/group_bounds.py": "19a440a31d164832f61c84b2c1277a7a8798681f6830700bf03511dce7524557" + }, + "identity_recomputation": { + "rule": "recomputed against the merged checkout and the isolated interpreter; no value equals either branch's, so none is an ours/theirs choice", + "values": [ + {"artifact": "spec_engine/inventory_coverage.py EXPECTED_HASHES.seed_protocol", "grouped": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47", "snapshots": "d052fd875d014f6fa28d6f5b052735912f464d4efb2c98ecf373346b8cc454d3", "recomputed": "57e21b6522cfccc9b2d5e725503d9d39255abe5d5c9b391563d08ee664dea988"}, + {"artifact": "spec_engine/inventory_coverage.py EXPECTED_HASHES.seed_map", "grouped": "be2d24ad74bdc6339dfa073d01469074baee4e3a691eaa350451d36fa17d13dc", "snapshots": "d5a9694aac3e351f987b4f08ef8add0c25fcd3390d6bb2a851246da03adde516", "recomputed": "4ae2a0634e3a92061beba5bf5059db7c30b4ff6f1898e639697d246cb7e73878"}, + {"artifact": "us resolved spec_sha256 (us-f0-coverage.json and test_us_multispine_pool_tool.py)", "grouped": "f03e4d08d0ba4e15da53aab86bd2fbf50286e19e248657ac859b760103af2405", "snapshots": "ff2c97039853e18bfc0eb5388eb08c688cae33068b742f232d4c06a18b8efd88", "recomputed": "15e61194b4077de58272b14d83af95f7f54427aeab251429ada3cda01d93cbc8"}, + {"artifact": "test_spec_engine_loader.py minimal-loader golden", "grouped": "77f043a13853bbc109b6aa8cb42635ab1b2ebea69a1997b5bdb4c78a3726aeb3", "snapshots": "8a24089831b3a8f0cfe1be094769af7a30b7b74dbc7d4009e26e9f56444cba4c", "recomputed": "3868d14d5df04b5ab10ed4f0a56c2d9652c15c219840ed88d437f847fa481de7"}, + {"artifact": "parity kernels/calibrate/pins.json implementation_hash", "grouped": "59ca046b7c26bd872e56bd4cf3dd6170748fbe04e5a5da7b711e94aea8d3cc74", "snapshots": "46b52a20a7a5fac27d354c735a09ae5ba171d62bd9010fbc3079b06614cc2a32", "recomputed": "f2dd92eaaebba782e100cb92d90524d3ae6e2bab9ffedbacbe8df0257d5cae3d"}, + {"artifact": "parity kernels/calibrate/pins.json node_key", "grouped": "faaccb273d0aedeb1cca2e1b52c3e27678867512493f3c0379f596890c258cd1", "snapshots": "970720b71cb2579a5e2f714ca47deabc28c9249a83007ebfda83e2e7446d9311", "recomputed": "ae92142a3bf0934ac6b6751aca904c1847cdff3ca8533af83141d2cbf1ccf102"}, + {"artifact": "test_spec_engine_country_bundles.py am", "grouped": "c15bd12508731eb11fe061df810d9047b4c62c2887645cb19d3c54c7a104aa1c", "snapshots": "untouched-by-S", "recomputed": "258b940476e9f611d5e12d471dead7427479f36267eb40ed459fcd2f3dbd18ae"}, + {"artifact": "test_spec_engine_country_bundles.py be", "grouped": "5285b530974a9b8f7c21497fb140fdbd6060c3f1cc983987f35e9f244e71ee62", "snapshots": "untouched-by-S", "recomputed": "a61b65f7a1424c090081632fb1ddacdb85dae1b85920faff5dd7cb144cb1b5a1"}, + {"artifact": "test_spec_engine_country_bundles.py uk", "grouped": "1ac642230eb3166dd9ea2a113536985e35bf2898f914e289f5564ac8fde6e0b4", "snapshots": "untouched-by-S", "recomputed": "66837fd0fad4956566655f558d10f8d39e83523ad873b17e9f37ce0cb5012724"} + ], + "deliberately_not_repinned": [ + "packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json (G's value preserved; it hashes microcosm/frame/bundle.py, not calibrate.solve)", + "packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/ (neither head changed microcosm/fit/qrf.py)", + "parity direct.csv outputs (the calibrate kernel's outputs did not move, only its implementation hash)" + ], + "tooling": { + "us-f0-coverage.json": "tools/spec_engine_coverage.py (then --check exits 0)", + "calibrate parity pin": "tools/graph_parity_repin.py calibrate", + "loader golden and bundle digests": "taken from the observed value the failing identity assertion reports" + } + }, + "diff_summary": { + "vs_grouped_head": {"files": 25, "insertions": 4287, "deletions": 20, "note": "every deletion is an identity pin line, an extended import, an extended docstring, or a line S replaced; nothing from the grouped head was lost"}, + "vs_snapshot_head": {"files": 432, "insertions": 148099, "deletions": 3251, "note": "the grouped branch's whole delta"} + }, + "controls": { + "microcosm-calibrate shard": {"passed": 443, "failed": 0}, + "microcosm-graph shard + test_us_multispine_pool_tool + 4 spec-engine identity files": {"passed": 775, "failed": 0}, + "per_file": { + "test_target_snapshots.py": {"cases": 52, "added": 0}, + "test_grouped_fixed_support.py": {"cases": 42, "added": 18}, + "test_group_bounds.py": {"cases": 82, "added": 3}, + "test_solve_us_launch_integration.py": {"cases": 12, "added": 4} + }, + "tools/spec_engine_coverage.py --check": "exit 0", + "tools/ci_test_groups.py --verify": "verification=ok; no test file changed lane and none is [defaulted]", + "ruff": "format clean, check clean on packages/microcosm-calibrate" + }, + "environment": { + "interpreter": "borrowed root .venv/bin/python 3.14.4 run with -I -B -S and an explicit sys.path", + "sys_path_order": "this worktree's packages/*/src first, then the repo root, then the borrowed site-packages", + "imports_asserted_in_worktree": ["microcosm.frame", "microcosm.calibrate", "microcosm.build", "microcosm.fit", "microcosm.graph"], + "threads": "OMP/OPENBLAS/MKL/NUMEXPR/VECLIB all 1", + "pytest": "PYTEST_DISABLE_PLUGIN_AUTOLOAD=1, -p no:randomly", + "locked_versions": {"numpy": "2.4.6", "pandas": "3.0.3", "scipy": "1.17.1", "torch": "2.12.0", "platform": "arm64/darwin/py3.14"}, + "nothing_installed": true + }, + "not_done_on_purpose": [ + "host observer wiring in us_runtime/graph_fiscal_dense_calibration.py", + "any global observer registry, Node.params callback or filesystem path in graph params", + "any claim that a required replay reruns the optimizer", + "native build, PUF fixture, country build, dependency install, push, PR, publication or merge" + ] +} diff --git a/experiments/grouped-target-snapshot-integration-20260912.md b/experiments/grouped-target-snapshot-integration-20260912.md new file mode 100644 index 000000000..5e317e0bf --- /dev/null +++ b/experiments/grouped-target-snapshot-integration-20260912.md @@ -0,0 +1,130 @@ +# Grouped solver x calibration target snapshot integration — 2026-09-12 + +Historical scope note, 12 September 2026: this records grouped core +`2f62bb8448010bcc9b9e1419ea24aafcb85917d1` and its earlier test run. The later +[fiscal-host integration](fiscal-target-snapshot-host-20260912.md) adds the +host observer and PR #914's completed-update convention. The 443-test result +and host-wiring gaps below remain evidence of this earlier revision. + +Scope, evidence and residual risks for branch +`grouped-target-snapshot-integration-20260912`. Source-only; no native build, +no PUF fixture, no country build, no release action, nothing pushed. + +## What this is + +The bounded integration of the reviewed calibration target snapshot observer +(microcosm#908, head `b43369dc49e175803f62020cc1e72fa53926aed8`) with the +reviewed US grouped/fixed-zero Adam solver (head +`536f1ceefcdafda3cc619c14b4da18e012a7be57`), on a fresh `origin/main` +(`116d46ee9dc2aafdc68259b7c06e4c3462522e8b`). Main had not moved from the +shared base, so no main drift had to be preserved. + +Executed against the read-only design checklist +`grouped-snapshot-integration-review.md` and its pins. All four pinned source +hashes on both heads matched the pin file byte for byte before the merge. + +## What it is not + +- **Not** host wiring. `us_runtime/graph_fiscal_dense_calibration.py` still + calls `calibrate` without an observer, so this branch produces no snapshot + file for the actual US fiscal path. Wiring it is a separate, explicitly + host-owned step. No global observer registry, no callback or filesystem path + in graph `Node.params`, and no claim that any replay reruns the optimizer. +- **Not** a dashboard, a staging upload, or a cadence/performance acceptance. +- **Not** a release, a publication or a promotion of any artifact. + +## The one real code change + +A clean textual merge leaves grouped runs uninstrumented: `_optimize` returns +early into `_optimize_grouped` before reaching any snapshot hook S added to the +ordinary Adam loop, so a caller passing `target_snapshots` to a grouped +`calibrate` received exactly one closing snapshot and no trajectory. + +1. `_optimize` forwards its bound observer into `_optimize_grouped` + (`snapshots=snapshots`), and `_optimize_grouped` takes a default-`None` + `snapshots` argument. +2. `_optimize_grouped` emits inside the epoch loop, after the existing progress + callback and before `backward()`, from the exact float32 `estimate` tensor + the epoch's loss was computed from: `iterate=current`, `precision=float32`, + `loss=trajectory[epoch]`, `best_retained={available: False, epoch: None, + loss: None}`. +3. `calibrate` reads retain-best off the solver actually used rather than + inferring it from an empty `iterate_selection_receipt`, and stamps the + grouped closing snapshot at `epoch == epochs`. The receipt itself stays + empty, exactly as before. +4. Grouped snapshots carry three bounded aggregate scalars in `selection`: + `rule: "closing_state"`, `constraint_mode: "grouped_upper_bounds"`, + `grouped_preserve_zeros: `. + +Nothing else in either solver moved. No optimizer or matrix dtype, active mask, +zero template, L2 denominator, log/exp operation, optimizer construction, seed, +projection call, accepted-buffer assignment or `log_w` overwrite changed; there +is still exactly one `log_w` overwrite per accepted projection, the final no-op +projection and accepted-byte equality are unchanged, and the stored-result +admission (weight bytes, fixed-zero mask, bounds, ordered household IDs) still +runs after the last sink. + +### Why the `selection` labels + +Grouped Adam is a closing-state algorithm; it never runs the retain-best rule. +Without a label, its `best_retained.available: False` is indistinguishable from +an ordinary run whose retain-best rule happened to be off, and a consumer would +read the two the same way. Checklist section 2 sanctions exactly this — +"Optional grouped labels must be bounded aggregate scalars, e.g. constraint +mode and whether fixed-zero support is enabled". The labels mirror the run's +own `options["iterate_selection"]` and reuse the receipt's `rule` vocabulary; +they are three JSON scalars carrying no record-level content. This is the one +place the integration goes beyond the checklist's literal field list for +in-loop snapshots, and it is a single expression to remove if root disagrees. + +## Evidence + +- Grouped runs are bit-identical with the observer on and off across + dense x CSR, positive x fixed-zero support, and plain / warm-started / L2: + returned weights, loss trajectory, options, closing loss, stored frame + weights, ordered IDs, fixed-zero bytes, and the full sequence of private + post-projection proof payloads. +- Exactly 20 `_apply_constraint` and 3 `problem.estimates` calls with the + observer on and off on the 20-epoch fixture: cadence buys observation, not + evaluation. Torch and numpy RNG state are identical after each pair of runs. +- Each in-loop row carries the exact float32 tensor the epoch's loss was + computed from, verified against a wrapped `_apply_constraint`, and differs on + all 20 epochs from a recomputation on the accepted vector the private + observer reports next. +- The closing row equals the reused float64 final diagnostics estimates. +- A sink that ruins every delivered payload changes neither the weights nor the + next payload. A sink that mutates live frame IDs at the closing emission + still trips the ordered-ID admission guard. +- Zero epochs through the internal grouped seam emits nothing and evaluates + nothing. +- The deliberately oscillating example still selects grouped closing output + while ordinary Adam selects an earlier best, and each solver's closing + snapshot now says which rule produced it. + +Identity recomputation. Editing `solve.py` — an attested kernel module — moves +the seed protocol digest, every resolved-spec digest that folds it in, and the +calibrate kernel's implementation hash. Each value below was recomputed against +this checkout with the isolated Python 3.14 interpreter; none equals either +branch's value, so none is an ours/theirs choice. The simulate parity pin and +the fit.qrf pin were deliberately left alone: neither hashes +`microcosm.calibrate.solve`, and both are byte-unchanged from the grouped head. + +## Residual risks + +1. **Cadence cost on a real US run is unmeasured here.** The only overhead + evidence in the tree is #908's synthetic benchmark; this branch adds no + native measurement. A country-scale cadence choice is still unowned. +2. **The `selection` labels are an integration judgement**, not a checklist + requirement, on the in-loop rows specifically. See above. +3. **Host wiring remains the gap between this and any dashboard.** Green tests + here say the solver seam is correct; they say nothing about whether any real + US calibration emits anything, because none does yet. +4. **The identity recomputation is only as good as its interpreter.** Digests + fold locked dependency versions; a lane on different locked versions will + compute different parity keys. The values here were taken under + numpy 2.4.6 / pandas 3.0.3 / scipy 1.17.1 / torch 2.12.0 on + arm64/darwin/py3.14, which is what the calibrate parity pin records. +5. **The engine-gated identity tests were exercised with the engines present** + in the borrowed environment. A lane without the US extra will skip + `test_spec_engine_loader.py`'s golden and the country-bundle proofs rather + than check them. diff --git a/experiments/native-all-person-hours-20260919.json b/experiments/native-all-person-hours-20260919.json new file mode 100644 index 000000000..d74218984 --- /dev/null +++ b/experiments/native-all-person-hours-20260919.json @@ -0,0 +1,53 @@ +{ + "actual_asec_observation_audit": { + "counts": { + "age/age15": 2174, + "age/age16plus": 112272, + "age/under15": 27679, + "result/age15/asec_positive_source_hours": 243, + "result/age15/asec_source_nonwork_completion": 1931, + "result/age16plus/asec_positive_source_hours": 72217, + "result/age16plus/asec_source_nonwork_completion": 40055, + "result/under15/asec_source_universe_unavailable": 27679, + "rows": 142125, + "topcoded_hours/age16plus": 77, + "unresolved/under15": 27679 + }, + "member": "pppub25.csv", + "member_sha256": "06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5", + "receipt_sha256": "03b924aadbf49d5b66dd55fbdd9ffb8ae7a2ee449ce4531a1210f4ff4edb3fbc", + "refusals": 0, + "rows": 142125, + "scope": "Unweighted full-member observation diagnostic; original9bcd recoder, not source-owner authority or new child-completion execution" + }, + "base_commit": "9bcd039631ae9865f8caeaa8fe87b9b528265106", + "date": "2026-09-19", + "earlier_failed_attempt": "Attempt1 refused incomplete invented earnings columns; fixture repaired before fresh attempt2. Production field checks unchanged.", + "invented_tests": { + "cpu_seconds": 158.045512, + "failed": 0, + "passed": 144, + "peak_rss_bytes": 447184896, + "result_sha256": "7e62b4b78a0f1bb55aa6cc3820d9689ac029f78aeca67764a8f1b62f309ada24", + "skipped": 0, + "source_unchanged": true, + "wall_seconds": 226.9725150829181, + "xml_sha256": "1ba5fe413bd251639ea169175ba6b90f3436d2e2a734b57d310db20a572652bf" + }, + "remaining": [ + "Actual complete source-owner execution", + "Graph leaf and exact clone attachment", + "Engine input and release no-worse checks" + ], + "review": { + "independent_new_peer": "Unavailable after quota exhaustion; not claimed", + "root": "Complete source diff, source diagnostic, XML and current exact-byte checks reviewed" + }, + "scope": "Original survey person hours proposals; no clone/graph/engine/release qualification", + "source_and_test_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_usual_hours.py": "77878c90f89384077dd5d3f3397144b4fdcaaf2a6117d899295dc959fb151d55", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_hours_source.py": "86ab8112c7a34c4c3059352480949ec22e037698ba46bd98aa7a6d0b4c2c3b5c", + "packages/microcosm-build/tests/test_us_current_asec_usual_hours.py": "875a51b4afe5226d2a271f2b5d81da9c34b6f7808a8872edec3c90088432ca18", + "packages/microcosm-build/tests/test_us_current_survey_hours_source.py": "8e3d7383863c457bc7d3643fc9f05253cbd093f4014c3107aa64579e411f1d3f" + } +} diff --git a/experiments/native-byte-transports/agreement-sweep-receipt.json b/experiments/native-byte-transports/agreement-sweep-receipt.json new file mode 100644 index 000000000..8264b4cf0 --- /dev/null +++ b/experiments/native-byte-transports/agreement-sweep-receipt.json @@ -0,0 +1,9 @@ +{ + "scope": "Seeded agreement sweep over the segmented, digested and matched byte transports this lane introduced, against their whole-bytes oracles. Not a build, not a certification, not release eligible.", + "release_eligible": false, + "seed": 20260918, + "documents": 2000, + "comparisons": 34000, + "refusals": 12000, + "disagreements": [] +} diff --git a/experiments/native-byte-transports/agreement_sweep.py b/experiments/native-byte-transports/agreement_sweep.py new file mode 100644 index 000000000..00355e18c --- /dev/null +++ b/experiments/native-byte-transports/agreement_sweep.py @@ -0,0 +1,284 @@ +"""Seeded agreement sweep over every byte transport this lane changed the shape of. + +Max's standard from the retention lane (its report §11, question 3): wherever a +predicate becomes a fold, a seeded sweep runs beside the hand-built battery. +Five predicates became folds here, one per encoder, and each is swept against +the whole-bytes oracle it replaces: + +* ``graph_survey_population._segmented_json`` against ``_bounded_json`` at + random segment sizes down to the longest token; ``_json_sha256`` against + ``_sha(_bounded_json)``; ``_json_matches`` against ``==`` on the whole + bytes, on every mutation (a flipped byte, a truncation, an extension, an + empty payload) and on the unmutated bytes. +* ``acs_person_coverage_authentication._json_roster`` against ``_json`` at + random segment sizes; ``_json_sha256`` against ``_sha(_json)``. +* ``graph_child_property_income._json`` against ``adapter._json`` + (``json.dumps`` with the adapter's settings) at random accumulation sizes. +* ``survey_origin_budget``'s streamed ``household_ids``/``group_indices`` + lists: ``append_list`` is exercised through the module by encoding a list + element by element and comparing with ``_json(list)``; the sweep reproduces + the loop's bytes here because the loop is a closure inside ``_document``. + +Every document is drawn from a seeded generator over nested dicts and lists of +ints, floats (finite, including -0.0 and subnormals), ASCII and non-ASCII +strings, None and booleans, at random depth and width. Refusals are also +swept: below the longest token every segmented encoder must refuse with its +per-accumulation code, and one byte over the total every one must refuse with +its total code. + +Not a build, not a certification, not release eligible. + + python agreement_sweep.py [pairs] +""" + +from __future__ import annotations + +import hashlib +import json +import pathlib +import random +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[2] +sys.path[:0] = [str(path) for path in sorted((ROOT / "packages").glob("*/src"))] + +from microcosm.build.us_runtime import acs_person_coverage_authentication as auth +from microcosm.build.us_runtime import graph_child_property_income as child +from microcosm.build.us_runtime import graph_survey_population as graph +from microcosm.build.us_runtime import survey_origin_budget as budget +from microcosm.fit import graph_joint_empirical as adapter + +for _module in (auth, child, graph, budget, adapter): + if not pathlib.Path(_module.__file__).resolve().is_relative_to(ROOT): + raise SystemExit(f"{_module.__name__} resolved outside {ROOT}") + +SEED = 20260918 +ALPHABET = "abcXYZ019 _-./\\\"'\n\té中\U0001f600" + + +def _scalar(rng, floats): + # The coverage encoder admits str, int, bool, None, list and dict only + # (acs_person_coverage_authentication._json_size refuses CANONICAL_TYPE on + # anything else), so floats are drawn only for the encoders that take them. + kind = rng.random() + if kind < 0.25: + return rng.randint(-(2**62), 2**62) + if kind < 0.45 and floats: + return rng.choice( + [ + 0.0, + -0.0, + 1.5, + 137.0 / 7, + 5e-324, + 1e300, + rng.random() * 1e6, + -rng.random(), + ] + ) + if kind < 0.75: + return "".join(rng.choice(ALPHABET) for _ in range(rng.randint(0, 24))) + if kind < 0.85: + return None + return rng.random() < 0.5 + + +def _document(rng, depth=0, floats=True): + if depth >= 3 or rng.random() < 0.3: + return _scalar(rng, floats) + if rng.random() < 0.5: + return [_document(rng, depth + 1, floats) for _ in range(rng.randint(0, 12))] + keys = [ + "".join(rng.choice(ALPHABET[:12]) for _ in range(rng.randint(1, 8))) + for _ in range(rng.randint(0, 8)) + ] + return {k: _document(rng, depth + 1, floats) for k in keys} + + +def _roster(rng, floats=True): + # A document with a dict at the top and a long list somewhere, like every + # whole-roster document on the path. + return { + "protocol": "invented/sweep", + "rows": [_document(rng, 1, floats) for _ in range(rng.randint(1, 60))], + "header": _document(rng, 1, floats), + } + + +def _mutations(rng, whole): + yield "truncated", whole[:-1] + yield "extended", whole + b"x" + yield "empty", b"" + if whole: + position = rng.randrange(len(whole)) + flipped = bytearray(whole) + flipped[position] ^= 1 << rng.randrange(8) + yield "flipped", bytes(flipped) + + +def sweep_graph(rng, receipt): + document = _roster(rng) + whole = graph._bounded_json(document, 64 * 1024**2) + pieces = list(graph._canonical_pieces(document, 64 * 1024**2)) + longest = max(map(len, pieces)) + for _ in range(3): + segment = rng.randint(longest, max(longest, len(whole)) + 8) + receipt["comparisons"] += 1 + if ( + graph._segmented_json(document, segment=segment, maximum=len(whole)) + != whole + ): + receipt["disagreements"].append( + ("graph._segmented_json", segment, whole[:80]) + ) + receipt["comparisons"] += 1 + if ( + graph._json_sha256(document, maximum=len(whole)) + != hashlib.sha256(whole).hexdigest() + ): + receipt["disagreements"].append(("graph._json_sha256", whole[:80])) + receipt["comparisons"] += 1 + if graph._json_matches(document, whole, maximum=len(whole)) is not True: + receipt["disagreements"].append(("graph._json_matches whole", whole[:80])) + for name, mutated in _mutations(rng, whole): + receipt["comparisons"] += 1 + if graph._json_matches(document, mutated, maximum=2**40) is not False: + receipt["disagreements"].append(("graph._json_matches " + name, whole[:80])) + if longest > 1 and longest < len(whole): + receipt["refusals"] += 1 + try: + graph._segmented_json(document, segment=longest - 1, maximum=len(whole)) + receipt["disagreements"].append( + ("graph accumulation refusal missing", whole[:80]) + ) + except graph.SurveyPopulationGraphError as error: + if str(error) != "TRANSPORT_LIMIT": + receipt["disagreements"].append(("graph accumulation code", str(error))) + receipt["refusals"] += 1 + try: + graph._segmented_json(document, segment=longest, maximum=len(whole) - 1) + receipt["disagreements"].append(("graph total refusal missing", whole[:80])) + except graph.SurveyPopulationGraphError as error: + if str(error) != "TRANSPORT_ROSTER_LIMIT": + receipt["disagreements"].append(("graph total code", str(error))) + + +def sweep_coverage(rng, receipt): + document = _roster(rng, floats=False) + whole = auth._json(document, auth.MAX_ROSTER_BYTES) + pieces = list(auth._json_chunks(document, auth.MAX_ROSTER_BYTES)) + longest = max(map(len, pieces)) + for _ in range(3): + segment = rng.randint(longest, max(longest, len(whole)) + 8) + receipt["comparisons"] += 1 + if auth._json_roster(document, segment=segment, maximum=len(whole)) != whole: + receipt["disagreements"].append(("auth._json_roster", segment, whole[:80])) + receipt["comparisons"] += 1 + if auth._json_sha256(document, len(whole)) != hashlib.sha256(whole).hexdigest(): + receipt["disagreements"].append(("auth._json_sha256", whole[:80])) + if longest > 1 and longest < len(whole): + receipt["refusals"] += 1 + try: + auth._json_roster(document, segment=longest - 1, maximum=len(whole)) + receipt["disagreements"].append( + ("auth accumulation refusal missing", whole[:80]) + ) + except auth.ACSCoverageAuthenticationError as error: + if str(error) != "CANONICAL_SIZE": + receipt["disagreements"].append(("auth accumulation code", str(error))) + receipt["refusals"] += 1 + try: + auth._json_roster(document, segment=longest, maximum=len(whole) - 1) + receipt["disagreements"].append(("auth total refusal missing", whole[:80])) + except auth.ACSCoverageAuthenticationError as error: + if str(error) != "CANONICAL_SIZE": + receipt["disagreements"].append(("auth total code", str(error))) + + +def sweep_child(rng, receipt, monkey): + document = _roster(rng) + whole = adapter._json(document) + encoder = json.JSONEncoder(sort_keys=True, separators=(",", ":"), allow_nan=False) + longest = max(len(piece.encode()) for piece in encoder.iterencode(document)) + for _ in range(3): + segment = rng.randint(longest, max(longest, len(whole)) + 8) + monkey(child, "MAX_ARTIFACT_BYTES", segment) + monkey(child, "MAX_ROSTER_BYTES", len(whole)) + receipt["comparisons"] += 1 + if child._json(document) != whole: + receipt["disagreements"].append(("child._json", segment, whole[:80])) + if longest > 1 and longest < len(whole): + monkey(child, "MAX_ARTIFACT_BYTES", longest - 1) + monkey(child, "MAX_ROSTER_BYTES", 2**40) + receipt["refusals"] += 1 + try: + child._json(document) + receipt["disagreements"].append( + ("child accumulation refusal missing", whole[:80]) + ) + except ValueError as error: + if "GRAPH_ARTIFACT_SIZE" not in str(error): + receipt["disagreements"].append(("child accumulation code", str(error))) + monkey(child, "MAX_ARTIFACT_BYTES", 64 * 1024**2) + monkey(child, "MAX_ROSTER_BYTES", len(whole) - 1) + receipt["refusals"] += 1 + try: + child._json(document) + receipt["disagreements"].append(("child total refusal missing", whole[:80])) + except ValueError as error: + if "GRAPH_ARTIFACT_LIMIT" not in str(error): + receipt["disagreements"].append(("child total code", str(error))) + + +def sweep_budget_lists(rng, receipt): + # The two header lists are streamed one element at a time inside + # _document's append_list; the bytes those appends produce are + # b"[" + b",".join(_json(v)) + b"]", which must equal _json(list). + values = [rng.randint(-(2**62), 2**62) for _ in range(rng.randint(0, 200))] + streamed = b"[" + b",".join(budget._json(v) for v in values) + b"]" + receipt["comparisons"] += 1 + if streamed != budget._json(values): + receipt["disagreements"].append(("budget append_list", values[:5])) + + +def main() -> int: + out = pathlib.Path(sys.argv[1]) + pairs = int(sys.argv[2]) if len(sys.argv) > 2 else 2000 + rng = random.Random(SEED) + receipt = { + "scope": "Seeded agreement sweep over the segmented, digested and matched byte transports this lane introduced, against their whole-bytes oracles. Not a build, not a certification, not release eligible.", + "release_eligible": False, + "seed": SEED, + "documents": pairs, + "comparisons": 0, + "refusals": 0, + "disagreements": [], + } + saved = {} + + def monkey(module, name, value): + saved.setdefault((module, name), getattr(module, name)) + setattr(module, name, value) + + try: + for _ in range(pairs): + sweep_graph(rng, receipt) + sweep_coverage(rng, receipt) + sweep_child(rng, receipt, monkey) + sweep_budget_lists(rng, receipt) + finally: + for (module, name), value in saved.items(): + setattr(module, name, value) + receipt["disagreements"] = [ + [str(part)[:200] for part in item] for item in receipt["disagreements"] + ] + out.write_text(json.dumps(receipt, indent=1) + "\n") + print( + f"documents={pairs} comparisons={receipt['comparisons']} " + f"refusals={receipt['refusals']} disagreements={len(receipt['disagreements'])}" + ) + return 1 if receipt["disagreements"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/native-byte-transports/byte-census-measurements.json b/experiments/native-byte-transports/byte-census-measurements.json new file mode 100644 index 000000000..51e313efd --- /dev/null +++ b/experiments/native-byte-transports/byte-census-measurements.json @@ -0,0 +1,170 @@ +{ + "scope": "Byte measurements for the byte-transport lane, through each module's own encoder, over the recovered 1/1000 artifact's catalogue counts and the full captured public ACS PUMS archive. Not a build, not a certification, not release eligible.", + "release_eligible": false, + "counts": { + "acs_households": 1531614, + "acs_persons": 3422888, + "asec_households": 55762, + "asec_persons": 142125, + "stacked_households": 1587376, + "stacked_persons": 3565013, + "clone_households": 3174752, + "clone_persons": 7130026 + }, + "origin_budget": { + "encoder": "survey_origin_budget._json (graph._bounded_json) per record", + "rows": { + "1587376": { + "record_bytes": 709, + "per_group_bytes": 742 + }, + "3174752": { + "record_bytes": 709, + "per_group_bytes": 742 + } + }, + "per_group_bytes": 742, + "full_source_bytes": 1177832992, + "tenth_bytes": 117782854, + "households_the_64MiB_cap_admits": 90443, + "household_ids_list_bytes": 25398017, + "group_indices_list_bytes": 25398017 + }, + "numeric_bounds": { + "encoder": "graph._bounded_json(document, graph_survey_calibration.MAX_BYTES)", + "bytes_at_groups": { + "2": 585, + "2002": 304525 + }, + "per_group_bytes": 151.97, + "per_clone_row_bytes": 75.98, + "full_source_bytes": 241233530, + "tenth_bytes": 24123261, + "clone_rows_the_64MiB_cap_admits": 883185, + "max_rows": 524288, + "clone_rows_full_source": 3174752 + }, + "age_artifact": { + "encoder": "MAGIC + header + values.tobytes(order='C'), values encode_recipient_matrix, FEATURES=5", + "bytes_per_tax_unit": 48, + "tax_units_per_household_at_1_1000": 1.343434, + "clone_one_tax_units_full_source_SCALED_FROM_1_1000": 2132535, + "full_source_bytes_SCALED": 102361680, + "tenth_bytes_SCALED": 10236144, + "tax_units_the_64MiB_cap_admits": 1398101, + "note": "tax units per household is a roster ratio from the 1/1000 artifact, not a catalogue count" + }, + "child_draw": { + "encoder": "graph_child_property_income._json(adapter._draw_document(...)) [producer]; adapter._document(payload, MAX_DRAW_BYTES) [consumer]", + "per_eligible_child_bytes": 273.0, + "eligible_children_full_source_ESTIMATE": null, + "children_the_64MiB_cap_admits": 245820, + "max_recipients_fit": 1048576, + "max_draw_bytes_fit": 67108864, + "max_artifact_bytes_graph": 67108864, + "note": "eligible-child count is not in the 1/1000 artifact; a census agent estimates it from ACS/ASEC age shares" + }, + "selection_block": { + "encoder": "survey_origin_budget._json(view.receipt['selection']) for selection_sha256 (graph._bounded_json, 64 MiB)", + "per_selected_row_bytes_at_1_1000": 243.13, + "full_source_bytes_SCALED": 385942815, + "tenth_bytes_SCALED": 38594135, + "rows_the_64MiB_cap_admits": 276017, + "note": "row widths at 1/1000 (inclusion_probability pairs, ids) differ slightly from full-source widths; scaled, not encoded at full-source widths" + }, + "acs_household_archive": { + "records": 1631969, + "occupied_records": 1531614, + "raw_bytes": 977481380, + "old_charge_bytes_all": 7536024536, + "old_charge_bytes_occupied": 7095161910, + "np_total": 3422888, + "serialno_widths": { + "13": 1631969 + }, + "members": { + "psam_husa.csv": 827133, + "psam_husb.csv": 804836 + }, + "serialno_list_bytes_full_source": 24505825, + "seconds": 13.9, + "derived": { + "old_charge_occupied_over_64MiB": 105.72615131735802, + "np_total_equals_person_records": true, + "serialno_list_over_1MiB": 23.37057590484619, + "serialno_list_over_2MiB": 11.685287952423096, + "serialnos_the_1MiB_cap_admits": 65535 + } + }, + "acs_person_archive": { + "records": 3422888, + "raw_bytes": 2405464349, + "raw_max": 877, + "old_charge_bytes": 17937823406, + "body_bytes": 308941698, + "body_row_max": 102, + "cells_json_bytes": 198264171, + "cells_json_max": 59, + "key_pair_bytes": 68461899, + "sporder_two_digit_rows": 4139, + "states": { + "observed_code": 5704812, + "outside_age_universe": 1140964 + }, + "members": { + "psam_pusa.csv": 1743751, + "psam_pusb.csv": 1679137 + }, + "key_list_bytes": 68461900, + "seconds": 182.7, + "derived": { + "old_charge_admits_rows_at_64MiB": 12805, + "old_charge_full_source_over_64MiB": 267.2943980395794, + "body_full_source_over_64MiB": 4.603590041399002, + "body_admits_rows_at_64MiB": 743525, + "key_list_full_source_over_64MiB": 1.0201618075370789, + "body_bytes_per_row_mean": 90.25761228529826, + "old_charge_over_body": 58.06216358013285 + } + } +} diff --git a/experiments/native-byte-transports/byte_census_measure.py b/experiments/native-byte-transports/byte_census_measure.py new file mode 100644 index 000000000..8349edafe --- /dev/null +++ b/experiments/native-byte-transports/byte_census_measure.py @@ -0,0 +1,543 @@ +"""Measure every byte transport this lane censuses, through each module's own encoder. + +The row-count lane established the full-source counts from the recovered +1/1000 artifact's catalogues (1,531,614 selectable ACS households, 3,422,888 +ACS persons, 55,762 ASEC households, 142,125 ASEC persons, 1,587,376 stacked +households, 3,565,013 stacked persons, 3,174,752 clone households). This +measures the *bytes* each transport meets at those counts: + +* the ACS archive is streamed in full -- every person record and every + household record of the pilot's captured public ACS PUMS archive -- so the + coverage body, the pre-allocation charge, the selected-key list and the + serialno list are sums over the real rows, not samples scaled up; +* each JSON roster document is encoded through its own module's encoder on + rows shaped at full-source id widths, and the marginal cost per row is a + difference between two row counts so fixed preambles cancel; +* each binary artifact is computed from the module's own width constants. + +Where a count is a roster ratio scaled from 1/1000 rather than a catalogue +count (tax units per household, eligible children), the receipt says so. + +Nothing outside the output path is written. Not a build, not a certification, +not release eligible. + + python byte_census_measure.py +""" + +from __future__ import annotations + +import csv +import io +import json +import pathlib +import sys +import time +import zipfile +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +ROOT = pathlib.Path(__file__).resolve().parents[2] +sys.path[:0] = [str(path) for path in sorted((ROOT / "packages").glob("*/src"))] + +from microcosm.build.us_runtime import acs_person_coverage_authentication as auth +from microcosm.build.us_runtime import acs_person_coverage_columns as literal +from microcosm.build.us_runtime import current_survey_household_roles as roles +from microcosm.build.us_runtime import current_survey_predictors as predictors +from microcosm.build.us_runtime import graph_child_property_income as child_graph +from microcosm.build.us_runtime import graph_combined_clone as clone +from microcosm.build.us_runtime import ( + graph_current_survey_household_roles as roles_graph, +) +from microcosm.build.us_runtime import graph_survey_age_artifact as ages +from microcosm.build.us_runtime import graph_survey_calibration as numeric +from microcosm.build.us_runtime import graph_survey_population as graph +from microcosm.build.us_runtime import puf_detail_transfer as detail +from microcosm.build.us_runtime import survey_origin_budget as budget_owner +from microcosm.fit import graph_joint_empirical as adapter + +for _module in (auth, literal, graph, budget_owner, numeric, ages, roles, adapter): + if not pathlib.Path(_module.__file__).resolve().is_relative_to(ROOT): + raise SystemExit(f"{_module.__name__} resolved outside {ROOT}") + +# Measured full-source counts; see experiments/native-row-ceilings/roster-census.json. +ACS_HOUSEHOLDS = 1_531_614 +ACS_PERSONS = 3_422_888 +ASEC_HOUSEHOLDS = 55_762 +ASEC_PERSONS = 142_125 +STACKED_HOUSEHOLDS = 1_587_376 +STACKED_PERSONS = 3_565_013 +CLONE_HOUSEHOLDS = 3_174_752 +CLONE_PERSONS = 7_130_026 +MIB = 1024**2 + + +def _members(archive, prefix): + with zipfile.ZipFile(archive) as zf: + for info in sorted(zf.infolist(), key=lambda m: m.filename): + name = info.filename.casefold() + if name.startswith(prefix) and name.endswith(".csv"): + yield zf, info + + +def _rows(stream, first): + """Raw record bytes and decoded cells, the way the owner decodes them.""" + header = None + for raw in stream: + text = raw.decode("utf-8-sig" if header is None and first else "utf-8") + values = next(csv.reader(io.StringIO(text, newline=""))) + if header is None: + header = values + yield raw, header, None + continue + yield raw, header, values + + +def acs_persons(archive, contract): + fields = contract["fields"] + out = { + "records": 0, + "raw_bytes": 0, + "raw_max": 0, + "old_charge_bytes": 0, # 6 * len(raw) + 1024, the shipped pre-charge + "body_bytes": 0, # exact NDJSON body: _json(row) + newline + "body_row_max": 0, + "cells_json_bytes": 0, # _json([*READ_COLUMNS cells, member, ordinal]) + "cells_json_max": 0, + "key_pair_bytes": 0, # _json([SERIALNO, SPORDER]) per person, plus commas + "sporder_two_digit_rows": 0, + "states": {}, + "members": {}, + } + started = time.monotonic() + for zf, info in _members(archive, "psam_pus"): + positions = None + ordinal = 0 + with zf.open(info) as stream: + for raw, header, values in _rows(stream, True): + if values is None: + positions = [header.index(c) for c in literal.READ_COLUMNS] + continue + ordinal += 1 + cells = [values[p] for p in positions] + serial, sporder, agep, mil, esr = cells + out["records"] += 1 + out["raw_bytes"] += len(raw) + out["raw_max"] = max(out["raw_max"], len(raw)) + out["old_charge_bytes"] += 6 * len(raw) + 1024 + states = [ + literal._field_state( + agep, + value, + minimum_age=fields[name]["minimum_age"], + codes=fields[name]["codes"], + ) + for name, value in (("MIL", mil), ("ESR", esr)) + ] + for state in states: + out["states"][state] = out["states"].get(state, 0) + 1 + row = auth._json( + [ + serial, + int(sporder), + agep, + mil, + esr, + *states, + info.filename, + ordinal, + ], + auth.MAX_RECORD_BYTES - 1, + ) + out["body_bytes"] += len(row) + 1 + out["body_row_max"] = max(out["body_row_max"], len(row) + 1) + cells_json = len( + auth._json( + [*cells, info.filename, ordinal], auth.MAX_RECORD_BYTES - 1 + ) + ) + out["cells_json_bytes"] += cells_json + out["cells_json_max"] = max(out["cells_json_max"], cells_json) + out["key_pair_bytes"] += len(auth._json([serial, int(sporder)])) + 1 + if int(sporder) >= 10: + out["sporder_two_digit_rows"] += 1 + out["members"][info.filename] = ordinal + out["key_list_bytes"] = out["key_pair_bytes"] - 1 + 2 # no trailing comma; brackets + out["seconds"] = round(time.monotonic() - started, 1) + return out + + +def acs_households(archive): + out = { + "records": 0, + "occupied_records": 0, + "raw_bytes": 0, + "old_charge_bytes_all": 0, + "old_charge_bytes_occupied": 0, + "np_total": 0, + "serialno_widths": {}, + "members": {}, + } + started = time.monotonic() + for zf, info in _members(archive, "psam_hus"): + positions = None + ordinal = 0 + with zf.open(info) as stream: + for raw, header, values in _rows(stream, True): + if values is None: + positions = [header.index(c) for c in ("SERIALNO", "NP")] + continue + ordinal += 1 + serial, np_ = (values[p] for p in positions) + out["records"] += 1 + out["raw_bytes"] += len(raw) + charge = 6 * len(raw) + 1024 + out["old_charge_bytes_all"] += charge + width = str(len(serial)) + out["serialno_widths"][width] = out["serialno_widths"].get(width, 0) + 1 + n = int(np_) + if n > 0: + out["occupied_records"] += 1 + out["old_charge_bytes_occupied"] += charge + out["np_total"] += n + out["members"][info.filename] = ordinal + # 16 canonical bytes per 13-character key: quotes, 13 chars, separator. + out["serialno_list_bytes_full_source"] = 16 * out["occupied_records"] + 1 + out["seconds"] = round(time.monotonic() - started, 1) + return out + + +def origin_budget(): + from fractions import Fraction + + design, probability, share = Fraction(137, 1), Fraction(1, 1), Fraction(1, 2) + actual = float(design * share / probability) + reference = budget_owner._reference(design, probability, share, actual) + rows = {} + for magnitude in (STACKED_HOUSEHOLDS, CLONE_HOUSEHOLDS): + record = { + "source": "acs", + "source_year": 2024, + "survey_year": 2024, + "raw_native_id": f"2024HU{magnitude % 10_000_000:07d}", + "selected_receiving_household_id": magnitude, + "combined_household_id": magnitude, + "statistical_unit": "occupied_housing_unit", + "original_design_float64_bytes": "0" * 16, + **reference, + "members": [[magnitude, 0], [magnitude * 2, 1]], + "incoming_clone_float64_bytes": ["0" * 16, "0" * 16], + } + encoded = len(budget_owner._json(record)) + ids = len(str(magnitude * 2)) * 2 + 2 + groups = len(str(magnitude)) * 2 + 2 + rows[str(magnitude)] = { + "record_bytes": encoded, + "per_group_bytes": encoded + 1 + ids + groups, + } + per_group = rows[str(STACKED_HOUSEHOLDS)]["per_group_bytes"] + return { + "encoder": "survey_origin_budget._json (graph._bounded_json) per record", + "rows": rows, + "per_group_bytes": per_group, + "full_source_bytes": per_group * STACKED_HOUSEHOLDS, + "tenth_bytes": per_group * (STACKED_HOUSEHOLDS // 10), + "households_the_64MiB_cap_admits": budget_owner.MAX_PAYLOAD_BYTES // per_group, + # The two header lists alone, each one entry per clone household row. + "household_ids_list_bytes": (len(str(CLONE_HOUSEHOLDS)) + 1) * CLONE_HOUSEHOLDS + + 1, + "group_indices_list_bytes": (len(str(STACKED_HOUSEHOLDS)) + 1) + * CLONE_HOUSEHOLDS + + 1, + } + + +def numeric_bounds(): + def document(groups): + ids = list(range(CLONE_HOUSEHOLDS - 2 * groups, CLONE_HOUSEHOLDS)) + indices = [STACKED_HOUSEHOLDS - groups + i // 2 for i in range(2 * groups)] + upper = [float(137 * 2 * (1 + i / 7)).hex() for i in range(groups)] + row_upper = [float(137 * 8 * (1 + i / 7)).hex() for i in range(2 * groups)] + incoming = [float(137 * (1 + i / 7)).hex() for i in range(2 * groups)] + return { + "protocol": numeric.BOUNDS_PROTOCOL, + "budget_sha256": "a" * 64, + "population": clone.COMBINED_CLONE_NODE, + "household_ids": ids, + "group_indices": indices, + "group_upper_hex": upper, + "row_upper_hex": row_upper, + "incoming_hex": incoming, + } + + small, large = 2, 2_002 + bytes_small = len(graph._bounded_json(document(small), numeric.MAX_BYTES)) + bytes_large = len(graph._bounded_json(document(large), numeric.MAX_BYTES)) + per_group = (bytes_large - bytes_small) / (large - small) + return { + "encoder": "graph._bounded_json(document, graph_survey_calibration.MAX_BYTES)", + "bytes_at_groups": {str(small): bytes_small, str(large): bytes_large}, + "per_group_bytes": round(per_group, 2), + "per_clone_row_bytes": round(per_group / 2, 2), + "full_source_bytes": int(per_group * STACKED_HOUSEHOLDS), + "tenth_bytes": int(per_group * (STACKED_HOUSEHOLDS // 10)), + "clone_rows_the_64MiB_cap_admits": int(numeric.MAX_BYTES // (per_group / 2)), + "max_rows": numeric.MAX_ROWS, + "clone_rows_full_source": CLONE_HOUSEHOLDS, + } + + +def age_artifact(): + header = ages._json( + { + "protocol": "microcosm.us.survey-household-age-counts.v1", + "population": clone.COMBINED_CLONE_NODE, + "columns": list(ages._COLUMNS), + "rows": CLONE_HOUSEHOLDS, + "people": CLONE_PERSONS, + "age_convention": "integer_years_completed", + } + ) + row = ages._WIDTH * 8 + fixed = len(ages.MAGIC) + 4 + len(header) + reserved = len(ages.MAGIC) + 4 + ages.MAX_HEADER_BYTES + return { + "encoder": "MAGIC + header + values.tobytes(order='C'), values encode_recipient_matrix, FEATURES={len(detail.FEATURES)}", + "bytes_per_tax_unit": row, + "tax_units_per_household_at_1_1000": round(ratio, 6), + "clone_one_tax_units_full_source_SCALED_FROM_1_1000": clone_one, + "full_source_bytes_SCALED": row * clone_one, + "tenth_bytes_SCALED": row * (clone_one // 10), + "tax_units_the_64MiB_cap_admits": (64 * MIB) // row, + "note": "tax units per household is a roster ratio from the 1/1000 artifact, not a catalogue count", + } + + +def child_draw(): + key = ("acs", 2024, 2024, "2024HU0001234567", "12", "12") + n = 1_001 + draw = SimpleNamespace( + values=np.tile(np.array([[1234.5, 0.0]]), (n, 1)), + patterns=np.ones(n, dtype=np.int64), + donor_keys=[("asec", 2024, 2025, "1234567", "12", "1")] * n, + model_sha256="b" * 64, + ) + params = { + "support_sha256": "c" * 64, + "donor_source_sha256": "d" * 64, + "source_sha256": "e" * 64, + "scenario_sha256": "f" * 64, + "transport_json": '{"kind":"nearest"}', + "transport_sha256": "0" * 64, + "stream": ["child-property", 1], + "coordinate_suffix": ("child-property-v1",), + } + + def encoded(rows): + ids = np.arange(STACKED_PERSONS - rows, STACKED_PERSONS) + keys = [key] * rows + partial = SimpleNamespace( + values=draw.values[:rows], + patterns=draw.patterns[:rows], + donor_keys=draw.donor_keys[:rows], + model_sha256=draw.model_sha256, + ) + return len(adapter._json(adapter._draw_document(partial, ids, keys, params))) + + a, b = encoded(1), encoded(n) + per_row = (b - a) / (n - 1) + return { + "encoder": "graph_child_property_income._json(adapter._draw_document(...)) [producer]; adapter._document(payload, MAX_DRAW_BYTES) [consumer]", + "per_eligible_child_bytes": round(per_row, 2), + "eligible_children_full_source_ESTIMATE": None, + "children_the_64MiB_cap_admits": int(64 * MIB // per_row), + "max_recipients_fit": adapter.MAX_RECIPIENTS, + "max_draw_bytes_fit": adapter.MAX_DRAW_BYTES, + "max_artifact_bytes_graph": child_graph.MAX_ARTIFACT_BYTES, + "note": "eligible-child count is not in the 1/1000 artifact; a census agent estimates it from ACS/ASEC age shares", + } + + +def selection_block(preparation): + selected = preparation["selection"]["selected"] + + def enc(v): + return len( + json.dumps( + v, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode() + ) + + per_row = enc(selected) / len(selected) + return { + "encoder": "survey_origin_budget._json(view.receipt['selection']) for selection_sha256 (graph._bounded_json, 64 MiB)", + "per_selected_row_bytes_at_1_1000": round(per_row, 2), + "full_source_bytes_SCALED": int(per_row * STACKED_HOUSEHOLDS), + "tenth_bytes_SCALED": int(per_row * (STACKED_HOUSEHOLDS // 10)), + "rows_the_64MiB_cap_admits": int(64 * MIB // per_row), + "note": "row widths at 1/1000 (inclusion_probability pairs, ids) differ slightly from full-source widths; scaled, not encoded at full-source widths", + } + + +def main() -> int: + snapshot = pathlib.Path(sys.argv[1]) + preparation = json.loads(pathlib.Path(sys.argv[2]).read_bytes()) + out = pathlib.Path(sys.argv[3]) + contract = literal.coverage_field_contract() + record = { + "scope": ( + "Byte measurements for the byte-transport lane, through each module's own " + "encoder, over the recovered 1/1000 artifact's catalogue counts and the full " + "captured public ACS PUMS archive. Not a build, not a certification, not " + "release eligible." + ), + "release_eligible": False, + "counts": { + "acs_households": ACS_HOUSEHOLDS, + "acs_persons": ACS_PERSONS, + "asec_households": ASEC_HOUSEHOLDS, + "asec_persons": ASEC_PERSONS, + "stacked_households": STACKED_HOUSEHOLDS, + "stacked_persons": STACKED_PERSONS, + "clone_households": CLONE_HOUSEHOLDS, + "clone_persons": CLONE_PERSONS, + }, + "origin_budget": origin_budget(), + "numeric_bounds": numeric_bounds(), + "age_artifact": age_artifact(), + "roles_projection": roles_projection(), + "predictor_projection": predictor_projection(), + "diagnostic_matrix": diagnostic_matrix(preparation), + "child_draw": child_draw(), + "selection_block": selection_block(preparation), + } + out.write_text(json.dumps(record, indent=1) + "\n") + print("encoder measurements written; streaming the ACS archive ...", flush=True) + record["acs_household_archive"] = acs_households(snapshot / "csv_hus.zip") + out.write_text(json.dumps(record, indent=1) + "\n") + print( + "household archive done", + record["acs_household_archive"]["seconds"], + "s", + flush=True, + ) + record["acs_person_archive"] = acs_persons(snapshot / "csv_pus.zip", contract) + people = record["acs_person_archive"] + people["derived"] = { + "old_charge_admits_rows_at_64MiB": int( + auth.MAX_BODY_BYTES / (people["old_charge_bytes"] / people["records"]) + ), + "old_charge_full_source_over_64MiB": people["old_charge_bytes"] + / auth.MAX_BODY_BYTES, + "body_full_source_over_64MiB": people["body_bytes"] / auth.MAX_BODY_BYTES, + "body_admits_rows_at_64MiB": int( + auth.MAX_BODY_BYTES / (people["body_bytes"] / people["records"]) + ), + "key_list_full_source_over_64MiB": people["key_list_bytes"] + / auth.MAX_BODY_BYTES, + "body_bytes_per_row_mean": people["body_bytes"] / people["records"], + "old_charge_over_body": people["old_charge_bytes"] / people["body_bytes"], + } + hh = record["acs_household_archive"] + hh["derived"] = { + "old_charge_occupied_over_64MiB": hh["old_charge_bytes_occupied"] + / auth.MAX_BODY_BYTES, + "np_total_equals_person_records": hh["np_total"] == people["records"], + "serialno_list_over_1MiB": hh["serialno_list_bytes_full_source"] / MIB, + "serialno_list_over_2MiB": hh["serialno_list_bytes_full_source"] / (2 * MIB), + "serialnos_the_1MiB_cap_admits": (MIB - 1) // 16, + } + out.write_text(json.dumps(record, indent=1) + "\n") + print( + json.dumps({"people": people["derived"], "households": hh["derived"]}, indent=1) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/native-byte-transports/census-brief.md b/experiments/native-byte-transports/census-brief.md new file mode 100644 index 000000000..1723e4a6a --- /dev/null +++ b/experiments/native-byte-transports/census-brief.md @@ -0,0 +1,156 @@ +# Byte-transport census: brief for every census and verification agent + +Read this whole file before reading any module. Every line number you cite is +this tree's, at the head you read it. You are read-only: do not edit, format, +or create any file under `packages/`, `docs/` or `tools/`; write nothing +outside the scratchpad path you are given. Do not run pytest. Do not read +gated data; the only data inputs are the two files named in §4. + +## 1. What a row is + +One row per **byte bound**: a module-level constant or inline literal that +bounds a byte count (`MAX_*_BYTES`, `*_MAX_BYTES`, `_MAX_*`, a `_bounded_json` +/ `_encode` / `_json` limit argument, a `len(payload) <= N` check, a +`read(N + 1)` read cap, a struct width times a row ceiling, a JSON-token or +record cap). A bound that is defined in one module and enforced in another is +one row, keyed by its definition, listing every enforcement site. + +Each row must carry, from the code you read at this head: + +- `constant`: `module.py:line NAME` (or `module.py:line ` for a literal). +- `value`: the expression and its integer value. +- `enforced_at`: every site `module.py:line`, with the refusal code literal and + exception type raised there. +- `encodes`: what document the bound sits in front of. Say precisely: a + per-row roster (which roster: ACS households, ACS persons, ASEC households, + ASEC persons, stacked households, stacked persons, clone households, clone + persons, selected serialnos, eligible children, tax units, groups), a + fixed-shape receipt, a catalogue, an evidence list, a header, a numpy body, + an upstream file, a single scalar/token. +- `shape`: `roster` (grows with the selection fraction), `fixed` (does not), + `upstream` (an input file's own size, identical at every fraction), + `width` (a fixed-width field or encoder width). +- `site_kind`: `materialises` (the bytes exist whole before or after the check) + or `streams` (a digest or a token-at-a-time accumulation; no whole copy). +- `bytes_at_tenth`, `bytes_at_full_source`: for a `roster` row, the bytes the + document meets at 1/10 and at full source **through the module's own + encoder** at the counts in §3. Cite `byte-census-measurements.json` when it + already measured the document; otherwise measure it yourself with the venv + python (`.venv/bin/python`) on invented rows at full-source id widths, as a + difference between two row counts so fixed preambles cancel, and put the + script and its receipt in your scratchpad. For a `fixed`/`upstream`/`width` + row say why the bytes do not move with the fraction and give the one number + you have (the recovered 1/1000 artifact's, or the input file's). +- `binds_at_tenth`, `binds_at_full_source`: `true` only if the measured bytes + exceed the value **at the value the base branch shipped**; record the base + value and this head's value separately when they differ (`base_value`, + `head_value`), because several bounds were lifted on this branch already. +- `reachable_from`: `19` (the nineteen-node financial graph), `45` (the + forty-five-node pilot graph), `age` (`survey_age_calibration`), or `none`, + with the call chain in one line (runner → function → module). §2 gives the + rosters. A bound in a module the runners import but whose enforcing function + no runner reaches is `none`, and you say which function would reach it. +- `classification`, one of: + - `segment`: a whole-roster canonical stream that meets its bound; takes the + transport lane's segmented shape (bytes unchanged, accumulation ceiling + unchanged, explicit total). + - `consumer`: a consumer-side cap on bytes a segmented producer issues; takes + the producer's total. + - `resource`: a single body that exists whole before the bound (a numpy + buffer) and meets its bound; becomes a fixed multiple of full source. + - `stays`: a roster-shaped document that does not meet its bound at full + source (give the headroom multiple). + - `must-not-move`: protects an encoder width, an upstream file's real size, + a fixed-width field, or a token/scalar. + - `not-a-transport`: a row count wearing a byte name (say which row-count + rule applies), or a memory ceiling with no document behind it. +- `notes`: anything a reviewer needs; keep it to what the code says. + +## 2. Reachability, from the code + +**The nineteen-node financial graph** (`_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/graph.json`, run by `graph_atomic_survey_financial.run_atomic_survey_financial` with no property, tax, status or roles): + +| node | kernel | +|---|---| +| `survey_population.create`, `.allocate`, `.observed_geography` | `graph_survey_population`, `graph_atomic_survey_population`, `survey_population_preparation` (source admission), `acs_native_coverage_binding`, `acs_person_coverage_authentication`, `acs_housing_universe_source`, `acs_population_catalogue`, `asec_2024_native_population`, `asec_population_catalogue`, `survey_catalogue_selection`, `survey_population_domains`, `survey_observed_age`, `native_household_origin` | +| `combined_survey_puf_support_clone`, `.owned` | `graph_combined_clone` | +| `geography.support.0`, `.assign`, `.derive`, `.gate` | `survey_atomic_geography`, `graph_geography`, `microcosm.graph.codecs` (`raw-bytes-v1`) | +| `survey_predictors.source_projection`, `.asec_design_donor`, `.asec_current_columns`, `.fit.000-002`, `.apply.000-002`, `.attach` | `graph_current_survey_predictors`, `current_survey_predictors`, `asec_current_money*`, `_asec_current_money_codec`, `microcosm.fit.model_input`, `microcosm.fit.qrf`, `microcosm.fit.graph_qrf*` | + +After the graph returns, the runner replays and verifies: `survey_population_replay`, `graph_atomic_survey_financial` verification, `current_survey_predictors.verify_materialized_current_survey_predictors`. + +**The forty-five-node pilot graph** (`_recovered/pilot-runs/native45-v5/`, run through `graph_survey_completion_host` with `property_income`, `rebase_property_taxes=True`, `person_status=False`, `household_roles=False`): the nineteen above, plus the property-income nodes of `graph_current_survey_property.current_survey_property_nodes` (`graph_property_income`, `graph_property_income_receipts`, `current_property_income_sources`, `current_asec_property_basis`, `property_income_constants`), the three property-tax nodes of `graph_property_tax_leaves`, and the ten completion nodes: `survey_completion.receiving` (`graph_survey_completion`), the six `child_property.*` nodes (`graph_child_property_income`, `current_child_property_income_source`, `current_property_completion_routing`, `microcosm.fit.graph_joint_empirical`, `microcosm.fit.joint_empirical`), and the tax trio again over the completed population. Household roles (`current_survey_household_roles`, `graph_current_survey_household_roles`) and person status (`graph_current_survey_person_status`) are reachable only from the 49/51-node variants: record them, mark `reachable_from: none` with that note. + +**`survey_age_calibration`** is neither graph. It runs `survey_origin_budget.freeze_survey_origin_budget` and `.checked_view`, `graph_survey_budget`, `graph_survey_calibration`, `graph_survey_age_artifact`, `survey_financial_successor`, `demographic_calibration_graph`, `graph_national_age_counts`, `survey_calibration_diagnostics`. The brief that commissioned this census names the origin budget explicitly, so these are in scope under `reachable_from: age`. + +**Not on any of these paths** (record with `reachable_from: none` if you meet them; do not spend measurement effort): `graph_survey_puf55`, `puf55_survey_recipients`, `graph_current_survey_puf_transfer`, `puf_diagnostic_consumer.current_survey_recipient_matrix`, `current_survey_amounts`, `graph_us_survey_enrichment`, `graph_fiscal_*`, `graph_acs_housing_universe` (the housing *graph*; the housing *source* `acs_housing_universe_source` is on the path through `acs_native_coverage_binding`). + +## 3. The counts (measured, not extrapolated) + +From the recovered 1/1000 artifact's catalogues, which are counts of the whole +upstream files and so are full-source counts (`experiments/native-row-ceilings/roster-census.json`): + +| quantity | full source | at 1/10 | at 1/15 | +|---|---:|---:|---:| +| ACS households (selectable) | 1,531,614 | 153,161 | 102,107 | +| ACS persons | 3,422,888 | 342,288 | 228,192 | +| ASEC households | 55,762 | 5,576 | 3,717 | +| ASEC persons | 142,125 | 14,212 | 9,475 | +| stacked households (= selected = groups) | 1,587,376 | 158,737 | 105,825 | +| stacked persons | 3,565,013 | 356,501 | 237,667 | +| clone households | 3,174,752 | 317,475 | 211,650 | +| clone persons | 7,130,026 | 713,002 | 475,335 | + +Entities other than person and household (tax units, families, marital units, +SPM units) are roster ratios from the 1/1000 artifact (1,584 households carry +2,128 tax units, 1,594 families, 2,774 marital units, 1,585 SPM units); say +`SCALED` when you use one. + +## 4. Inputs you may read + +- `experiments/native-byte-transports/byte-census-measurements.json`: this + lane's measurements through each module's own encoder (origin budget, + numeric bounds, age artifact, roles projection, predictor projection, + diagnostic matrix, child draw, selection block, and the whole captured ACS + archive: coverage body, pre-allocation charge, key list, serialno list). +- `experiments/native-row-ceilings/census.json`: the row-count lane's census + (146 bounds). Its byte rows are a starting list to **verify at this head**, + not a source of truth: several of its values and verdicts predate lifts on + this branch, and it missed modules (the property family, the housing source's + prepared receipt, consumer-side re-encodings). +- `_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json` + (the 1/1000 preparation receipt, 1,136,063 bytes) for row shapes. +- `docs/us-native-scale-transport.md` §2 and `docs/us-native-row-ceilings.md` + for the two neighbouring arguments. + +## 5. What has already moved on this branch (record base and head values) + +- `acs_person_coverage_authentication`: `MAX_ROSTER_BYTES = 64 * MAX_BODY_BYTES` + added; the coverage body, the issuance receipt and the key-list digest are + segmented/streamed; `SELECTED_BODY_BUDGET` charges the NDJSON line bound. +- `acs_native_coverage_binding`: the serialno pre-check sizes without encoding + under `coverage.MAX_ROSTER_BYTES`; the issuance receipt is `_json_roster`. +- `graph_survey_population`: `_canonical_pieces`, `_segmented_json`, + `_json_sha256`, `_json_matches`; `PREPARATION_ROSTER_BYTES`. +- `survey_origin_budget`: `MAX_ROSTER_BYTES`; the budget document is + segmented; `selection_sha256` streamed; `checked_view` streams its + comparison. +- `graph_survey_budget`, `graph_survey_calibration` (`MAX_ROSTER_BYTES`, + `MAX_ROWS`), `graph_survey_age_artifact` (`MAX_BYTES` 2 GiB), + `current_survey_predictors` (segmented projection), + `graph_child_property_income` (segmented `_json`, `MAX_ROSTER_BYTES`, + `MAX_ORIGINALS`), `current_child_property_income_source` + (`MAX_RECIPIENT_ROWS`), `microcosm.fit.graph_joint_empirical` + (`MAX_DRAW_BYTES`). + +Use `git diff origin/native-row-ceilings -- ` to see exactly what moved +in a module; the base value is the `-` side. + +## 6. Verification agents + +You receive one row and must try to refute it by re-reading the code: the +value, every enforcement site, the encoded document's shape, whether the site +streams or materialises, the reachability chain, the byte figures (recompute +one), and the classification. Return `agrees: true/false`, the exact +correction if false, and the line-cited evidence either way. Default to +`agrees: false` when you cannot verify a claim from the code. diff --git a/experiments/native-byte-transports/census-raw.json b/experiments/native-byte-transports/census-raw.json new file mode 100644 index 000000000..1dd5fac1d --- /dev/null +++ b/experiments/native-byte-transports/census-raw.json @@ -0,0 +1,8047 @@ +{ + "scope": "Raw census rows from the nine family agents at head 475c9ba9d (verification pending). Not a build, not a certification, not release eligible.", + "release_eligible": false, + "families": [ + { + "family": "Predictors and the fit package", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py (full, 644 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py (full, 616)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_diagnostic_consumer.py (full, 763)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py (full, 847)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_detail_transfer.py (full, 452)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_detail_transfer.py (full, 780)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py (full, 245)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py (full, 343)", + "packages/microcosm-fit/src/microcosm/fit/model_input.py (full, 140)", + "packages/microcosm-fit/src/microcosm/fit/qrf.py (full, 1826)", + "packages/microcosm-fit/src/microcosm/fit/_graph_legacy_qrf.py (full, 197)", + "packages/microcosm-fit/src/microcosm/fit/graph_legacy_qrf.py (full, 186)", + "packages/microcosm-fit/src/microcosm/fit/graph_legacy_apply.py (full, 40)", + "packages/microcosm-fit/src/microcosm/fit/graph_legacy_apply_matrix.py (full, 107)", + "packages/microcosm-fit/src/microcosm/fit/graph_legacy_train.py (full, 147)", + "packages/microcosm-fit/src/microcosm/fit/_graph_legacy_apply.py (full, 139)", + "no packages/microcosm-fit/src/microcosm/fit/graph_qrf*.py exists at this head (the fit graph modules are the graph_legacy_* files above)", + "packages/microcosm-fit/src/microcosm/fit/qrf_target.py (lines 240-340: to_bytes / from_trusted_bytes; not in the family list, read because graph_legacy_train/apply call it)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py (lines 100-120, 255-380, 405-500: _require, _bounded_json, _canonical_pieces, _segmented_json, _json_sha256, _json_matches, _instruction_document, _allocation_payload)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py (lines 40-70: MAX_PAYLOAD_BYTES, MAX_SEGMENT_BYTES, MAX_ROSTER_BYTES; :78-85 error class)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py (lines 180-380: qualify_current_asec_demographics, the producer of the projections row 7 re-encodes)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py (lines 36-120: health_coverage_seal consumer re-encode)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py (lines 95-130, 840-910, 1280-1470, 1830-1990: reachability)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py (grep: _check_asec_projection :208-215)", + "packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py (lines 230-245: MONEY_FIELDS roster)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_domains.py (lines 27, 52-56: Source, HouseholdKey)", + "packages/microcosm-graph/src/microcosm/graph/store.py (lines 1050-1095: put_bytes/load_bytes have no size cap)", + "experiments/native-byte-transports/census-brief.md (full)", + "experiments/native-byte-transports/byte_census_measure.py (full)", + "experiments/native-byte-transports/byte-census-measurements.json (full)", + "experiments/native-row-ceilings/census.json (this family's rows and the shared-constant rows)", + "experiments/native-scale-transport/harness19_base.py (lines 25-60, 525-560: pilot arguments)", + "docs/us-native-scale-transport.md (section 2d)", + "git diff origin/native-row-ceilings -- current_survey_predictors.py, graph_joint_empirical.py" + ], + "rows": [ + { + "constant": "survey_population_preparation.py:50 MAX_PAYLOAD_BYTES (segment) and :58 MAX_ROSTER_BYTES (maximum), as passed at current_survey_predictors.py:522-523 for the predictor projection", + "value": "segment = 64 * 1024**2 = 67,108,864; maximum = 64 * MAX_SEGMENT_BYTES (:57, = MAX_PAYLOAD_BYTES) = 4,294,967,296", + "enforced_at": [ + "current_survey_predictors.py:512-524 (host.survey_graph._segmented_json(document, segment=source.MAX_PAYLOAD_BYTES, maximum=source.MAX_ROSTER_BYTES))", + "graph_survey_population.py:294 (segment <= 64 MiB) \"TRANSPORT_LIMIT\" SurveyPopulationGraphError(ValueError) (:116, raised by _require :120)", + "graph_survey_population.py:303 and :305 (one token > segment) \"TRANSPORT_LIMIT\" SurveyPopulationGraphError", + "graph_survey_population.py:325 (maximum type/positive) \"TRANSPORT_ROSTER_LIMIT\" SurveyPopulationGraphError", + "graph_survey_population.py:331 (segment accumulation) \"TRANSPORT_LIMIT\" SurveyPopulationGraphError", + "graph_survey_population.py:332 and :337 (running total and joined payload <= maximum) \"TRANSPORT_ROSTER_LIMIT\" SurveyPopulationGraphError", + "graph_survey_population.py:288 / :310 (encoder failure) \"TRANSPORT_ENCODING\" SurveyPopulationGraphError" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT", + "TRANSPORT_ROSTER_LIMIT", + "TRANSPORT_ENCODING" + ], + "encodes": "The predictor source projection: the fixed evidence dict (protocol, digests, money header, split contract, model judgments, optional atomic_geography receipt) plus \"origins\": one [person_id, native_person_id, source] row per stacked person (current_survey_predictors.py:515-517), plus money_fields[*].status_hex/validity_hex/zero_origin_hex, three hex strings of one byte per selected ASEC person for each of the five MONEY_FIELDS (:400-406).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "8,996,644 roster part (356,501 stacked-person origin rows x 24.04 B = 8,570,284; 14,212 ASEC persons x 30 B hex = 426,360) plus the unmeasured fixed evidence preamble", + "bytes_at_full_source": "89,966,662 roster part (3,565,013 x 24.04 B = 85,702,912 origins; 142,125 x 30 B = 4,263,750 hex evidence) plus the fixed preamble", + "measurement_basis": "Measured at this head through graph_survey_population._segmented_json with the module's exact segment/maximum (scratchpad measure_pred.py -> measurements.json: per_stacked_person_origin_row_bytes 24.04, per_asec_person_hex_evidence_bytes 30.0, difference of 1001 vs 1 rows, ids at 7-digit full-source width). The 85,702,912 origins figure matches byte-census-measurements.json predictor_projection.full_source_origins_bytes, which was taken through _bounded_json at the base; segmented bytes equal bounded bytes (measured: segmented_equals_bounded_bytes true). The JSON omitted the per-ASEC-person hex evidence, added here.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "_bounded_json(document, source.MAX_PAYLOAD_BYTES): one accumulation of 67,108,864 (git diff origin/native-row-ceilings: '- projection = host.survey_graph._bounded_json(' / '- source.MAX_PAYLOAD_BYTES,')", + "head_value": "_segmented_json(document, segment=67,108,864, maximum=4,294,967,296) (current_survey_predictors.py:512-524)", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1389) -> current_survey_predictors.qualify_current_survey_predictors -> host.survey_graph._segmented_json (:512); re-derived by every predictor kernel through _Kernel._qualified (graph_current_survey_predictors.py:301, nodes survey_predictors.source_projection/.asec_design_donor/.asec_current_columns/.attach) and by verify_materialized_current_survey_predictors (:578; runner :1841 and :854). The 45-node run reaches it through the same base call (graph_atomic_survey_financial.py:1319-1331). Consumers hold the whole payload: graph store put_bytes (store.py:1057-1074, no cap), projection.payload == result.projection (graph_current_survey_predictors.py:359), projection == qualified.projection (:586), codec.sha(qualified.projection) (:138).", + "classification": "segment", + "notes": "Already took the segmented shape on this branch (brief section 5). At the base value 89,966,662 > 67,108,864 so a full-source run refused with TRANSPORT_LIMIT; a single 64 MiB accumulation admits 2,791,550 stacked persons, i.e. about 78% of the source. At the head the total is 2.1% of the 4 GiB ceiling. The producer's comment at :508-511 states the 85,702,912 figure. Off-path sibling: current_survey_amounts.py:369 still passes predictors.source.MAX_PAYLOAD_BYTES to _bounded_json for its own (digest-only, fixed-shape) projection." + }, + { + "constant": "model_input.py:18 MATRIX_HEADER_MAX_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "model_input.py:86-87 (encode: len(header) > MATRIX_HEADER_MAX_BYTES) ValueError(\"Recipient matrix header exceeds its size bound.\")", + "model_input.py:105-106 (decode: length == 0 or length > MATRIX_HEADER_MAX_BYTES or len(payload) < start + length) ValueError(\"Recipient matrix header is truncated or exceeds its bound.\")" + ], + "refusal_codes": [ + "Recipient matrix header exceeds its size bound.", + "Recipient matrix header is truncated or exceeds its bound." + ], + "encodes": "The JSON header of the recipient-matrix envelope (schema_version, entity, rows, columns, index_name, index_identity = a fixed five-field digest of the id index), behind a 4-byte big-endian length field (:90, :103). The body that follows (entity_ids int64 + float64 features, 8 x (1 + n_features) bytes per row) has no cap.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "header 462 B including the 40-byte magic and 4-byte length (measured at 1 row; grows only by the digits of \"rows\"); the uncapped body is 16,432,339 B for 342,288 ACS persons x 48 B (five features under demographic_conditioning=True)", + "bytes_at_full_source": "header ~462 B; uncapped body 164,319,623 B (3,422,888 ACS persons x 48.006 B measured; byte-census-measurements.json predictor_projection.matrix.full_source_bytes = 164,298,624 body only)", + "measurement_basis": "Measured through model_input.encode_recipient_matrix at this head (measure_pred.py recipient_matrix: bytes_per_acs_person 48.006, header_plus_magic_bytes_at_1_row 462); cited byte-census-measurements.json for the body.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "19", + "reach_chain": "qualify_current_survey_predictors -> model_input.encode_recipient_matrix (current_survey_predictors.py:505); decode at complete_predictor_columns (:590), read_current_survey_draws (graph_current_survey_predictors.py:469), LegacyQRFApplyMatrixKernel.run (graph_legacy_apply_matrix.py:76, nodes survey_predictors.apply.000-002), runner graph_atomic_survey_financial.py:1263. The 45-node run also reaches the codec via current_property_income_sources.py:213/239.", + "classification": "must-not-move", + "notes": "Protects a fixed-shape header behind a fixed-width length field; the header is row-count independent apart from the digits of \"rows\". The matrix BODY has no byte bound anywhere on the path: the encoder caps only the header, store.put_bytes stores whole with no size check, and every _Kernel._qualified plus the verifier re-encodes it (164 MB at full source). Off path, puf_diagnostic_consumer.py:398-406 encodes a 207,692-row PUF donor matrix (~10 MB) through the same codec and puf_detail_transfer.py:452 the diagnostic matrix (row 8 arm b)." + }, + { + "constant": "model_input.py:127 len(body) != rows * 8 + rows * len(columns) * 8", + "value": "exact body width: 8 x (1 + n_columns) bytes per row (48 with the five demographic-conditioning features, 32 with three); an equality, not a ceiling", + "enforced_at": [ + "model_input.py:127-128 ValueError(\"Recipient matrix body length differs from its dimensions.\")" + ], + "refusal_codes": [ + "Recipient matrix body length differs from its dimensions." + ], + "encodes": "The recipient-matrix body: int64 entity ids followed by little-endian float64 features, one row per recipient (ACS persons for the predictor chain).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "16,429,824 body bytes at 342,288 rows x 48 (a memoryview over the loaded payload, np.frombuffer views; no second copy)", + "bytes_at_full_source": "164,298,624 body bytes at 3,422,888 rows x 48", + "measurement_basis": "Width read from model_input.py:88-94 (encoder) and :126-135 (decoder); counts from the brief section 3; product also in byte-census-measurements.json predictor_projection.matrix.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "same expression", + "head_value": "same expression (unchanged)", + "reachable_from": "19", + "reach_chain": "Every decode_recipient_matrix site in the previous row (complete_predictor_columns, read_current_survey_draws, LegacyQRFApplyMatrixKernel.run, runner :1263).", + "classification": "must-not-move", + "notes": "A struct-width identity the decoder needs to slice ids from features (:129, :135); it cannot bind on row count because rows is read from the header and the body is exactly rows wide by construction." + }, + { + "constant": "_graph_legacy_qrf.py:191 len(body) != len(index) * 8 (read_raw_target)", + "value": "exact raw-draw body width: 8 bytes (float64) per recipient row; an equality, not a ceiling. encode_raw_target (:170-175) has no cap.", + "enforced_at": [ + "_graph_legacy_qrf.py:188-193 ValueError(\"Raw legacy target/index/order binding changed.\") (also refuses a missing newline separator or a metadata mismatch)", + "_graph_legacy_qrf.py:172-173 (encoder: values.shape != (len(index),)) ValueError(\"Raw legacy target must be aligned finite float64 draws.\")" + ], + "refusal_codes": [ + "Raw legacy target/index/order binding changed.", + "Raw legacy target must be aligned finite float64 draws." + ], + "encodes": "One raw target draw artifact per apply node: _RAW_MAGIC + canonical JSON {target, index identity} + newline + float64 draws, one per recipient row (ACS persons for survey_predictors.apply.000-002).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "2,738,304 body bytes per raw_draw artifact (342,288 x 8) plus ~250 B metadata; three artifacts on the predictor chain", + "bytes_at_full_source": "27,383,104 body bytes per raw_draw artifact (3,422,888 x 8); three artifacts", + "measurement_basis": "Width read from _graph_legacy_qrf.py:170-175 and :186-194; counts from the brief section 3 (measure_pred.py raw_target restates the product).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "same expression", + "head_value": "same expression (unchanged)", + "reachable_from": "19", + "reach_chain": "LegacyQRFApplyMatrixKernel.run (graph_legacy_apply_matrix.py:95) -> _graph_legacy_apply.run_application -> codec.encode_raw_target (:115) and codec.read_raw_target for each prior (:111-113); CurrentSurveyPredictorAttachKernel.run -> read_current_survey_draws -> codec.read_raw_target (graph_current_survey_predictors.py:508-510); verifier (:589). Also 45 via graph_current_survey_property.py:493 and graph_property_income.py:320 (property family).", + "classification": "must-not-move", + "notes": "payload[len(_RAW_MAGIC):].partition(b\"\\n\") (:186) copies metadata and body once; np.frombuffer then views. No ceiling exists on this artifact anywhere on the path (store.put_bytes uncapped)." + }, + { + "constant": "_graph_legacy_qrf.py:45 encode_json(value) != payload (decode_json canonical re-encode)", + "value": "none (a canonical-form identity: json.loads then json.dumps(sort_keys, separators, allow_nan=False, ensure_ascii default) must reproduce the payload byte for byte)", + "enforced_at": [ + "_graph_legacy_qrf.py:45-46 ValueError(\"Legacy graph checkpoint must be a canonical JSON object.\") (also :36-37 \"Repeated legacy graph checkpoint JSON key.\", :41-42 \"Nonfinite legacy graph checkpoint JSON number.\")", + "callers on path: _graph_legacy_qrf.py:142 read_training, :155 read_application; graph_legacy_apply_matrix.py:28 decode_matrix_apply_state; graph_current_survey_predictors.py:251 (host_pins round trip), :391 (allocation, see its own row), :483-485 read_application(encode_json(packet[\"application\"])); model_input.py:107 (matrix header)" + ], + "refusal_codes": [ + "Legacy graph checkpoint must be a canonical JSON object.", + "Repeated legacy graph checkpoint JSON key.", + "Nonfinite legacy graph checkpoint JSON number." + ], + "encodes": "Every JSON packet the fit graph decodes: training_state and apply_state checkpoints (fixed-shape: a QRFChainState dict whose donor/recipient indices are five-field digests, qrf.py:592-600, plus three model/raw-target history entries), the recipient-matrix header, the host_pins dict. The one roster-shaped document that passes through it on the path is the allocation payload (next row).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "a few KB per packet (fixed-shape); unchanged by the fraction", + "bytes_at_full_source": "a few KB per packet; unchanged", + "measurement_basis": "Read from qrf.py:767-791 (QRFChainState.to_dict carries no row values) and _graph_legacy_qrf.py:123-167; not measured because no field scales with rows.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "same expression", + "head_value": "same expression (unchanged)", + "reachable_from": "19", + "reach_chain": "LegacyQRFTrainKernel.run -> codec.read_training (graph_legacy_train.py:96); run_application -> codec.read_training (:51) and state_decoder (:89-91); LegacyQRFApplyMatrixKernel.read_state -> decode_matrix_apply_state (graph_legacy_apply_matrix.py:83); CurrentSurveyPredictorAttachKernel.run -> read_current_survey_draws (:477, :483-485).", + "classification": "must-not-move", + "notes": "Protects the canonical-form contract every digest in the chain (matrix_sha256, model sha256, raw sha256 histories) depends on; it is not a size ceiling. The whole-document re-encode is harmless for these fixed packets; its one roster-scale use is recorded separately as the allocation consumer row." + }, + { + "constant": "graph_current_survey_predictors.py:391 codec.decode_json(loaded[\"allocation\"]) (consumer-side re-encoding of graph_survey_population's allocation payload)", + "value": "consumer cap: none. Producer caps: graph_survey_population.py:62 ALLOCATION_MAX_BYTES = 67,108,864 per segment, :68 ALLOCATION_ROSTER_BYTES = 64 * ALLOCATION_MAX_BYTES = 4,294,967,296 total", + "enforced_at": [ + "graph_current_survey_predictors.py:386-393 (CurrentSurveyPredictorProjectionKernel.run: codec.sha(loaded[\"frame_context\"]) == codec.decode_json(loaded[\"allocation\"])[\"output_context_sha256\"]) \"HOST_EDGE_BYTES\" ValueError(\"CURRENT_SURVEY_PREDICTOR_HOST_EDGE_BYTES\") on mismatch; the canonical check itself raises at _graph_legacy_qrf.py:45-46", + "off path: graph_puf_diagnostic_consumer.py:612-618 (same decode of allocated.payload) \"SURVEY_HOST_EDGE_BYTES\"", + "producer sites (other family): graph_survey_population.py:481-497 \"ALLOCATION_LIMIT\" / \"ALLOCATION_ROSTER_LIMIT\" SurveyPopulationGraphError" + ], + "refusal_codes": [ + "Legacy graph checkpoint must be a canonical JSON object.", + "HOST_EDGE_BYTES" + ], + "encodes": "The allocation artifact: {\"households\": [one instruction document per selected household: household_id, source, source_year, survey_year, raw_native_id, selected_receiving_household_id, four Fraction pairs, a float64 hex], ...six metadata keys}, streamed by the producer in 64 MiB segments and joined once (graph_survey_population.py:459-497).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "49,684,681 (158,737 selected households x 313.0 B) held three ways at once: the loaded payload, the json.loads tree, and the full re-encoded copy encode_json builds for the comparison", + "bytes_at_full_source": "496,848,688 (1,587,376 x 313.0 B), likewise held as payload + tree + re-encoded copy", + "measurement_basis": "Measured at this head through graph_survey_population._allocation_payload (its own _bounded_json(_instruction_document(row), 4096) per row) on invented rows at full-source id widths, difference of 1001 vs 1 rows (scratchpad measure_allocation.py -> allocation-measurement.json: per_selected_household_bytes 313.0; codec_decode_json_roundtrip_equal true). docs/us-native-scale-transport.md section 2d and the row-ceilings census cite 316.665 B per instruction at 1/1000 widths.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (consumer); producer 67,108,864 per segment / 4,294,967,296 total (the producer was segmented on native-scale-transport, docs section 2d; this consumer is unchanged on this branch)", + "head_value": "none (consumer); producer unchanged", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial -> node survey_predictors.source_projection -> CurrentSurveyPredictorProjectionKernel.run (graph_current_survey_predictors.py:369-397) -> host.shared.artifact(context, \"allocation\", ALLOCATION_TYPE) (:374) -> codec.decode_json(loaded[\"allocation\"]) (:391).", + "classification": "consumer", + "notes": "The decode exists only to read output_context_sha256; decode_json re-encodes the entire ~497 MB tree to prove canonical form, so the consumer accumulates the producer's whole roster in one bytes object (json.dumps builds it whole) with no ceiling of its own; the producer's total is 4 GiB and the full-source document is 11.6% of it. The streaming comparison the lane already uses for this shape is graph_survey_population._json_matches (:356-374; used at survey_origin_budget.py:935). Line 388 also hashes the whole loaded payload (codec.sha) but that streams over an already-materialised artifact." + }, + { + "constant": "current_survey_predictors.py:71 codec.sha(table.reset_index(drop=True).to_json(orient=\"table\").encode()) == document[f\"{entity}_projection_sha256\"] (consumer-side re-encoding of current_asec_demographics' person and household projections)", + "value": "none on either side: the producer (current_asec_demographics.py:359-364) hashes the same pandas to_json text with no cap", + "enforced_at": [ + "current_survey_predictors.py:59-74 _check_demographic_values, loop :68-74, \"DEMOGRAPHIC_PROJECTION_CHANGED\" ValueError(\"CURRENT_SURVEY_PREDICTOR_DEMOGRAPHIC_PROJECTION_CHANGED\"); called at :89 and :170 (_demographic_features) and :573 (final seal), i.e. three re-encodes of both tables per qualify", + "sibling site in another family: current_survey_geography.py:208-215 _check_asec_projection (household table only)" + ], + "refusal_codes": [ + "DEMOGRAPHIC_PROJECTION_CHANGED", + "DEMOGRAPHIC_RECEIPT" + ], + "encodes": "values.person: one row per selected ASEC person (person_id, native_person_id, asec_A_SEX, asec_AXSEX, asec_sex_binding_state, asec_sex_allocation_state, is_female, sex_known, sex_universe, sex_origin; current_asec_demographics.py:311-343) and values.household: one row per selected ASEC household (household_id, native_household_id, H_SEQ_integer, GESTFIPS, state_fips, state_status, state_known; :283-299), each rendered as pandas orient=\"table\" JSON text.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "persons 3,451,995 (14,212 x 242.9 B); households 919,443 (5,576 x 164.9 B)", + "bytes_at_full_source": "persons 34,521,167 (142,125 x 242.9 B); households 9,194,763 (55,762 x 164.9 B)", + "measurement_basis": "Measured at this head through the same expression on invented tables with the producer's exact columns and dtypes, difference of 1001 vs 1 rows at full-source id widths (measure_pred.py asec_demographic_projection_reencode: per_asec_person_bytes 242.893, per_asec_household_bytes 164.893).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none", + "head_value": "none (unchanged)", + "reachable_from": "19", + "reach_chain": "qualify_current_survey_predictors -> _demographic_features (:358-360, only when demographic_conditioning=True, which the pilot sets: experiments/native-scale-transport/harness19_base.py:532 demographic_conditioning=True, n_estimators=2) -> observed_geography.demographics.qualify_current_asec_demographics (:85) -> _check_demographic_values (:89, :170) and again at :573.", + "classification": "stays", + "notes": "No bound exists to meet; recorded because the brief counts consumer-side re-encodings of another module's document. values.person.index must equal the selected ASEC person set (:114-122), so the roster grows with the fraction until the whole ASEC file is selected and then plateaus at the upstream count (142,125 persons). Against the 64 MiB single-accumulation ceiling the neighbouring transports use, the person text is 1.94x under and the household text 7.3x under." + }, + { + "constant": "puf_diagnostic_consumer.py:671 CURRENT_SURVEY_MAX_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "puf_diagnostic_consumer.py:682-685 (0 < len(projection) <= CURRENT_SURVEY_MAX_BYTES) \"SURVEY_HOST_PROJECTION_BOUND\" bare ValueError (require = detail.require, :27; puf_detail_transfer.py:79-82 raises ValueError(reason))", + "puf_diagnostic_consumer.py:833 (len(matrix) <= CURRENT_SURVEY_MAX_BYTES) \"SURVEY_MATRIX_BOUND\" bare ValueError", + "graph_puf_diagnostic_consumer.py:403 (survey_graph._bounded_json(document, consumer.CURRENT_SURVEY_MAX_BYTES)) \"TRANSPORT_LIMIT\" SurveyPopulationGraphError at graph_survey_population.py:272 (limit <= 64 MiB), :281, :283", + "graph_puf_diagnostic_consumer.py:648-652 (kernel re-check of the projection payload) \"SURVEY_HOST_PROJECTION_BOUND\" bare ValueError" + ], + "refusal_codes": [ + "SURVEY_HOST_PROJECTION_BOUND", + "SURVEY_MATRIX_BOUND", + "TRANSPORT_LIMIT" + ], + "encodes": "Two documents. (a) The current-survey host projection: fixed digests plus asec.rows, one 7-element row [stacked_person_id, native_person_id, 2024, 16-hex amount, status, validity, zero_origin] per native ASEC person (:733-742, :754-757). (b) The recipient matrix from puf_detail_transfer.recipient_matrix (:830-832): 48 B per clone-one tax unit (8-byte id + five float64 FEATURES, puf_detail_transfer.py:29, :448-452).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "(a) 682,176 (14,212 native ASEC persons x 48.0 B) plus fixed preamble; (b) 10,236,144 SCALED (213,253 clone-one tax units x 48)", + "bytes_at_full_source": "(a) 6,822,000 (142,125 x 48.0 B); (b) 102,361,680 SCALED (2,132,535 clone-one tax units x 48, ratio 1.343434 tax units per household from the 1/1000 artifact)", + "measurement_basis": "(a) measured at this head through graph._bounded_json(document, CURRENT_SURVEY_MAX_BYTES) on 7-element rows at full-source widths (measure_pred.py diagnostic_host_projection: per_native_asec_person_row_bytes 48.0). (b) cited byte-census-measurements.json diagnostic_matrix (SCALED from the 1/1000 tax-unit ratio); width 8 x (1 + 5) confirmed from puf_detail_transfer.FEATURES and model_input.py:88-94.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "67,108,864", + "head_value": "67,108,864 (unchanged on this branch)", + "reachable_from": "none", + "reach_chain": "Reached only by qualify_current_survey_host / CurrentSurveyHostProjectionKernel.run / CurrentSurveyMatrixKernel.run / current_survey_recipient_matrix, whose sole callers are graph_current_survey_puf_transfer.py:224 and :380 (brief section 2: not on any path). The module itself is on the 19-path only for survey_source/survey_budget/survey_graph aliases, current_survey_host_edges (:421), _current_context_frame (:454) and _CurrentSurveyKernel (:511), none of which enforce this constant.", + "classification": "resource", + "notes": "Arm (b) is a numpy body that exists whole before the check and meets the bound at full source (102,361,680 > 67,108,864; 64 MiB admits 1,398,101 clone-one tax units, about 66% of the source); the row-ceilings census verdict (binds at full source) is confirmed at this head, SCALED. Arm (a) is capped by the real ASEC file at 142,125 rows and never binds. The encode side cannot be lifted in place: _bounded_json refuses any limit above 64 MiB at graph_survey_population.py:272." + }, + { + "constant": "puf_detail_transfer.py:34 MAX_BYTES", + "value": "128 * 1024 * 1024 = 134,217,728", + "enforced_at": [ + "puf_detail_transfer.py:114-119 (isinstance(payload, bytes) and len(payload) <= MAX_BYTES and payload.startswith(MAGIC)) \"PRICE_ENVELOPE\" bare ValueError (require :79-82)", + "graph_puf_diagnostic_consumer.py:104-107 (pinned size: 0 < size <= detail.MAX_BYTES) \"PRICE_TRANSPORT_SIZE\" bare ValueError", + "graph_puf_diagnostic_consumer.py:108-109 (handle.read(size + 1) read cap on each raw-bytes-v1 source, then consumer.check_pin :110-112 \"PRICE_TRANSPORT_PIN:\")" + ], + "refusal_codes": [ + "PRICE_ENVELOPE", + "PRICE_TRANSPORT_SIZE", + "PRICE_TRANSPORT_PIN" + ], + "encodes": "The local PUF price-restatement arrays envelope: MAGIC (38 B) + 8-byte big-endian header length + JSON header (<= 65,536) + 22 column bodies at 155 B per ordinary PUF return (ARRAY_DTYPES :50-61: 13 x f8, RECID i8, provenance recid i8, source AGI f8, S006 f8, FLPDYR i4, demographic_status i4, S006_delivered i8, FLPDYR_delivered i2, demographic_status_delivered i1).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "~32,192,260 + header (207,692 ordinary PUF returns x 155 B); invariant to the survey fraction", + "bytes_at_full_source": "~32,192,260 + header (same)", + "measurement_basis": "Width summed from ARRAY_DTYPES at this head (155 B); the 207,692-row count is the row-ceilings census's PUF figure (207,696 delivered minus four aggregate RECIDs), not re-measured here.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "134,217,728", + "head_value": "134,217,728 (unchanged)", + "reachable_from": "none", + "reach_chain": "decode_price_arrays is called only by graph_puf_detail_transfer.py:290 (FixturePriceExportKernel.run), :339 (FixtureDonorKernel.run) and puf_diagnostic_consumer.py:351 (qualify_price_transport via DiagnosticDonorKernel.run, graph_puf_diagnostic_consumer.py:113); none of those kernels is declared by the 19-, 45- or age runners.", + "classification": "must-not-move", + "notes": "Protects an upstream file's real size (4.2x headroom); the operative row ceiling on the same envelope is the inline 250_000 at :113 and MAX_DONORS, both row counts. The read cap at graph_puf_diagnostic_consumer.py:109 is the only read(N + 1) in this family." + }, + { + "constant": "puf_detail_transfer.py:128 0 < size <= 65536 (PRICE_HEADER_BOUND)", + "value": "65,536 on the envelope's JSON header, whose length is written as an 8-byte big-endian field (graph_puf_detail_transfer.py:271, read at puf_detail_transfer.py:126)", + "enforced_at": [ + "puf_detail_transfer.py:128 (0 < size <= 65536 and len(payload) >= start + size) \"PRICE_HEADER_BOUND\" bare ValueError" + ], + "refusal_codes": [ + "PRICE_HEADER_BOUND" + ], + "encodes": "The envelope header: six identity fields plus 22 column descriptors {dtype, rows, sha256} (graph_puf_detail_transfer.py:264-268), about 2.5 KB.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "~2.5 KB (fixed: 22 descriptors x ~100 B + six header fields); invariant", + "bytes_at_full_source": "~2.5 KB (same)", + "measurement_basis": "Shape read from graph_puf_detail_transfer.py:256-268 and puf_detail_transfer.py:149-166; not measured (row-count independent apart from the digits of \"rows\").", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "none", + "reach_chain": "Same callers as the MAX_BYTES row (fixture and diagnostic donor kernels only).", + "classification": "must-not-move", + "notes": "Protects a fixed-width length field's admitted range; the header carries no per-row content." + }, + { + "constant": "puf_detail_transfer.py:169-170 end = cursor + rows * np.dtype(entry[\"dtype\"]).itemsize; require(end <= len(payload)) (PRICE_BODY_TRUNCATED)", + "value": "struct width x declared rows per column, with rows bounded by max_rows (:161-166, 0 < rows <= max_rows; :113 0 < max_rows <= 250_000; default MAX_ROWS = 4096 at :33): at most 250,000 x 155 B = 38,750,000 B of column bodies", + "enforced_at": [ + "puf_detail_transfer.py:169-170 \"PRICE_BODY_TRUNCATED\" bare ValueError", + "puf_detail_transfer.py:177 (cursor == len(payload)) \"PRICE_TRAILING_BYTES\" bare ValueError", + "puf_detail_transfer.py:161-166 \"PRICE_COLUMN_TYPE_ROWS\", :168 \"PRICE_ROW_ALIGNMENT\", :113 \"PRICE_ROW_BOUND\" (the row-count arms)" + ], + "refusal_codes": [ + "PRICE_BODY_TRUNCATED", + "PRICE_TRAILING_BYTES", + "PRICE_COLUMN_TYPE_ROWS", + "PRICE_ROW_ALIGNMENT", + "PRICE_ROW_BOUND" + ], + "encodes": "Each column body of the price-restatement envelope, sliced at rows x itemsize and viewed with np.frombuffer (:171-173).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "207,692 x 155 = 32,192,260 B across the 22 columns; invariant to the survey fraction", + "bytes_at_full_source": "32,192,260 B (same)", + "measurement_basis": "Width from ARRAY_DTYPES (:50-61); PUF row count from the row-ceilings census.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "same expression; max_rows domain 250,000; MAX_ROWS default 4,096", + "head_value": "unchanged", + "reachable_from": "none", + "reach_chain": "Same callers as the MAX_BYTES row.", + "classification": "must-not-move", + "notes": "A struct-width product whose row factor is a PUF (upstream) count: 207,692 rows sit under the 250,000 max_rows the qualified route passes (puf_diagnostic_consumer.py:355 max_rows=rows) but above the 4,096 default, so a direct caller using the default refuses with PRICE_COLUMN_TYPE_ROWS. MAX_ROWS (:33), the inline 250_000 (:113) and puf_diagnostic_consumer.py:32 MAX_DONORS (250_000, enforced :331-341 \"PRICE_ORDINARY_ROSTER\") are row counts, not byte transports (row-count rule: the PUF ordinary-return count is fixed at 207,692 by the delivery)." + }, + { + "constant": "puf_diagnostic_consumer.py:77 0 < len(payload) <= 16_000_000 (parse_document)", + "value": "16,000,000", + "enforced_at": [ + "puf_diagnostic_consumer.py:76-79 \"DIAGNOSTIC_DOCUMENT_BOUND\" bare ValueError; callers :125 (the six DEA receipts acceptance/proposal/config/completion/cold/required), :287 and :293 (definition), :289 and :298 (projection_definition)" + ], + "refusal_codes": [ + "DIAGNOSTIC_DOCUMENT_BOUND", + "DIAGNOSTIC_DUPLICATE_KEY", + "DIAGNOSTIC_NONFINITE_JSON", + "DIAGNOSTIC_DOCUMENT_TYPE" + ], + "encodes": "Eight upstream JSON documents supplied as raw-bytes-v1 sources with reviewed content pins (graph_puf_diagnostic_consumer.py:56-57, :99-112): six DEA helper receipts, the PUF raw-source definition and the AGI projection definition.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "in-tree packaged twins: puf_2015_raw_source_definition.json 9,707 B, puf_2015_monetary_agi_source_projection.json 285,396 B; the six DEA receipts are not in this tree (sizes unknown here); invariant to the survey fraction", + "bytes_at_full_source": "same (upstream files)", + "measurement_basis": "wc -c of the packaged resources under packages/microcosm-build/src/microcosm/build/us_runtime/ at this head; the DEA receipts are fixture/packaged inputs the brief's read-only rule keeps me from fetching.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "16,000,000", + "head_value": "16,000,000 (unchanged)", + "reachable_from": "none", + "reach_chain": "qualify_price_transport is called only from DiagnosticDonorKernel.run (graph_puf_diagnostic_consumer.py:113), declared only by diagnostic_nodes (:242-351), which no runner calls.", + "classification": "must-not-move", + "notes": "Protects upstream files' real sizes; the packaged definition and projection are 0.06% and 1.8% of the bound." + }, + { + "constant": "graph_puf_diagnostic_consumer.py:583 and :703 len(pins_bytes) <= 4096 / len(encoded_pins) <= 4096 (SURVEY_HOST_EDGE_PIN_BOUND)", + "value": "4,096", + "enforced_at": [ + "graph_puf_diagnostic_consumer.py:582-583 (CurrentSurveyHostProjectionKernel.run: context.params[\"host_edges\"].encode()) \"SURVEY_HOST_EDGE_PIN_BOUND\" bare ValueError", + "graph_puf_diagnostic_consumer.py:702-703 (current_survey_host_nodes: codec.encode_json(host_pins)) \"SURVEY_HOST_EDGE_PIN_BOUND\" bare ValueError" + ], + "refusal_codes": [ + "SURVEY_HOST_EDGE_PIN_BOUND", + "SURVEY_HOST_EDGE_PIN_ROSTER", + "SURVEY_HOST_EDGE_PIN" + ], + "encodes": "The three-edge host pin document (preparation, allocation, frame_context; :421-441), each pin {producer_key, artifact_key, payload_sha256} of 64-hex strings (:696-701).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "about 700 B (3 pins x ~230 B); invariant", + "bytes_at_full_source": "about 700 B (same)", + "measurement_basis": "Shape read from :696-701 (three 64-hex digests per pin, codec._hash) and :421-441 (three edges); not measured.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4,096", + "head_value": "4,096 (unchanged)", + "reachable_from": "none", + "reach_chain": "current_survey_host_nodes and CurrentSurveyHostProjectionKernel.run are called only from graph_current_survey_puf_transfer.py (brief section 2: not on any path). The on-path twin, graph_current_survey_predictors._params (:139 \"host_edges\": codec.encode_json(host_pins).decode()), carries the same pins with no byte bound.", + "classification": "must-not-move", + "notes": "A fixed-shape token document; a fourth edge (the geography validation pin the predictor kernels add, graph_current_survey_predictors.py:123-124) would still sit near 900 B." + }, + { + "constant": "current_survey_health_source.py:316 projection = raw.reset_index().to_json(orient=\"table\", index=False).encode() (no cap; consumer re-encode at graph_current_survey_health.py:66-68)", + "value": "none on either side", + "enforced_at": [ + "graph_current_survey_health.py:64-70 health_coverage_seal: qualified.projection == qualified.raw.reset_index().to_json(orient=\"table\", index=False).encode() and evidence[\"projection_sha256\"] == sha256(projection) \"PROJECTION_BINDING\" ValueError(\"CURRENT_SURVEY_HEALTH_PROJECTION_BINDING\") (require = health.require, current_survey_health_coverage.py:98-100)", + "current_survey_health_source.py:326 (projection_sha256 = sha256(projection)) records the digest of the whole materialised text" + ], + "refusal_codes": [ + "PROJECTION_BINDING" + ], + "encodes": "The health raw-literal projection: one row per person of the preparation frame (origins = document[\"origins\"][\"persons\"], :108-127, checked equal to state.frame.person :128-140), carrying source, source_year, survey_year and the 35 RAW_COLUMNS literal tokens (ASEC NOW_* values and I_* flags, ACS HINS*/FHINS*P/FHINS*C), the other survey's columns null (:183-191).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "232,053,630 (356,501 stacked persons x 650.9 B), and a second whole copy in the consumer's re-encode", + "bytes_at_full_source": "2,320,538,261 (3,565,013 x 650.9 B), twice", + "measurement_basis": "Measured at this head through the same pandas expression on invented rows with the module's RAW_COLUMNS, one-character tokens on the row's own survey and None elsewhere, difference of 1001 vs 1 rows at full-source id widths (measure_pred.py health_projection: per_stacked_person_bytes 650.92).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none", + "head_value": "none (unchanged)", + "reachable_from": "none", + "reach_chain": "qualify_current_survey_health is called only through graph_current_survey_health.qualify_health_coverage (:43), imported by graph_us_survey_enrichment.py:51 (brief section 2: not on any path); current_survey_housing.py:31 and current_survey_person_status_source.py:25 import this module for _scan (housing is off path; person status is the 49/51-node variant only).", + "classification": "stays", + "notes": "There is no bound to stay under; the full-source text is 34.6x a 64 MiB accumulation and the seal holds it twice. While scanning, this module also enforces other families' bounds it does not define: asec_coverage_authentication._BODY_MAX (268,435,456) through asec._capture(..., budget=[asec._BODY_MAX]) at :265-271, acs_person_coverage_authentication.MAX_RECORD_BYTES (400,000) and MAX_CSV_HEADER_BYTES through records._records/_decode_record at :83-84, and acs_housing_universe_source._copy(..., exact_size=acs_size) at :287-293; those owners' rows should list these sites." + }, + { + "constant": "current_survey_health_coverage.py:124 len(x) <= 64 (LITERAL_CONTRACT) and current_survey_health_source.py:100 len(row[c]) <= 64 (SOURCE_TOKEN_BOUND)", + "value": "64 characters per source token", + "enforced_at": [ + "current_survey_health_coverage.py:122-126 (all(type(x) is str and len(x) <= 64 for x in (*values, *flags))) \"LITERAL_CONTRACT\" ValueError(\"CURRENT_SURVEY_HEALTH_LITERAL_CONTRACT\")", + "current_survey_health_source.py:100 (all(len(row[c]) <= 64 for c in columns)) \"SOURCE_TOKEN_BOUND\" ValueError(\"CURRENT_SURVEY_HEALTH_SOURCE_TOKEN_BOUND\")" + ], + "refusal_codes": [ + "LITERAL_CONTRACT", + "SOURCE_TOKEN_BOUND" + ], + "encodes": "Single CSV tokens: ASEC NOW_*/I_* codes and ACS HINS*/FHINS* codes (one character in the dictionaries) plus the key columns PERIDNUM (22 digits), PH_SEQ, A_LINENO, SERIALNO (13 chars), SPORDER.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "per token, at most 22 characters in the real files (PERIDNUM); invariant", + "bytes_at_full_source": "same", + "measurement_basis": "Token shapes from _key (:41-67: PERIDNUM [0-9]{22}, SERIALNO 2024(HU|GQ)[0-9]{7}) and the recode dictionaries (:127-134); not measured.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64", + "head_value": "64 (unchanged)", + "reachable_from": "none", + "reach_chain": "recode is reached from recode_field (graph_current_survey_health._completed, :103) and _scan from qualify_current_survey_health (:273, :300), both off path as in the previous row.", + "classification": "must-not-move", + "notes": "Protects a token width; checked per row as the CSV streams, no accumulation. The row-count bound beside it, :97 count <= maximum (maximum = the pinned ASEC member rows :278 or the ACS catalogue people count :305, \"SOURCE_ROW_SHAPE\"), and the 65,536-byte drain chunk at :308 (while stream.read(65536): pass, exhausting auxiliary zip members for their CRC) are not byte transports: the first is an exact upstream row pin, the second a chunk size with no ceiling and no document." + }, + { + "constant": "qrf.py:407 _PREDICT_CHUNK_ROWS", + "value": "10_000 rows per draw chunk (chunk = max(1, min(_PREDICT_CHUNK_ROWS, -(-n // (4 * workers)))), :353); bounds the (rows x 203-point _QUANTILE_GRID) float64 prediction matrix to about 16 MB per chunk", + "enforced_at": [ + "qrf.py:349-363 (_Forest.draw slices rows into chunks; no refusal, no exception)" + ], + "refusal_codes": [], + "encodes": "No document: an in-memory prediction workspace during the per-target draw over the recipient rows (ACS persons for the predictor chain).", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "about 16.2 MB per chunk (10,000 x 203 x 8) regardless of the 342,288 recipient rows", + "bytes_at_full_source": "about 16.2 MB per chunk regardless of the 3,422,888 rows", + "measurement_basis": "Product of the constant and _QUANTILE_GRID (203 points, :394-400) read from the module; not measured.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "10_000", + "head_value": "10_000 (unchanged)", + "reachable_from": "19", + "reach_chain": "LegacyQRFApplyMatrixKernel.run -> run_application -> qrf_target.apply_target -> qrf._draw_target_with_rng (:1003-1041) -> _Forest.draw (:305-382).", + "classification": "not-a-transport", + "notes": "A memory ceiling with no document behind it. The module's other ceilings are likewise not byte transports: _DISCRETE_Y_LEAF_BOUND = 4_096 (:432, applied at :527-528 as max_samples_leaf for near-discrete targets) and _DISCRETE_Y_UNIQUE_MAX = 32 (:424) bound the forest's per-leaf value store, a model hyperparameter, not an artifact." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/pred/measure_pred.py (predictor projection through _segmented_json at the module's segment/maximum; recipient matrix; ASEC demographic projection re-encode; diagnostic host projection; health projection; raw target width) -> measurements.json", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/pred/measure_allocation.py (allocation payload per selected household through graph_survey_population._allocation_payload; codec.decode_json round trip) -> allocation-measurement.json", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/pred/RECEIPT.md (head, interpreter, sys.path and counts used)" + ], + "open_questions": [ + "The brief's _recovered/pilot-runs/... inputs (the 1/1000 preparation.json, graph.json and the recovered projection/matrix artifacts) do not exist in this worktree, and the read-only rule kept me inside it; the projection's fixed evidence preamble (including the atomic_geography validation receipt) is therefore unmeasured and every roster figure above is the roster part only. Reachability was traced from graph_atomic_survey_financial.py and experiments/native-scale-transport/harness19_base.py:532 (demographic_conditioning=True, n_estimators=2) rather than from the recovered graph.json.", + "The recipient-matrix body (48 B per ACS person under demographic conditioning: 164.3 MB at full source, 16.4 MB at 1/10) has no byte bound anywhere on the 19-path: model_input caps only the header, store.put_bytes has no size check, and the body is re-encoded whole by every _Kernel._qualified (seven kernels), the runner and the verifier. Whether the lane wants a row-count or byte ceiling on it, and whether the qualify-per-kernel re-encoding is a cost the lane accepts, are design questions for the owner.", + "origins rows were measured with native_person_id at the 7-digit stacked-person width, as byte-census-measurements.json did; if full-source ACS native person ids are wider, each extra digit adds 1 B per stacked-person row (3.6 MB per digit at full source). The full-source id widths of the spine_source_id_column were not verified from a full-source artifact.", + "qrf_target.py (not in the family list; read at :240-340 because graph_legacy_train/apply call it) carries the fitted-model envelope: _MAGIC + 8-byte header length + JSON header + pickle body with no byte bound on either side (from_trusted_bytes only checks 0 < size < len(data) - offset). The pickle grows with donor rows (selected ASEC persons, 142,125 at full source) x n_estimators (2 in the pilot, 100 default) x leaf retention (max_samples_leaf=None keeps every leaf sample). Whichever family owns qrf_target should record it.", + "Other families' constants are enforced from this family's modules and should appear as enforcement sites on the owners' rows: asec_coverage_authentication.py:39 _BODY_MAX at current_survey_health_source.py:265-271; acs_person_coverage_authentication.py:48 MAX_RECORD_BYTES and MAX_CSV_HEADER_BYTES via records._records/_decode_record at current_survey_health_source.py:83-84; acs_housing_universe_source._copy exact_size at current_survey_health_source.py:287-293; graph_survey_population.py:62/:68 ALLOCATION_MAX_BYTES/ALLOCATION_ROSTER_BYTES consumed whole at graph_current_survey_predictors.py:391 (row above) and graph_puf_diagnostic_consumer.py:616; survey_population_preparation.py:50/:58 MAX_PAYLOAD_BYTES/MAX_ROSTER_BYTES at current_survey_predictors.py:522-523 (row above) and, off path, current_survey_amounts.py:369. current_survey_geography.py:208-215 _check_asec_projection is the household half of the demographic re-encode row in another family's module.", + "The preparation receipt (about 1.2 GB at full source per the row-ceilings census) is json.loads'ed whole at current_survey_predictors.py:105 and :336 (CheckedSurveyPopulationView) and current_survey_health_source.py:218; a decode, not a bound, so no row, but the source family may want the sites.", + "graph_survey_completion_host passes state.demographic_conditioning through to the base financial run (:895), so the 45-node pilot's setting is whatever native45-v5's harness passed; that harness is not in this tree, so the demographic re-encode row's 45-node reachability is inferred from the 19-node harness only.", + "The diagnostic matrix figure (row CURRENT_SURVEY_MAX_BYTES arm b) is SCALED from the 1/1000 tax-unit ratio (1.343434 per household), per the brief's rule, and the row is off path; a verifier wanting a measured count needs a tax-unit catalogue that does not exist in the counts." + ] + }, + { + "family": "ASEC current-money donor columns codec", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py (full, 143 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py (full, 1099 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py (full, 549 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py (full, 607 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py (full, 411 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py (full, 114 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py (full, 34 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py (full, 625 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py (full, 377 lines)", + "current_survey_predictors.py (lines 330-500; def index; :600 call)", + "graph_current_survey_predictors.py (lines 270-305; :301, :540, :556-578, :606 calls)", + "asec_person_income_source.py (lines 150-232, 300-330, 380-420, 436-490)", + "asec_2024_native_population.py (lines 660-745; _ISSUED/parent greps)", + "graph_asec_prepared.py (lines 520-575; selection/receipt greps)", + "survey_population_preparation.py (:1965 in prepare_authenticated_survey_population, :2167 in _current_survey_wage_projection; graph_sources uses)", + "graph_survey_population.py (:746, :1012 prepare_authenticated_survey_population)", + "graph_atomic_survey_financial.py (lines 1090-1115; :1786)", + "graph_property_tax_leaves.py (:39, :68-69, :384 cps_carried constants)", + "asec_coverage_authentication.py (:349, :356 money uses)", + "education_assistance_source.py (:117, :139, :161 rows pins; :346)", + "native_household_origin.py (:147, :157, :167 hhpub rows pins)", + "asec_income_observations.py (lines 400-412)", + "graph_composed_asec_binding.py (lines 77-92 imports; grep of :593, :781, :799, :956-1061)", + "graph_implementation.py (:109-117 loader registry via grep)", + "experiments/native-byte-transports/census-brief.md (full)", + "experiments/native-byte-transports/byte_census_measure.py (full)", + "experiments/native-byte-transports/byte-census-measurements.json (full dump; no money-family entry)", + "experiments/native-row-ceilings/census.json (the 21 rows matching this family)" + ], + "rows": [ + { + "constant": "asec_current_money.py:46 RESOURCE_MAX_BYTES", + "value": "512 * 1024 = 524,288", + "enforced_at": [ + "asec_current_money.py:84 (`_parse` default `limit`: `_require(type(value) is bytes and 0 < len(value) <= limit, \"JSON_SIZE_OR_TYPE\")`) \u2014 \"JSON_SIZE_OR_TYPE\", MoneyRefusalError (ValueError subclass, asec_current_money.py:59-70)", + "in-family callers inheriting the default: asec_current_money.py:223 (_restored_source), :401 (three packaged resources in compile_asec_current_money_spec), :482 (source identity), :794 (MoneyBindings, authenticated evidence), :923-925 (classify_asec_money scope/index identities), :1029; asec_current_money_source.py:295-297 (_input_binding), :504 (validate); asec_current_money_units.py:269, :289, :391, :396, :403; asec_current_money_graph_resources.py:25 (`_parse(payload)` of the 3,953 B graph consumers file)", + "out-of-family callers inheriting the default: asec_person_income_source.py:232, :465; asec_student_controls.py:81, :180, :318, :329, :424, :453; asec_income_observations.py:147, :179, :307, :477, :487; asec_prepared_source.py:417, :506; graph_asec_income.py:120, :164, :217, :359, :467, :477, :585 (prepared receipt payload), :586 (ASEC household selection payload)" + ], + "refusal_codes": [ + "JSON_SIZE_OR_TYPE" + ], + "encodes": "Upstream packaged JSON resources (asec_current_money_domains_v1.json 136,308 B; asec_current_money_consumers_v1.json 17,494 B; asec_current_money_price_basis_v1.json 689 B; asec_current_money_graph_consumers_v1.json 3,953 B), all sha256-pinned (RESOURCE_PINS :39-43, GRAPH_CONSUMERS_SHA256), plus fixed-shape identity documents: the source evidence (synthetic 1,236 B; authenticated schema-2 evidence adds three keys), the spec identity (2,217 B), the scope identity (684 B: eight digests), index descriptors, the money header when parsed at units:403 (1,094 B), the tax receipt and implementation identity. Off-path only, graph_asec_income.py:585-586 also parses the prepared receipt and the per-selected-ASEC-household selection payload under this default (see open questions).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "136,308 (largest document under this cap: asec_current_money_domains_v1.json); identical at every fraction", + "bytes_at_full_source": "136,308; identical at every fraction (packaged files and fixed-key identity documents; 3.85x headroom on the largest)", + "measurement_basis": "ls -l of the four packaged resource files at this head; identity document sizes from scratchpad receipt.json fixed_documents (synthetic_source_identity 1,236; spec_identity 2,217; execution_identity 475; scope_identity 684) via measure_money_bodies.py", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "524,288 (git diff origin/native-row-ceilings 0319b1c73 is empty for this module)", + "head_value": "524,288", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial \u2192 graph_survey_population (:746/:1012 owner.prepare_authenticated_survey_population) \u2192 survey_population_preparation.prepare_authenticated_survey_population (:1965 asec_native.load_authenticated_asec_2024_native_population) \u2192 asec_2024_native_population (:702 restoration.load_authenticated_restored_current_money_source) \u2192 asec_person_income_source (:455 verify \u2192 :316 legacy.load_authenticated_current_money_source; :483 compile_asec_current_money_spec(load_current_money_resources(), authority)) \u2192 asec_current_money_source._load (:604 compile_asec_current_money_spec) \u2192 asec_current_money.compile_asec_current_money_spec :401 `_parse(v)` for domains/price/consumers; also survey_population_preparation.py:2167 `current_money._origin_code(parent.spec, \"WSAL_VAL\")` \u2192 :223 `_parse(source.identity)`; every `_spec` re-check (:493-498) recompiles and re-parses", + "classification": "must-not-move", + "notes": "The three resources are pinned bit-for-bit (`tuple(_sha(v) for v in (resources.domains, resources.price, resources.consumers)) == RESOURCE_PINS`, :405-411), so the cap is a parse guard in front of already-fixed bytes. No document under this cap on the 19/45 paths is roster-shaped. graph_asec_income.py:586 `_parse(values[\"selection\"].payload)` is the one roster-shaped consumer (selected ASEC households, produced by graph_asec_prepared.py:507-518 `\"selection\": _json(selection)`), reachable only through the PUF diagnostic consumer (puf_diagnostic_consumer.py:19, graph_puf_diagnostic_consumer.py:36), which the brief lists as off every path; it belongs to the graph_asec_income family and is flagged, not measured." + }, + { + "constant": "asec_current_money.py:47 HEADER_MAX_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "asec_current_money.py:84 via explicit limit at :147 (`_parse(self.evidence, HEADER_MAX_BYTES)` in AuthenticatedAsecSource._validate_content), :417 (execution identity), :739 (MoneyBindings.__post_init__ header), :868 (_validate_money), :1050 (restate_asec_current_money) \u2014 \"JSON_SIZE_OR_TYPE\", MoneyRefusalError", + "_asec_current_money_codec.py:115-118 (decode_current_money: `0 < size <= HEADER_MAX_BYTES and start + size <= len(payload) - 32`) \u2014 \"HEADER_SIZE\", MoneyRefusalError; :120 (`_parse(header, HEADER_MAX_BYTES)`)", + "asec_current_money_selection.py:193, :221 (header_data), :275-278 (parse_current_money_body) \u2014 \"HEADER_SIZE\"; :281; :399 (_validate_selected); :503-506 (decode_selected_current_money) \u2014 \"HEADER_SIZE\"; :508", + "out-of-family: graph_composed_asec_binding.py:1016 (`len(header) <= HEADER_MAX_BYTES`) \u2014 \"ARM_ROWS_HEADER_SIZE\", PreparedGraphError; :1057 \u2014 \"ARM_ROWS_HEADER_SIZE\"; :1061 (`_parse(header, HEADER_MAX_BYTES)`)", + "additive term of _asec_current_money_codec.py:27 PAYLOAD_MAX_BYTES, asec_current_money_selection.py:86 SELECTED_PAYLOAD_MAX_BYTES and graph_composed_asec_binding.py:959 ARM_ROWS_PAYLOAD_MAX_BYTES" + ], + "refusal_codes": [ + "JSON_SIZE_OR_TYPE", + "HEADER_SIZE", + "ARM_ROWS_HEADER_SIZE" + ], + "encodes": "Fixed-shape headers: the money header (closed key set :740-758 plus provenance: digests, two row counts, the 32-name field list) = 1,094 B at three-cohort digit widths; the selected header (_SELECTED_HEADER_KEYS :58-81 incl. 32 field_entities pairs) = 2,043 B; the source evidence (:158-175, schema 2 adds three keys) \u2248 1.2-1.5 KB; the execution identity 475 B. The header travels as a uint32 length prefix (`struct.pack(\" 64", + "value": "len(value) == 64 (hex sha256 width) = 64", + "enforced_at": [ + "asec_current_money.py:106-111 `_digest` (`type(value) is str and len(value) == 64 and all(c in \"0123456789abcdef\" ...)`), a predicate consumed by: asec_current_money.py:191-204 \u2014 \"SOURCE_PIN\"; :205-211 \u2014 \"RESTORED_SOURCE_SCHEMA\"; :266-276 \u2014 \"SOURCE_PIN\"; :281-287 \u2014 \"COHORT_PINS\"; :296-302 \u2014 \"SIDECAR_PINS\"; :431-433 \u2014 \"EXECUTION_MODULES\"; :781-792 \u2014 \"HEADER_DIGEST\" (all MoneyRefusalError)", + "asec_current_money_selection.py:269 \u2014 \"EXPECTED_HEADER_DIGEST\"; :270 \u2014 \"EXPECTED_CONTENT_DIGEST\"; :333-344 \u2014 \"SELECTION_BINDING_DIGEST\"; :415-428 \u2014 \"SELECTED_HEADER_DIGEST\"; :488-499 \u2014 \"SELECTION_BINDING_DIGEST\" (MoneyRefusalError)" + ], + "refusal_codes": [ + "SOURCE_PIN", + "RESTORED_SOURCE_SCHEMA", + "COHORT_PINS", + "SIDECAR_PINS", + "EXECUTION_MODULES", + "HEADER_DIGEST", + "EXPECTED_HEADER_DIGEST", + "EXPECTED_CONTENT_DIGEST", + "SELECTION_BINDING_DIGEST", + "SELECTED_HEADER_DIGEST" + ], + "encodes": "A single token: one 64-character lowercase-hex sha256 string (pins, digests inside headers and evidence).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 (fixed token width); does not move with any roster", + "bytes_at_full_source": "64", + "measurement_basis": "code: sha256().hexdigest() is 64 hex characters; every digest checked is produced by _sha (:73-74)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 (no diff vs origin/native-row-ceilings)", + "head_value": "64", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_current_money_source._load :603 AuthenticatedAsecSource \u2192 _validate_content :191-204; predictor chain \u2192 classify :1036 MoneyBindings \u2192 :781-792", + "classification": "must-not-move", + "notes": "An exact width, not a ceiling; changing it would reject every real digest. Listed so the `len(...)` check is not silently dropped from the census." + }, + { + "constant": "asec_current_money.py:447 128", + "value": "0 < len(v) <= 128 \u2192 128 characters per execution-identity string", + "enforced_at": [ + "asec_current_money.py:445-452 (compile_asec_current_money_spec: `all(type(v) is str and 0 < len(v) <= 128 for group in (execution[\"dependencies\"], execution[\"platform\"]) for v in group.values())`) \u2014 \"EXECUTION_IDENTITY\", MoneyRefusalError" + ], + "refusal_codes": [ + "EXECUTION_IDENTITY" + ], + "encodes": "Single scalar tokens: the seven dependency/platform strings of the execution identity built at asec_current_money_resources.py:46-63 (python version, numpy, pandas versions; system, machine, byteorder, python_implementation).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "\u2264 7 characters each in this venv ('3.14.4', '2.4.6', '3.0.3', 'Darwin', 'arm64', 'little', 'CPython'); whole execution identity 475 B; fixed", + "bytes_at_full_source": "same; no roster behind it", + "measurement_basis": "receipt.json fixed_documents.execution_identity_bytes = 475 via packaged_current_money_resources at this head; venv versions printed by the sanity check", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "128 (no diff vs origin/native-row-ceilings)", + "head_value": "128", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_current_money_source._load :604 / asec_person_income_source.py:483 `compile_asec_current_money_spec(load_current_money_resources(), authority)` \u2192 :445-452; re-run on every `_spec` check (:493-498) from _validate_money/classify on the predictor chain", + "classification": "must-not-move", + "notes": "Protects a fixed-width field of a fixed-shape identity document; not a transport." + }, + { + "constant": "asec_current_money.py:573 256", + "value": "0 < len(k) <= 256 \u2192 256 characters per native key", + "enforced_at": [ + "asec_current_money.py:565-575 (AsecMoneyScope.__post_init__: `all(type(k) is str and 0 < len(k) <= 256 for k in keys)` for person_native_keys and household_native_keys) \u2014 \"NATIVE_KEYS\", MoneyRefusalError" + ], + "refusal_codes": [ + "NATIVE_KEYS" + ], + "encodes": "A per-element width on a per-row roster: one native key string per ASEC person (PERIDNUM, decoded at asec_current_money_source.py:272-275; an exact 22-digit string per education_assistance_source.py:346) and one per ASEC household (`str(int(v)) for v in keys.H_SEQ`, :276, \u2264 6 digits). The roster length is bounded by MAX_PERSONS/MAX_HOUSEHOLDS, not by this literal.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "22 characters per person key, \u2264 6 per household key (fixed widths); the whole tuple is 432,523 + H3 strings at every fraction", + "bytes_at_full_source": "same; 11.6x headroom on the 22-character PERIDNUM", + "measurement_basis": "code widths (education_assistance_source.py:346 PERIDNUM 22 digits; H_SEQ str(int)); measurement scripts used 22-character keys and passed the check", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "256 (no diff vs origin/native-row-ceilings)", + "head_value": "256", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_current_money_source._load :584 `_scope(frame)` \u2192 :265-277 AsecMoneyScope(...) \u2192 __post_init__ :570-575; predictor chain \u2192 validate :512 `_scope(self.frame)`", + "classification": "must-not-move", + "notes": "Fixed-width field. The same tuples are also JSON-encoded whole and uncapped by `identity` (:583-585); see open questions." + }, + { + "constant": "_asec_current_money_codec.py:26 PAYLOAD_MAX_BYTES", + "value": "len(MAGIC) + 4 + HEADER_MAX_BYTES + 11 * (32 * MAX_PERSONS + MAX_HOUSEHOLDS) + 32 = 9 + 4 + 65,536 + 11 * (32,000,000 + 400,000) + 32 = 356,465,581", + "enforced_at": [ + "_asec_current_money_codec.py:56 (_ready_parts, shared by encode_current_money :79 and current_money_content_sha256 :65: `_require(sum(map(len, parts)) + 32 <= PAYLOAD_MAX_BYTES, \"PAYLOAD_SIZE\")`) \u2014 \"PAYLOAD_SIZE\", MoneyRefusalError", + "_asec_current_money_codec.py:97-101 (decode_current_money: `len(MAGIC) + 4 + 32 < len(payload) <= PAYLOAD_MAX_BYTES`) \u2014 \"PAYLOAD_SIZE\"; :122-125 exact length `start + size + 11 * (32 * data[\"person_rows\"] + data[\"household_rows\"]) + 32` \u2014 \"PAYLOAD_LENGTH\"", + "asec_current_money_selection.py:264-268 (parse_current_money_body, imported as BODY_PAYLOAD_MAX_BYTES :28) \u2014 \"PAYLOAD_SIZE\"; :298-301 \u2014 \"PAYLOAD_LENGTH\"", + "consumer-side re-encodings of the whole body through _ready_parts (cap :56): asec_current_money_units.py:299-305 (`current_money_content_sha256(self.money) == receipt[\"money_content_sha256\"]`) \u2014 \"TAX_RESULT_CHANGED\"; asec_student_controls.py:476; current_survey_person_status_source.py:516; graph_current_survey_person_status.py:279; and byte comparisons _asec_current_money_codec.py:137 (`tuple(fields) == expected.fields`) \u2014 \"REPLAY_CONTENT\", asec_current_money_selection.py:305-307 (content re-hash) \u2014 \"BODY_CONTENT_BINDING\"" + ], + "refusal_codes": [ + "PAYLOAD_SIZE", + "PAYLOAD_LENGTH" + ], + "encodes": "The numpy money body of the whole authenticated ASEC current-money source: MAGIC (9) + uint32 header length + header (1,094 B) + for each of 32 person fields four buffers of 8+1+1+1 bytes per ASEC person (`field.amount_bytes, status_bytes, validity_bytes, zero_origin_bytes`, :47-55) + 11 bytes per ASEC household for HTOTVAL + 32-byte sha256. The source carries all three income years (asec_current_money_source.py:264), so the body is 432,523 persons plus the three-cohort households at every selection fraction.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "154,115,924 (432,523 persons pinned; households 169,699 ESTIMATE) \u2014 at most 155,190,448 (households \u2264 267,383); identical at every fraction because the source is the whole pinned file set. A 2024-cohort-only body would be 50,642,520 (measured) but the loader refuses a source without all three years.", + "bytes_at_full_source": "154,115,924 (ESTIMATE households) / \u2264 155,190,448 (hard bound); headroom 2.31x / 2.30x against 356,465,581", + "measurement_basis": "measure_three_cohort_body.py through _asec_current_money_codec.encode_current_money on invented zero amounts (asserted == 9 + 4 + len(header) + 11 * (32P + H) + 32); marginals 352 B/person and 11 B/household from measure_money_bodies.py at (200,40)/(1200,40)/(200,140); person pins education_assistance_source.py:117/139/161 enforced at asec_person_income_source.py:171; receipts receipt_three_cohort.json and receipt.json", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "356,465,581 (no diff vs origin/native-row-ceilings)", + "head_value": "356,465,581", + "reachable_from": "none", + "reach_chain": "none from 19/45: the predictor lane slices the in-process ready object (current_survey_predictors.py:391-394 `field = ready.field(name)`, `amounts = field.amounts[take]`) and never calls the codec. Functions that would reach it: asec_prepared_source.py:471 `encode_current_money(ready)` / :503 (load_graph_asec_prepared, registered at graph_implementation.py:117); asec_income_observations.py:490; asec_student_controls.py:332/:476/:487; current_survey_person_status_source.py:346/:516 (49/51-node person status); asec_current_money_units.py:301/:390 (reconstruct_current_money_tax_units \u2190 asec_prepared_source.py:440); parse_current_money_body only at graph_asec_prepared.py:422 and graph_composed_asec_binding.py:593 (PUF diagnostic consumer path)", + "classification": "stays", + "notes": "Derived: it moves only with MAX_PERSONS/MAX_HOUSEHOLDS/HEADER_MAX_BYTES; the row widths (8,1,1,1) are fixed at :132 and :46-55 and the decoder recomputes the exact length from the header rows (:122-125), so the cap is the pre-parse budget, not the wire format. The bytes exist whole: `b\"\".join((*parts, checksum.digest()))` (:83); the content digest streams over parts that already exist as MoneyField bytes. The row-ceilings census's ~154.1 MB agrees; its `count_at_full_source` for this row is fraction-independent, as here." + }, + { + "constant": "asec_current_money_selection.py:83 SELECTED_PAYLOAD_MAX_BYTES", + "value": "len(SELECTED_MAGIC) + 4 + HEADER_MAX_BYTES + 11 * (32 * MAX_PERSONS + MAX_HOUSEHOLDS) + 32 = 9 + 4 + 65,536 + 356,400,000 + 32 = 356,465,581", + "enforced_at": [ + "asec_current_money_selection.py:461 (_selected_parts, used by encode_selected_current_money :467: `_require(sum(map(len, parts)) + 32 <= SELECTED_PAYLOAD_MAX_BYTES, \"PAYLOAD_SIZE\")`) \u2014 \"PAYLOAD_SIZE\", MoneyRefusalError", + "asec_current_money_selection.py:483-487 (decode_selected_current_money: `len(SELECTED_MAGIC) + 4 + 32 < len(payload) <= SELECTED_PAYLOAD_MAX_BYTES`) \u2014 \"PAYLOAD_SIZE\"; :523-524 exact length \u2014 \"PAYLOAD_LENGTH\"", + "consumer-side re-encoding of the whole selected body under the :461 cap: graph_asec_income.py:380 (`_sha(encode_selected_current_money(selected)) == selected_money_sha256`); producers graph_asec_prepared.py:563/:569, graph_composed_asec_binding.py:799; decoders graph_asec_income.py:607, graph_asec_prepared.py:691, graph_composed_asec_measures.py:202" + ], + "refusal_codes": [ + "PAYLOAD_SIZE", + "PAYLOAD_LENGTH" + ], + "encodes": "The selected-subset money body: whole selected ASEC households and their persons, a strictly increasing position slice of the body (`_positions` :122-132; `field.amounts[index].tobytes()` :360-366): SELECTED_MAGIC + uint32 + selected header (2,043 B) + 352 B per selected ASEC person + 11 B per selected ASEC household + 32.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "5,066,046 (14,212 selected ASEC persons, 5,576 households; header 2,041 B)", + "bytes_at_full_source": "50,643,470 (142,125 persons, 55,762 households; header 2,043 B); 7.04x headroom", + "measurement_basis": "measure_money_bodies.py: parse_current_money_body(encode_current_money(ready), \u2026) \u2192 select_current_money(positions 0..P-1, 0..H-1) \u2192 encode_selected_current_money, asserted == 9 + 4 + len(header) + 11 * (32P + H) + 32 and round-tripped through decode_selected_current_money; measured at 1/10 and full 2024-cohort counts from the brief \u00a73 (receipt.json full_source_body_selected_tenth / full_source_body_selected_all)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "356,465,581 (no diff vs origin/native-row-ceilings)", + "head_value": "356,465,581", + "reachable_from": "none", + "reach_chain": "none from 19/45: select_current_money / encode_selected_current_money are called only from graph_asec_prepared.py:543/:563/:569 (load_graph_asec_prepared lane) and graph_composed_asec_binding.py:781/:799 (PUF diagnostic consumer via puf_diagnostic_consumer.py:498); decode only from graph_asec_income.py:607, graph_asec_prepared.py:691, graph_composed_asec_measures.py:202; the on-path predictor lane takes its ASEC slice directly from the ready object (current_survey_predictors.py:381-399)", + "classification": "stays", + "notes": "Bounded above by the parent body's rows (selection is a subset), so it can never exceed the PAYLOAD_MAX_BYTES row's document. The row-ceilings census recorded '~52 MB' at full source; the encoder gives 50,643,470 B." + }, + { + "constant": "asec_current_money_source.py:72 1024 * 1024", + "value": "1024 * 1024 = 1,048,576 (a read chunk, not a cap)", + "enforced_at": [ + "asec_current_money_source.py:72 (_stage_verified: `while chunk := src.read(1024 * 1024): digest.update(chunk); dst.write(chunk)`) \u2014 no size refusal; the loop digests and copies the whole file. Refusals on this path are identity refusals only: \"SOURCE_BYTES_CHANGED\" (:76-92, fstat before/after), \"SOURCE_BYTES_MISMATCH\" (:93), \"SOURCE_BYTES_UNAVAILABLE\" (:94-95); MoneyRefusalError" + ], + "refusal_codes": [], + "encodes": "Upstream files streamed whole into a private staged copy: the pinned parent checkpoint (parent.h5, _SOURCE_PINS[0]) and the household-observation attachment (attachment.h5, _SOURCE_PINS[1]) at :549/:555, and the three ASEC person member CSVs via asec_person_income_source.py:167.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "the input files' own sizes, identical at every fraction; not read here (gated data; their byte caps are asec_2024_native_population._MAX_CHECKPOINT_BYTES :720 and asec_person_income_source._ATTACHMENT_MAX_BYTES :39, other families)", + "bytes_at_full_source": "same files; the chunk size does not bound them", + "measurement_basis": "code only (no data read, per brief \u00a74)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,048,576 (no diff vs origin/native-row-ceilings)", + "head_value": "1,048,576", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_person_income_source :316 legacy.load_authenticated_current_money_source \u2192 asec_current_money_source._load :549 `_stage_verified(parent_path, _SOURCE_PINS[0], parent_copy)`, :555; also asec_person_income_source.py:167 `legacy._stage_verified(member_paths[year], pin, staged)`", + "classification": "must-not-move", + "notes": "Recorded because it is an inline byte literal on a read; it bounds nothing. The whole file is digested and copied; the only size-shaped observation is `before.st_size == after.st_size` (:81/:88)." + }, + { + "constant": "asec_current_money_source.py:126 1024 / 262144", + "value": "len(string_tokens) < 1024 and token_bytes + len(token) <= 262144 \u2192 1,024 cache entries / 262,144 bytes", + "enforced_at": [ + "asec_current_money_source.py:126-128 (_series_digest cache admission: `if len(string_tokens) < 1024 and token_bytes + len(token) <= 262144: string_tokens[value] = token`) \u2014 no refusal, nothing raised; a token past either ceiling is still digested (:129 `digest.update(token)`) but not memoised" + ], + "refusal_codes": [], + "encodes": "Nothing on the wire: a per-column memo of length-prefixed `checkpoint._encode_object_scalar` string tokens while `_frame_signature` (:136-165) streams every column of the whole source frame into sha256. The digest itself is unbounded.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "\u2264 262,144 retained per object column by construction; the digested stream is the whole three-cohort frame at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "code only", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,024 / 262,144 (no diff vs origin/native-row-ceilings)", + "head_value": "1,024 / 262,144", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_current_money_source._load :560 `_validate_attachment` \u2192 :394 `_same_table` \u2192 :334-335 `_series_digest`; :599 `_frame_signature(frame)`; asec_person_income_source.py:238/:239/:474; asec_coverage_authentication.py:356 `money._frame_signature(source.frame)`; predictor chain \u2192 AuthenticatedCurrentMoneySource.validate :511", + "classification": "not-a-transport", + "notes": "A memory ceiling with no document behind it; the comment at :114-117 states the intent (\"Both entry and payload bounds keep distinct or large strings bounded\"). Not a cap on the digested bytes." + }, + { + "constant": "asec_current_money_resources.py:104 1024 * 1024", + "value": "1024 * 1024 = 1,048,576", + "enforced_at": [ + "asec_current_money_resources.py:104 (load_current_money_resources: `_require(path.stat().st_size <= 1024 * 1024, \"MICROUNIT_MODULE_SIZE\")`) \u2014 \"MICROUNIT_MODULE_SIZE\", MoneyRefusalError", + "asec_current_money_resources.py:106-109 (`len(module) <= 1024 * 1024 and _sha(module) == MICROUNIT_SHA256`) \u2014 \"MICROUNIT_MODULE_SHA256\", MoneyRefusalError; OSError/PackageNotFoundError \u2192 \"MICROUNIT_UNAVAILABLE\" (:110-111)" + ], + "refusal_codes": [ + "MICROUNIT_MODULE_SIZE", + "MICROUNIT_MODULE_SHA256" + ], + "encodes": "Upstream file: the installed microunit/tax_unit_construction.py, whose sha256 must equal MICROUNIT_SHA256 (asec_current_money.py:45).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "31,259 (the venv's microunit 0.1.0 tax_unit_construction.py; sha256 79007065\u2026 matches the pin); identical at every fraction; 33.5x headroom", + "bytes_at_full_source": "31,259", + "measurement_basis": "importlib.metadata.distribution('microunit').locate_file(...).stat().st_size in the worktree venv, sha256 verified against MICROUNIT_SHA256", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,048,576 (no diff vs origin/native-row-ceilings)", + "head_value": "1,048,576", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_current_money_source._load :604 `compile_asec_current_money_spec(load_current_money_resources(), authority)`; asec_person_income_source.py:483 (restored source) \u2192 asec_current_money_resources.load_current_money_resources :72-114", + "classification": "must-not-move", + "notes": "Protects an upstream file whose exact bytes are pinned; the stat check precedes the read so the whole read is bounded before it materialises." + }, + { + "constant": "asec_current_money_units.py:114 1024 * 1024", + "value": "1024 * 1024 = 1,048,576", + "enforced_at": [ + "asec_current_money_units.py:114 (_runtime_file_bytes: `_require(path.stat().st_size <= 1024 * 1024, \"MICROUNIT_EXECUTION_SIZE\")`) and :116 (`_require(len(value) <= 1024 * 1024, \"MICROUNIT_EXECUTION_SIZE\")`), called at :129 for each of the seven _MICROUNIT_EXECUTION_PINS \u2014 \"MICROUNIT_EXECUTION_SIZE\", MoneyRefusalError (sha mismatch \u2192 \"MICROUNIT_EXECUTION_PIN\" :130)" + ], + "refusal_codes": [ + "MICROUNIT_EXECUTION_SIZE" + ], + "encodes": "Upstream files: the seven pinned microunit runtime files (__init__.py 1,799 B; core.py 6,439; diagnostics.py 2,655; registry.py 1,267; rule_helpers.py 4,638; tax_unit_construction.py 31,259; data/dependent_gross_income_limit.yaml 2,769 \u2014 venv, all sha-pinned at :102-110).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "31,259 (largest of the seven); identical at every fraction; 33.5x headroom", + "bytes_at_full_source": "31,259", + "measurement_basis": "sizes and sha256 of each pinned file from the worktree venv's microunit 0.1.0 distribution", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,048,576 (no diff vs origin/native-row-ceilings)", + "head_value": "1,048,576", + "reachable_from": "none", + "reach_chain": "none from 19/45: `_execution` (:120) is called only by `_reconstruct` (:324, :364) and `CurrentMoneyTaxUnitResult.validate` (:307); `reconstruct_current_money_tax_units` is called only from asec_prepared_source.py:440 (load_graph_asec_prepared lane); `CurrentMoneyTaxUnitResult` is consumed by asec_housing_universe_source.py:17 and asec_housing_status_source.py:24, the ASEC housing sources, not the on-path acs_housing_universe_source", + "classification": "must-not-move", + "notes": "Same shape as the resources-module cap; protects upstream pinned files. The units module also re-derives the whole money body without any byte cap (`expected = source.ready()`, `ready.fields == expected.fields`, :189-193) and JSON-encodes the person-by-tax-unit partition whole (`_sha(_json(_partitions(original)))`, :401-402); both off-path, see open questions." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/money/measure_money_bodies.py (encode_current_money / parse_current_money_body / select_current_money / encode_selected_current_money / decode_selected_current_money on invented zero amounts at full-source id widths; receipt: .../census/money/receipt.json)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/money/measure_three_cohort_body.py (encode_current_money at the three-cohort source counts 432,523 persons with 169,699 and 267,383 households; receipt: .../census/money/receipt_three_cohort.json)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/money/README.md (index; both scripts assert every imported module resolves inside the worktree, per the venv-shadowing memory note)" + ], + "open_questions": [ + "Three-cohort household count of the authenticated source (H3) is not pinned in code: only the 2024 cohort (55,762) is a catalogue count; 169,699 is the row-ceilings lane's estimate and 267,383 (all hhpub rows, native_household_origin.py:147/157/167) is the hard upper bound. The PAYLOAD_MAX_BYTES verdict does not depend on it (headroom 2.31x vs 2.30x), but the body's exact byte figure does.", + "Uncapped materialised JSON on the 19 path: AsecMoneyScope.identity (asec_current_money.py:583-585) encodes each roster vector whole with `_json(getattr(self, f.name))` before hashing \u2014 at the three-cohort source person_native_keys is 10,813,076 B and person_ids 3,027,662 B (measured). There is no bound and no re-encoding check, so no row; flag for the transport lane if whole-roster JSON materialisation is in scope.", + "Off-path roster-shaped consumer of RESOURCE_MAX_BYTES: graph_asec_income.py:586 `_parse(values[\"selection\"].payload)` parses the ASEC household selection payload (graph_asec_prepared.py:507-518, `\"selection\": _json(selection)`) under the 512 KiB default; that document is per selected ASEC household and belongs to the graph_asec_income family (PUF diagnostic consumer path, off 19/45). Not measured here; its owner should decide whether it binds at 55,762 households.", + "Off-path uncapped re-derivation: asec_current_money_units.current_money_microunit_view (:189-193) recomputes the whole money body via `source.ready()` and compares MoneyField bytes with no byte cap; `_partitions` JSON (:401-402) is a whole-roster JSON with no cap. Reached only from asec_prepared_source.py:440.", + "cps_carried.py and cps_carried_current.py carry no byte bound at all; cps_carried.py:79 US_REPORTED_COVERAGE_VINTAGE_GATE_MIN_ROWS = 5_000 is a row-count minimum enforced in support_provenance (not a byte bound) and gets no row. cps_carried is reached from the 45-node graph only as constants (graph_property_tax_leaves.py:68-69; graph_atomic_survey_financial.py:1100-1113 `tax.cps_carried`); cps_carried_current is reached from 19 via current_survey_predictors.py:479/:600 with no bound behind it.", + "asec_current_money_graph_resources.py defines no bound of its own; its `_parse(payload)` (:25) is an enforcement site of RESOURCE_MAX_BYTES on a 3,953 B pinned file, reached only from graph_asec_prepared.py:317.", + "The row-ceilings census (experiments/native-row-ceilings/census.json) lists SELECTED_PAYLOAD_MAX_BYTES count_at_full_source as '~52 MB'; the module's own encoder gives 50,643,470 B for 142,125 persons + 55,762 households. Its ~154.1 MB for the body matches 154,115,924 B here.", + "No bound in this family moved on this branch: `git diff origin/native-row-ceilings (0319b1c73)` is empty for all nine modules, so base_value == head_value throughout." + ] + }, + { + "family": "Geography and the graph package", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py (full, 639 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py (full, 320)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py (full, 277)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_geography.py (full, 395)", + "packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_geography.py (full, 384; no byte bound found)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py (full, 1141; no byte bound found)", + "packages/microcosm-build/src/microcosm/build/us_runtime/geography_ladder.py (full, 760; no byte bound found)", + "packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py (full, 276; no byte bound found)", + "packages/microcosm-graph/src/microcosm/graph/codecs.py (full, 611)", + "packages/microcosm-graph/src/microcosm/graph/store.py (full, 1451)", + "packages/microcosm-graph/src/microcosm/graph/executor.py (full, 3072)", + "packages/microcosm-graph/src/microcosm/graph/manifest.py (full, 1460)", + "packages/microcosm-graph/src/microcosm/graph/canonical.py (full, 106; no byte bound; canonical_json has no cap)", + "packages/microcosm-graph/src/microcosm/graph/serialize.py (full, 378; no byte bound; graph declaration is fixed-shape, 73,156 B at 1/1000)", + "packages/microcosm-graph/src/microcosm/graph/keys.py (full, 301)", + "supporting excerpts for enforcement chains and document shapes: survey_population_preparation.py:40-175,775-795 (_encode/_roster_segments); survey_origin_budget.py:1-60,180-260,440-470 (geography_contract attestation, reconstruct call); graph_atomic_survey_population.py:79-125,140-330; graph_atomic_survey_financial.py:1275-1440; survey_age_calibration.py:78-100,440-475; graph_survey_completion_host.py (def/call grep); graph_survey_population.py:292-375,881-925; current_asec_demographics.py:57-72,245-375; microcosm/graph/population.py:366-449,1656-1712 (expand_lineage_receipt); graph_combined_clone.py:226-266,612-657; microcosm/build/graph_atomic_geography.py:40-120; microcosm/build/atomic_geography.py:541-591; graph_context.py:104-178; puf_support.py:2324-2364 (clone id remap); atomic_block_sources.py:47-49, atomic_block_api_sources.py:30-32, puf55_survey_recipients.py:51-53 (_require types)", + "inputs: experiments/native-byte-transports/census-brief.md; experiments/native-byte-transports/byte_census_measure.py; experiments/native-byte-transports/byte-census-measurements.json (no geography document measured there); experiments/native-row-ceilings/census.json (6 rows for this family, stale on MAX_HOUSEHOLDS); docs/us-native-row-ceilings.md:100-140,205-235; docs/us-native-scale-transport.md (grep); _recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/{graph.json (node/kernel roster), preparation.json (id and key widths), manifest.json (receipt sizes and lineage row counts only)}; stat + zip central directory of _recovered/microcosm-native-sources-20260909/national-atomic-support.npz" + ], + "rows": [ + { + "constant": "packages/microcosm-graph/src/microcosm/graph/codecs.py:78 RAW_BYTES_MAX_BYTES", + "value": "64 * 1024 * 1024 = 67,108,864", + "enforced_at": [ + "codecs.py:571 `payload = handle.read(RAW_BYTES_MAX_BYTES + 1)` (read cap, no refusal)", + "codecs.py:576-580 `if len(payload) > RAW_BYTES_MAX_BYTES: raise ValueError(f\"Raw source at {source} is larger than the {RAW_BYTES_MAX_BYTES}-byte raw-bytes-v1 limit.\")` (message, no code literal; plain ValueError)", + "survey_atomic_geography.py:63 alias `MAX_SUPPORT_BYTES = RAW_BYTES_MAX_BYTES` enforced at :152 \"SUPPORT_SIZE\" ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_SIZE\") (own row below)", + "atomic_block_sources.py:365 `_require(len(payload) <= RAW_BYTES_MAX_BYTES, \"SUPPORT_BYTES\")` ValueError(\"ATOMIC_BLOCK_SOURCES_SUPPORT_BYTES\") (producer of the same support file; not on any runner path)", + "atomic_block_api_sources.py:264 `_require(len(payload) <= RAW_BYTES_MAX_BYTES, \"SUPPORT_BYTES\")` ValueError(\"ATOMIC_BLOCK_API_SOURCES_SUPPORT_BYTES\") (not on path)", + "puf55_survey_recipients.py:422 and :431 \"MATRIX_SIZE\" ValueError(\"PUF55_SURVEY_RECIPIENTS_MATRIX_SIZE\") (recipient matrix; reachable_from none per brief \u00a72)", + "call sites that reach codecs.py:571/:576: microcosm/build/graph_atomic_geography.py:72 `load_source_bytes(\"raw-bytes-v1\", ...)` (geography.support.0, on the 19/45/age paths); graph_geography.py:124-129 (us_puma_ladder_2020, us_cd_crosswalk_117_119; reachable_from none); registry binding graph_atomic_survey_population.py:199 `store.codecs.register_bytes(\"raw-bytes-v1\", load_raw_bytes)`", + "attested, not enforced: survey_origin_budget.py:213 folds geography.RAW_BYTES_MAX_BYTES into `geography_contract`; tests pin it at test_us_atomic_block_sources.py:243" + ], + "refusal_codes": [ + "(codecs) ValueError message \"Raw source at ... is larger than the 67108864-byte raw-bytes-v1 limit.\"", + "SUPPORT_SIZE", + "SUPPORT_BYTES", + "MATRIX_SIZE" + ], + "encodes": "an upstream file: one raw-bytes-v1 source read whole. On the 19/45/age paths the only raw-bytes source is `us_atomic_block_support` (graph.json sources: survey_population_source/us-survey-population-source-v1, us_atomic_block_support/raw-bytes-v1), the national 2020 block support NPZ; on the unreachable graph_geography path the PUMA-ladder NPZ and CD crosswalk CSV.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "28,862,508 (the support file's on-disk size; national block geography, identical at every fraction; stat verified this session)", + "bytes_at_full_source": "28,862,508 (unchanged; 43.0% of the cap)", + "measurement_basis": "ls -l of national-atomic-support.npz (sha256 pinned by the pilot configs; row-ceilings census.json measured the same 28,862,508); the file does not carry a survey roster (atomic_block_support.py:21-22 SYSTEM/SOURCE, :105-205 block arrays only)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "67,108,864 (origin/native-row-ceilings 0319b1c73; git diff empty)", + "head_value": "67,108,864", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1275) \u2192 :1355 atomic.run_atomic_survey_population \u2192 :231 run_graph \u2192 executor.py:2784 kernel.run for node geography.support.0 (kernel geography.support_import@1) \u2192 graph_atomic_geography.AtomicSupportImportKernel.run:72 load_source_bytes(\"raw-bytes-v1\") \u2192 codecs.load_source_bytes:602 \u2192 SourceCodecRegistry.load_bytes:198 \u2192 load_raw_bytes:571/:576. Same chain from 45 (run_atomic_survey_financial:1320 base run, then completion_host.extend) and age (survey_age_calibration.py:447 run_atomic_survey_population).", + "classification": "must-not-move", + "notes": "codecs.py:79 \"The most ``raw-bytes-v1`` will read from one source file (64 MiB).\"; :541-547 \"The read is bounded rather than trusted to ``st_size``, because a file may grow after it is measured ... this bound only keeps the codec from reading an unbounded amount first.\" Protects an upstream file's real size; the sha256 pin (survey_atomic_geography.py:162) is what asserts the file's content. Shared constant: moving it moves atomic_block_sources:365, atomic_block_api_sources:264 and the puf55 recipient-matrix cap at once. The recovered 1/1000 manifest shows geography.support.0's receipt at 1,190 B (support_sha256/areas/metadata), so the receipt side is fixed; only the artifact bytes (28.9 MB) are whole. No geography document is measured in byte-census-measurements.json." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:63 MAX_SUPPORT_BYTES", + "value": "RAW_BYTES_MAX_BYTES = 64 * 1024 * 1024 = 67,108,864 (alias of codecs.py:78)", + "enforced_at": [ + "survey_atomic_geography.py:151-154 `_require(stat.S_ISREG(before.st_mode) and 0 < before.st_size <= MAX_SUPPORT_BYTES, \"SUPPORT_SIZE\")` \u2192 ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_SIZE\") (_require at :68-70)", + "survey_atomic_geography.py:155 `payload = stream.read(before.st_size + 1)` read cap derived from the checked st_size; :158-164 `len(payload) == before.st_size and _sha(payload) == config.support_sha256` \"SUPPORT_CHANGED\" ValueError", + "survey_atomic_geography.py:420 first `_read_support(config)` and :594 second read `_read_support(config) == (support_payload, support_identity)` \"FINAL_SUPPORT\" (the file is read whole twice per reconstruct call)", + "attested, not enforced: survey_origin_budget.py:210 folds geography.MAX_SUPPORT_BYTES into `geography_contract`" + ], + "refusal_codes": [ + "SUPPORT_SIZE", + "SUPPORT_CHANGED", + "FINAL_SUPPORT" + ], + "encodes": "an upstream file: the national atomic block support NPZ named by AtomicSurveyReconstruction.support_path and pinned by support_sha256 (:81-84, :106)", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "28,862,508 (identical at every fraction)", + "bytes_at_full_source": "28,862,508 (43.0% of the cap)", + "measurement_basis": "stat of national-atomic-support.npz this session; national block geography carries no survey roster", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "67,108,864 (identical on origin/native-row-ceilings)", + "head_value": "67,108,864", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 \u2192 graph_atomic_survey_population.run_atomic_survey_population:169 reconstruction.reconstruct_atomic_survey_geography (again at :293, and a third call at graph_atomic_survey_financial.py:1408) \u2192 survey_atomic_geography.py:420/:594 _read_support \u2192 :152. Age: survey_age_calibration.py:447 (prefix run) and :457 (direct call). 45: same base run via run_atomic_survey_financial:1320.", + "classification": "must-not-move", + "notes": "Same number as codecs.py:78 by alias; recorded separately because it is a module-level definition with its own refusal literal. The row-ceilings verifier's correction stands at this head: the size ceiling is a bounded-read resource ceiling, the sha256 at :162 is the content claim. The _require helper is at :68-70 (census.json cited :67-69). Reconstruct is called three times per 19-node run and each call reads the file twice, so six whole 28.9 MB reads before the executor's own read at graph_atomic_geography.py:72." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:64 MAX_SUPPORT_EXPANDED_BYTES", + "value": "2 * 1024**3 = 2,147,483,648", + "enforced_at": [ + "survey_atomic_geography.py:173-178 `_require(0 < len(members) <= MAX_SUPPORT_MEMBERS and sum(member.file_size for member in members) <= MAX_SUPPORT_EXPANDED_BYTES, \"SUPPORT_EXPANDED_SIZE\")` \u2192 ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_EXPANDED_SIZE\")", + "attested, not enforced: survey_origin_budget.py:211" + ], + "refusal_codes": [ + "SUPPORT_EXPANDED_SIZE" + ], + "encodes": "an upstream file's declared decompressed size: the sum of ZipInfo.file_size over the support archive's members, checked before numpy opens any array (:168-169 comment)", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "1,061,673,840 (zip central directory: area 346,196,648 + tract 253,877,576 + puma 161,558,504 + county 115,398,968 + district 92,319,200 + population 46,159,664 + state 46,159,664 + metadata_json 3,616; identical at every fraction)", + "bytes_at_full_source": "1,061,673,840 (49.4% of the cap)", + "measurement_basis": "python zipfile.infolist() over the pinned support archive this session; agrees with the row-ceilings census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "2,147,483,648 (identical on origin/native-row-ceilings)", + "head_value": "2,147,483,648", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 \u2192 run_atomic_survey_population:169/:293 (and financial:1408) \u2192 reconstruct_atomic_survey_geography:421 _decode_support \u2192 :173-178. Also age (:447/:457) and 45.", + "classification": "must-not-move", + "notes": "Tightest relative headroom in the family (2.02\u00d7), but the quantity is national block geography (5.77M blocks at U15 = 60 B/row for area.npy) and cannot move with the survey fraction. At the archive's 36.78\u00d7 expansion ratio this ceiling would refuse at about 55.7 MiB compressed, before MAX_SUPPORT_BYTES does." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:65 MAX_SUPPORT_MEMBERS", + "value": "32", + "enforced_at": [ + "survey_atomic_geography.py:173-178 (same _require as MAX_SUPPORT_EXPANDED_BYTES) \"SUPPORT_EXPANDED_SIZE\" ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_EXPANDED_SIZE\")", + "attested, not enforced: survey_origin_budget.py:212" + ], + "refusal_codes": [ + "SUPPORT_EXPANDED_SIZE" + ], + "encodes": "a record cap on the upstream archive: the number of ZIP members the support NPZ declares (a fixed eight-array schema: area, county, district, metadata_json, population, puma, state, tract)", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "8 members (not bytes; identical at every fraction)", + "bytes_at_full_source": "8 members", + "measurement_basis": "zip central directory of the pinned archive this session", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "32 (identical on origin/native-row-ceilings)", + "head_value": "32", + "reachable_from": "19", + "reach_chain": "same as MAX_SUPPORT_EXPANDED_BYTES: reconstruct_atomic_survey_geography:421 _decode_support \u2192 :173-178 (19 via financial:1355; age via :447/:457; 45 via financial:1320).", + "classification": "must-not-move", + "notes": "A member count, not a byte count; listed because the brief counts record caps and it shares the byte bound's refusal. It bounds the per-member NPY header loop at :179-196. 4\u00d7 headroom over a fixed schema." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:190-195 SUPPORT_ARRAY_SIZE (math.prod(shape) * dtype.itemsize == member.file_size - stream.tell())", + "value": "an equality, no fixed integer: declared array bytes must equal the physical member payload length (per member); plus :181-187 NPY version 1.0/2.0 only (\"SUPPORT_NPY_VERSION\")", + "enforced_at": [ + "survey_atomic_geography.py:190-195 `_require(not dtype.hasobject and all(type(size) is int and size >= 0 for size in shape) and math.prod(shape) * dtype.itemsize == member.file_size - stream.tell(), \"SUPPORT_ARRAY_SIZE\")` \u2192 ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_ARRAY_SIZE\")", + "survey_atomic_geography.py:187 `raise ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_NPY_VERSION\")`" + ], + "refusal_codes": [ + "SUPPORT_ARRAY_SIZE", + "SUPPORT_NPY_VERSION" + ], + "encodes": "a struct-width product on the upstream file: each NPY member's header-declared shape \u00d7 itemsize against its physical bytes (:188-189 comment: \"A small ZIP member must not claim a huge allocation in its NPY header\")", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "largest member area.npy 346,196,648 bytes = 5,769,944 blocks \u00d7 U15 (60 B); identical at every fraction", + "bytes_at_full_source": "346,196,648 (unchanged)", + "measurement_basis": "zip central directory this session; row-ceilings verifier's area.npy row width", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "unchanged (identical on origin/native-row-ceilings)", + "head_value": "unchanged", + "reachable_from": "19", + "reach_chain": "reconstruct_atomic_survey_geography:421 _decode_support \u2192 :179-196 per member (19 via financial:1355 \u2192 run_atomic_survey_population:169; age via survey_age_calibration:447/:457; 45 via financial:1320).", + "classification": "must-not-move", + "notes": "Not a ceiling but an exact-length check that refuses a header over-claim; recorded because the brief lists struct-width products. The shared decoder atomic.decode_atomic_support (:197) owns every schema/value check after it." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:98 0 < len(config.support_path) <= 4096", + "value": "4096 characters", + "enforced_at": [ + "survey_atomic_geography.py:96-103 `_require(type(config.support_path) is str and 0 < len(config.support_path) <= 4096 and \"\\0\" not in ... and Path(...).is_absolute() and \"..\" not in parts, \"SUPPORT_PATH\")` \u2192 ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_PATH\"); called from __post_init__ :87, to_bytes :91, reconstruct :407 and :602" + ], + "refusal_codes": [ + "SUPPORT_PATH" + ], + "encodes": "a single scalar: the support file path literal carried into the config document (`canonical_json({... \"support_path\": ...})` :125-133)", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "the pilot path length (tens of characters); does not move with the fraction", + "bytes_at_full_source": "same", + "measurement_basis": "scalar length check; no roster behind it", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4096 (identical on origin/native-row-ceilings)", + "head_value": "4096", + "reachable_from": "19", + "reach_chain": "AtomicSurveyReconstruction(...) __post_init__ :86-87 \u2192 _config_bytes :94; reconstruct_atomic_survey_geography:407 (19 via financial:1355 \u2192 atomic:169; age :447/:457 and :442-446 to_bytes; 45 via financial:1320).", + "classification": "must-not-move", + "notes": "Protects a token/scalar. The seed range check at :124 (`0 <= config.seed < 2**63`, \"SEED\") is a value range, not a byte bound, and is not a row." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:116 0 < len(pair[1]) <= 4096", + "value": "4096 characters per source identity string (three strings: district, population, puma)", + "enforced_at": [ + "survey_atomic_geography.py:109-123 `_require(type(config.source_ids) is tuple and len(config.source_ids) == 3 and all(... 0 < len(pair[1]) <= 4096 and pair[1].strip() == pair[1] ...) and tuple(pair[0] ...) == (\"district\", \"population\", \"puma\"), \"SOURCE_IDENTITIES\")` \u2192 ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SOURCE_IDENTITIES\")" + ], + "refusal_codes": [ + "SOURCE_IDENTITIES" + ], + "encodes": "three scalars: the publisher source identities written into the config document (:130) and the assignment definition (:427 `source_ids=dict(config.source_ids)` \u2192 atomic_block_support.assignment_definition:223 `_sources`)", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "three short identity strings; does not move with the fraction", + "bytes_at_full_source": "same", + "measurement_basis": "scalar length check", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4096 (identical on origin/native-row-ceilings)", + "head_value": "4096", + "reachable_from": "19", + "reach_chain": "same as :98 (_config_bytes from __post_init__ and reconstruct:407/:602).", + "classification": "must-not-move", + "notes": "Protects a token/scalar. atomic_block_support._sources (:93-102) re-checks non-emptiness only, no length." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:263 len(part).to_bytes(8, \"little\")", + "value": "8-byte little-endian length prefix: 2**64 - 1 = 18,446,744,073,709,551,615 per storage part", + "enforced_at": [ + "survey_atomic_geography.py:262-264 in _population_stamp: `for part in _storage_parts(series, slice(None)): digest.update(len(part).to_bytes(8, \"little\")); digest.update(part)` (OverflowError from int.to_bytes if exceeded; no refusal code)", + ":265-267 `digest.update(canonical_json((entity, str(values.dtype), values.shape))); digest.update(values.tobytes())` for design weights (no length prefix; tobytes() copies the array)" + ], + "refusal_codes": [ + "(none; OverflowError from int.to_bytes)" + ], + "encodes": "an encoder width in a streaming seal: every column's raw storage parts of the population frame at each stage (stacked rows for create/allocate/observed_geography, clone rows for expand/claim/assign/derive/gate), plus strata and design weights, streamed into one sha256", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "largest single part at 1/10: an 8-byte column at 713,002 clone persons = 5,704,016 bytes (the width admits 2**64-1)", + "bytes_at_full_source": "largest single part: an 8-byte column at 7,130,026 clone persons = 57,040,208 bytes; design-weight tobytes() copy at 1,587,376 households = 12,699,008 bytes", + "measurement_basis": "rows \u00d7 dtype width from brief \u00a73 counts; no whole copy of the frame is made (parts are the live buffers, only tobytes() at :267 copies)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "8 bytes (identical on origin/native-row-ceilings)", + "head_value": "8 bytes", + "reachable_from": "19", + "reach_chain": "_population_stamp is called at reconstruct:418, :542-ish via stages (:562-564, :585, :590, :613-616) and by every runner: graph_atomic_survey_population.py:215/:218/:229/:320/:329-330, graph_atomic_survey_financial.py:152/:1379/:1645/:1809/:1928/:1943, survey_age_calibration via atomic_source, graph_survey_completion_host:255/:511/:514.", + "classification": "must-not-move", + "notes": "No accumulation: digest.update per part. The docstring (:231-238) says the seal folds physical storage parts and is never persisted. Recorded as a width row because the brief lists struct widths; it cannot bind at any survey scale." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:542 canonical_json(receipt) (stage receipt; the EXPAND stage's carries receipt[\"expand\"] = expand_lineage_receipt(lineage) from :342)", + "value": "none (canonical_json has no cap: canonical.py:43-58 json.dumps whole)", + "enforced_at": [ + "survey_atomic_geography.py:540-544 `stages.append(AtomicSurveyGeographyStage(node, current, canonical_json(receipt), artifacts))` for every ordered node; the EXPAND stage's receipt is built at :339-350 by _clone_expectations: `receipt = clone.USCombinedSurveyCloneExpandKernel._receipt(...)`; :342 `receipt[\"expand\"] = expand_lineage_receipt(lineage)`; :344-350 `receipt[\"expand_writes\"] = expand_writes_receipt(...)`", + "consumer decode of the same bytes: graph_atomic_survey_population.py:212 `{stage.node.id: json.loads(stage.receipt) for stage in geography.stages}` \u2192 _states:105 `dict(raw_receipts[node_id])` \u2192 graph_survey_population._check_node_states:922 `node.receipt == state[\"receipt\"]` (dict equality, no re-encode)", + "refusal on encode: canonical.py:25 ValueError(\"non-finite numbers are not canonical JSON\"), :33 TypeError(\"canonical mappings require string keys\"), :38-40 TypeError (unsupported type) \u2014 none size-based" + ], + "refusal_codes": [ + "(none; no cap)" + ], + "encodes": "a per-row roster document: the combined-clone EXPAND receipt whose `expand` key lists one [new_id, copied_id] pair per row of every entity of the expanded frame (population.py:424-449 `payload[entity] = [[target, source] ...]`, entities from survey_atomic_geography:332-335 `for entity in US_SCHEMA.entities`): clone households, clone persons, clone tax units, clone families, clone marital units, clone SPM units; plus the fixed clone facts (:626-657)", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "45,052,091 bytes for the lineage part at 1/10 widths (2,650,123 pairs \u00d7 17 B: households 317,475 + persons 713,003 + tax units 426,507 SCALED + families 319,480 SCALED + marital units 555,982 SCALED + SPM units 317,676 SCALED)", + "bytes_at_full_source": "503,523,275 bytes for the lineage part (26,501,225 pairs \u00d7 19 B at 8-digit clone ids over 7-digit native ids: households 3,174,752 + persons 7,130,026 + tax units 4,265,071 SCALED + families 3,194,795 SCALED + marital units 5,559,825 SCALED + SPM units 3,176,756 SCALED); the fixed remainder is under 5 KB", + "measurement_basis": "scratchpad geo_byte_census_measure.py M4: canonical_json on invented pairs, difference of 1,000 vs 2,000 rows = 19.0 B/pair (8/7 digits), 17.0 (7/6 digits, 1/10), 15.0 at the 1/1000 observed widths; corroborated by the recovered 1/1000 manifest: combined_survey_puf_support_clone receipt 198,514 B holding 13,129 pairs (family 1594, household 1584, marital_unit 2774, person 3464, spm_unit 1585, tax_unit 2128), sample [[1000001, 1], ...]; clone id width from puf_support.py:2324-2345 (id + clone_index * 10**digits(max_id))", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (identical on origin/native-row-ceilings)", + "head_value": "none", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 \u2192 run_atomic_survey_population:169 (and :293; financial:1408) \u2192 reconstruct_atomic_survey_geography \u2192 :483 _clone_expectations \u2192 :342 expand_lineage_receipt \u2192 :542 canonical_json(receipt) \u2192 graph_atomic_survey_population:212 json.loads(stage.receipt). Age: survey_age_calibration:447/:457. 45: financial:1320.", + "classification": "stays", + "notes": "No bound to meet, so it cannot bind; but it is a whole-roster canonical document held whole (about 0.50 GB at full source) three times per 19-node run in this module alone, then decoded whole again at graph_atomic_survey_population:212. The document is another family's (graph_combined_clone._receipt + population.expand_lineage_receipt); this row is the consumer-side re-encoding the brief asks for, with the encoder's cap recorded as none. The same document is also materialised by store.put_json and RunManifest (rows below) and by graph_combined_clone's own lineage digest (:217 canonical_json(payload)) and survey_age_calibration._node_identity:80-98 (includes node.receipt) \u2014 both outside this family. Whether an uncapped materialisation of this size should take the segmented shape is an open question for the lane, not a refusal at any fraction." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/store.py:1099 json.loads(_canonical_json(dict(payload))) in ContentStore.put_json (with :1102 _write_json \u2192 :121 _canonical_json again)", + "value": "none (_canonical_json :99-111 is json.dumps whole; put_json/put_bytes/put_frame carry no byte cap)", + "enforced_at": [ + "store.py:1097-1105 put_json: `normalized = json.loads(_canonical_json(dict(payload)))` then `_write_json(root / \"payload.json\", normalized)` \u2192 :120-121 `_write_bytes(path, _canonical_json(value))`; refusal only :108-111 TypeError(\"Store metadata must be finite, JSON-compatible data.\")", + "executor.py:1681-1708 _write_node builds `record = {... \"receipt\": dict(receipt), ...}` and :1703 `store.put_json(_cache_record_key(key), record, node_key=key, ...)` \u2014 for the EXPAND node the receipt carries the lineage appended at executor.py:1355 `receipt[\"expand\"] = expand_lineage_receipt(result.expand)` and :2910 `normalized_receipt[\"expand_writes\"]`", + "cache-hit read of the same bytes: executor.py:1860 _load_record \u2192 store.load_json:1107-1115 \u2192 _load_json_file:212-225 json.loads whole (no cap); executor.py:2323 `record[\"receipt\"] != receipt` (dict compare)", + "store.py:141-153 _payload_table records sha256 + size per file and :228-292 _verified_meta re-hashes every payload on each read (1 MiB chunks at :136, streaming)" + ], + "refusal_codes": [ + "(none size-based; TypeError \"Store metadata must be finite, JSON-compatible data.\")" + ], + "encodes": "every node's cache record (fixed for 18 of the 19 nodes; the recovered 1/1000 receipts are 471 B to 29,387 B) and, for combined_survey_puf_support_clone, the per-row EXPAND lineage roster (clone households, persons, tax units, families, marital units, SPM units) \u2014 the same document as the survey_atomic_geography:542 row", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "45,052,091 (lineage part; held as one bytes object twice: the normalisation at :1099 and the write at :1102)", + "bytes_at_full_source": "503,523,275 (lineage part), plus the same again on every cache-hit load", + "measurement_basis": "same canonical encoder and pair widths as the survey_atomic_geography:542 row (store._canonical_json uses the identical json.dumps parameters: sort_keys, separators (\",\", \":\"), ensure_ascii=False, allow_nan=False); 1/1000 manifest receipt 198,514 B for 13,129 pairs", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (identical on origin/native-row-ceilings)", + "head_value": "none", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 \u2192 run_atomic_survey_population:231 run_graph \u2192 executor._execute_graph:2784 kernel.run (combined_survey_puf_support_clone) \u2192 :1355 lineage appended in _validate_result \u2192 :2943 _write_node \u2192 :1703 store.put_json \u2192 store.py:1099/:1102. Same for age (survey_age_calibration:447) and 45 (financial:1320).", + "classification": "stays", + "notes": "docs/us-native-scale-transport.md:79 already records that put_bytes and put_frame carry no byte cap; put_json is the same. The store's own integrity table (:141-153) is a per-file sha256/size, not a bound. Executor.py:111 `_normal_json_mapping` (`json.loads(canonical_json(value))`) re-encodes the kernel's receipt at :331 before the lineage is appended at :1355, so on this path that earlier re-encode is fixed-shape (largest kernel receipt at 1/1000: survey_predictors.attach 29,387 B); see open questions for the asec_design_donor `selection` key." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/manifest.py:661 canonical_json(self.content_addressed) (RunManifest.key), with :731 canonical_json(payload) in to_json and :751 json.loads in from_json", + "value": "none (no cap anywhere in manifest.py; canonical_json and json.loads whole)", + "enforced_at": [ + "manifest.py:659-661 `key` property: `sha256_domain(\"manifest\", canonical_json(self.content_addressed))` \u2014 content_addressed (:578-591) holds every node's `_content_payload()` including `receipt` (:387, :417-428)", + "manifest.py:709-731 to_json: payload carries `\"key\": self.key` (:718, one full encode), `\"content_addressed\": self.content_addressed` (:721) and `\"nodes\": {... receipt._payload()}` (:723-725), then :731 `canonical_json(payload)` \u2014 the lineage roster appears twice inside one document and a third time for the key", + "manifest.py:744-830 from_json: :751 json.loads whole; :805-807 `recomputed_body = manifest.content_addressed` and `sha256_domain(\"manifest\", canonical_json(body))`; :976-978 `canonical_json(body_nodes[node_id]) != canonical_json(expected_nodes[node_id])` per node (re-encodes each receipt twice more); :833-893 load reads the file whole (:838) and :851-856 re-canonicalises raw_body", + "manifest.py:277 `_freeze_json(self.receipt)` in NodeReceipt.__post_init__ copies the receipt into MappingProxyType/tuples (in-memory copy of the roster, not bytes)", + "refusals: canonical.py:25/:33/:38-40 (type/finite only); :114 ValueError(\"manifest values must be finite\"), :120/:125 TypeError \u2014 none size-based" + ], + "refusal_codes": [ + "(none; no cap)" + ], + "encodes": "the run manifest: every node receipt, including the combined_survey_puf_support_clone EXPAND receipt with its per-row lineage roster (clone households, persons, tax units, families, marital units, SPM units)", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "45,052,091 per copy of the lineage part; to_json holds it twice in one document plus one more encode for key = about 135 MB of lineage bytes in flight, plus the str\u2192bytes copy at to_json_bytes:734", + "bytes_at_full_source": "503,523,275 per copy; to_json_bytes \u2248 1.01 GB of lineage in the document string plus its UTF-8 copy, plus the key encode", + "measurement_basis": "same canonical_json pair cost (19.0 B at full widths, 17.0 at 1/10) \u00d7 the \u00a73 clone counts (tax unit/family/marital/SPM SCALED); recovered 1/1000 manifest.json is 950,156 B of which the clone receipt is 198,514 B, consistent with 2\u00d7 (nodes + content_addressed) plus fixed nodes", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (identical on origin/native-row-ceilings)", + "head_value": "none", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1374 `prefix_manifest_bytes = prefix.manifest.to_json_bytes()` \u2192 manifest.py:733-734 \u2192 to_json:709-731 \u2192 key:661 (19 and 45 through the same base run). The age runner reaches manifest.py only through RunManifest/NodeReceipt construction (executor.py:3054, _freeze_json copy) and node.key; it does not call key/to_json (survey_age_calibration greps: node identities only, :78-98, :275, :546, :713, :723).", + "classification": "stays", + "notes": "RunManifest docstring (:435-445): the content-addressed body is every complete node receipt less run-level fields. Because the executor authors the lineage into the receipt (executor.py:1355) rather than into an artifact, the portable manifest and its key are roster-shaped by construction; a manifest saved with RunManifest.save (:736-741) writes the whole string. RunManifest.load (:833) re-reads it whole and from_json re-canonicalises each node twice (:976-978). No refusal at any fraction; memory only." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/executor.py:111 json.loads(canonical_json(value)) in _normal_json_mapping", + "value": "none (canonical_json whole; no cap)", + "enforced_at": [ + "executor.py:107-118 `_normal_json_mapping(value, label)`: `restored = json.loads(canonical_json(value))`; refusal :113-115 NodeRejected(f\"{label} must contain canonical JSON values: {error}\") on TypeError/ValueError only", + "executor.py:331 `receipt = _normal_json_mapping(result.receipt, f\"Node {node.id!r} receipt\")` in _validate_result, for every executed node's kernel-authored receipt (before :1355 appends the EXPAND lineage)" + ], + "refusal_codes": [ + "(none size-based; NodeRejected \"... must contain canonical JSON values: ...\")" + ], + "encodes": "every kernel-authored receipt on the path, re-encoded to detach it. On the 19-node path these are fixed-shape at the executor's re-encode point: the recovered 1/1000 kernel receipts are 471 B (create) to 29,387 B (survey_predictors.attach); the clone kernel's own receipt (graph_combined_clone._receipt :612-657: counts, digests, provenance counts) is fixed, the roster lineage is appended only afterwards at :1355", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "under 30 KB per receipt for the geography kernels (observed_geography 2,071 B; support 1,190; assign 761; derive 1,519; gate 1,307 at 1/1000; only integer counts and digests change)", + "bytes_at_full_source": "same order (digits change only)", + "measurement_basis": "recovered 1/1000 manifest.json receipt sizes; receipt shapes read at graph_current_survey_geography.py:226-233, graph_atomic_geography.py:80-84, survey_atomic_geography.py:497-528", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (identical on origin/native-row-ceilings)", + "head_value": "none", + "reachable_from": "19", + "reach_chain": "run_graph \u2192 _execute_graph:2830 _validate_result \u2192 :331 _normal_json_mapping for every node (19, 45, age all pass through run_graph: graph_atomic_survey_population:231, graph_atomic_survey_financial:1647, graph_survey_completion_host:786, survey_age_calibration via atomic_source:447).", + "classification": "stays", + "notes": "Recorded as the family's consumer-side re-encoding of kernel receipts. It is fixed-shape for every geography receipt; the only roster-shaped receipt on the path (the EXPAND lineage) is authored after this point. One receipt outside this family may be roster-shaped at this site: survey_predictors.asec_design_donor carries a `selection` key (12,876 B at 1/1000) \u2014 see open questions." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/executor.py:436 len(payload).to_bytes(8, \"little\") (with :457-458 np.full(rows, width, dtype=\" len(relative).to_bytes(8, \"little\") and :48 len(content).to_bytes(8, \"little\") in _directory_identity", + "value": "8-byte little-endian length prefixes: 2**64 - 1 per member name and per member content", + "enforced_at": [ + "keys.py:31-51 _directory_identity: `content = candidate.read_bytes()` (:45, whole file, no cap) then :46-49 prefix + name, prefix + content into sha256; OverflowError only", + "keys.py:54-71 source_content_key: regular file `content = source_path.read_bytes()` (:64, whole, no cap) \u2192 sha256; directory \u2192 _directory_identity", + "executor.py:2100-2104 _source_paths_and_keys \u2192 source_content_key/identities.key; :2824 per-node recheck for nodes declaring sources; :3030-3031 run-end `source_identities.derive` (cache bypassed) and :2511 _settle_failed_run" + ], + "refusal_codes": [ + "(none; OverflowError from int.to_bytes)" + ], + "encodes": "an encoder width over upstream files: the raw-bytes source `us_atomic_block_support` (28,862,508 B, read whole at :64) and every member of the `survey_population_source` directory (the ACS zips, ASEC h5/csv files listed at survey_population_preparation.py:62-73, each read whole at :45)", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "28,862,508 B for the support file (one whole read_bytes per derivation); the survey source directory members are the input archives' own sizes, identical at every fraction (not measured here; the coverage lane measured the uncompressed person archive at 2,405,464,349 B)", + "bytes_at_full_source": "same (upstream sizes do not move with the fraction)", + "measurement_basis": "stat of the support file; keys.py:45/:64 read_bytes has no cap; brief \u00a74 forbids reading the gated survey archive here", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "8 bytes (identical on origin/native-row-ceilings)", + "head_value": "8 bytes", + "reachable_from": "19", + "reach_chain": "run_graph \u2192 _execute_graph:2638 _source_paths_and_keys \u2192 keys.source_content_key:54 (run start), :2824 (after each source-declaring node: survey_population.create, geography.support.0), :3031 run end (19: graph_atomic_survey_population:221 also calls _source_paths_and_keys directly; 45 and age through the same runs).", + "classification": "must-not-move", + "notes": "The width never binds. What a reviewer should see is that source identity reads each upstream member whole with no cap, at least three times per run (start, per-node, end) \u2014 an upstream materialisation rather than a roster transport; the sizes are the input files' own." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/store.py:68 _KEY = re.compile(r\"[0-9a-f]{64}\\Z\")", + "value": "exactly 64 lowercase hex characters (a fixed-width field)", + "enforced_at": [ + "store.py:193-196 _require_key \u2192 ValueError(\"Content-store keys must be 64 lowercase hexadecimal digits.\") (object_path :706, evict :770, _put :800)", + "store.py:243-244 _verified_meta `metadata.get(\"key\") != path.name or _KEY.fullmatch(path.name) is None` \u2192 StoreCorrupt(\"Stored object ... does not bind its directory key.\")", + "store.py:261-262 payload table `_KEY.fullmatch(expected_hash) is None` \u2192 StoreCorrupt(f\"Payload {name!r} has an invalid SHA-256.\")" + ], + "refusal_codes": [ + "\"Content-store keys must be 64 lowercase hexadecimal digits.\" (ValueError)", + "\"... does not bind its directory key.\" (StoreCorrupt)", + "\"Payload ... has an invalid SHA-256.\" (StoreCorrupt)" + ], + "encodes": "a single token: every content-store object key and payload digest (sha256 hexdigests from keys.py)", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 characters, always", + "bytes_at_full_source": "64 characters", + "measurement_basis": "regex width", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 hex (identical on origin/native-row-ceilings)", + "head_value": "64 hex", + "reachable_from": "19", + "reach_chain": "every store put/load from executor._write_node (:1614/:1628/:1656/:1673/:1703) and _load_record/_load_cached_result (19, 45, age all through run_graph).", + "classification": "must-not-move", + "notes": "Protects a fixed-width field; listed for completeness." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/store.py:338 offsets = np.empty(len(chunks) + 1, dtype=np.int64) in _encode_offsets (utf8-offsets-v1 / object-scalars-v1)", + "value": "int64 offsets: 2**63 - 1 = 9,223,372,036,854,775,807 bytes of payload per string/object column", + "enforced_at": [ + "store.py:337-344 _encode_offsets: `payload = bytearray(); for chunk in chunks: payload.extend(chunk); offsets[index + 1] = len(payload)` then `np.frombuffer(bytes(payload), dtype=np.uint8).copy()` (the whole column payload is built twice: bytearray, then bytes, then a numpy copy)", + "store.py:461-480 _write_series StringDtype branch: `chunks.append(value.encode(\"utf-8\"))` per row, `_encode_offsets(chunks)`, values.npy/offsets.npy/mask.npy", + "store.py:481-486 object branch `_encode_object_scalar` per value \u2192 _encode_offsets", + "read side store.py:415-431 _validate_offsets requires dtype int64, monotone, last == payload length \u2192 StoreCorrupt(\"Stored offsets for ... are malformed.\"); :532-559 decodes each string from `raw = payload.tobytes()`" + ], + "refusal_codes": [ + "(no size refusal; StoreCorrupt \"Stored offsets for ... are malformed.\" on read shape only)" + ], + "encodes": "a numpy body per string column: put_column (executor.py:1614) writes survey_population.observed_geography's three string columns at stacked-household rows (survey_geography_origin_key, survey_observed_state, survey_observed_puma) and geography.assign/derive's string columns; put_frame (executor.py:1628, store.py:1194-1250 _write_frame) writes every string column of a structural version at clone rows (combined_survey_puf_support_clone) or filtered rows (survey_predictors.asec_design_donor)", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "survey_geography_origin_key column body 6,627,930 B at 158,737 stacked households (values + int64 offsets + mask), 13,255,852 B at 317,475 clone households", + "bytes_at_full_source": "66,279,466 B at 1,587,376 stacked households (ACS 33-byte keys \u00d7 1,531,614 + ASEC 26-byte keys \u00d7 55,762, + 8 B offset + 1 B mask per row); 132,558,924 B at 3,174,752 clone households; the width admits 2**63 - 1", + "measurement_basis": "scratchpad geo_byte_census_measure.py M5: store._encode_offsets on invented keys at the 1/1000-observed key widths (13-char SERIALNO, 5-char H_SEQ), difference of 1,000 vs 2,000 rows = 42.0 B/ACS row, 35.0 B/ASEC row", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "int64 (identical on origin/native-row-ceilings)", + "head_value": "int64", + "reachable_from": "19", + "reach_chain": "run_graph \u2192 _execute_graph:2943 _write_node \u2192 :1614 store.put_column (observed_geography, assign, derive) and :1628 store.put_frame (clone EXPAND, donor FILTER) \u2192 store.py:920/:1207 _write_series \u2192 :471/:483 _encode_offsets (19: graph_atomic_survey_population:231; 45 and age likewise).", + "classification": "must-not-move", + "notes": "Protects an encoder width (docstring :8-12: strings use one UTF-8 byte buffer plus int64 offsets and a null mask). No cap in front of it; put_frame at clone rows materialises each string column whole (bytearray + bytes + numpy copy). The largest string column on the path is the origin key measured here; other string columns (state 2 B, puma 7 B, provenance channel/source ids) are narrower." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:36 MAX_HOUSEHOLDS", + "value": "7_000_000 = 7,000,000 (a row count; comment :29-35 \"Four times [1,587,376], rounded up to the next whole million ... there is no byte transport behind this bound and it is an explicit row ceiling\")", + "enforced_at": [ + "current_survey_geography.py:73-78 `_require(type(origins) is list and 0 < len(origins) <= MAX_HOUSEHOLDS and len(origins) == len(households), \"HOUSEHOLD_COUNT\")` \u2192 ValueError(\"CURRENT_SURVEY_GEOGRAPHY_HOUSEHOLD_COUNT\") (_require :40-42)", + "current_survey_geography.py:190-198 `_require(... and 0 < len(household) <= MAX_HOUSEHOLDS and ..., \"PROJECTION_STORAGE\")` \u2192 ValueError(\"CURRENT_SURVEY_GEOGRAPHY_PROJECTION_STORAGE\")", + "attested, not enforced: survey_origin_budget.py:222 folds geography.observed.MAX_HOUSEHOLDS into `geography_contract`" + ], + "refusal_codes": [ + "HOUSEHOLD_COUNT", + "PROJECTION_STORAGE" + ], + "encodes": "a row count of the selected/stacked household roster (origins[\"households\"] from the preparation payload :235 and state.frame.table(\"household\") :246), pre-clone", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "n/a \u2014 row count: 158,737 selected households (2.3% of the ceiling)", + "bytes_at_full_source": "n/a \u2014 row count: 1,587,376 selected households (22.7%; headroom 4.41\u00d7)", + "measurement_basis": "brief \u00a73 counts; docs/us-native-row-ceilings.md:112 rule (4\u00d7 full source rounded up to the next million) and :228 (streams verdict)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "7,000,000 on origin/native-row-ceilings 0319b1c73 (git show confirms); experiments/native-row-ceilings/census.json still records the pre-lift 64 * 1024**2 // 128 = 524,288 and binds_at_full_source true \u2014 stale, superseded by its own verifier verdict", + "head_value": "7,000,000", + "reachable_from": "19", + "reach_chain": "reconstruct_atomic_survey_geography:431 observed.qualify_current_survey_geography \u2192 :244 _project (:75) and :261 _projection_digest (:195), again at :309/:317; and CurrentSurveyGeographyKernel.run:217 inside run_graph (registered graph_atomic_survey_population:200) \u2192 qualify \u2192 same sites; _check_projection:149 and run:274 call _projection_digest again. 19 via financial:1355 \u2192 atomic:169; age via :447/:457; 45 via financial:1320.", + "classification": "not-a-transport", + "notes": "A row count with no byte name; the row-count rule at docs/us-native-row-ceilings.md:105-116 and :214-228 applies (\"streams \u2014 moved in \u00a73\"). Listed because the brief counts record caps and because the row-ceilings census row for it is stale at this head." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:37 MAX_RECEIPT_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "current_survey_geography.py:262-290 `receipt = source._encode({...}, maximum=MAX_RECEIPT_BYTES)` \u2192 survey_population_preparation.py:116-122 _encode: `_require(len(result) + len(encoded) <= maximum, \"PAYLOAD_LIMIT\")` \u2192 SurveyPopulationPreparationError(\"PAYLOAD_LIMIT\") (ValueError subclass, :78; _require :85-87)", + "graph_current_survey_geography.py:133-141 _check_projection: `_require(... and type(projection.receipt) is bytes and 0 < len(projection.receipt) <= qualifier.MAX_RECEIPT_BYTES and source_graph._sha(projection.receipt) == receipt_sha256, \"PROJECTION_RECEIPT\")` \u2192 ValueError(\"SURVEY_GEOGRAPHY_PROJECTION_PROJECTION_RECEIPT\") (_require :41-43)", + "attested, not enforced: survey_origin_budget.py:223" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT", + "PROJECTION_RECEIPT" + ], + "encodes": "a fixed-shape receipt: 26 keys of protocol strings, four 64-hex digests, the three column names and five integer counts (households, acs_households, asec_households, state_unknown_households, puma_observed_households); no per-household value (:263-288)", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "1,223 bytes (through source._encode at 1/10 counts)", + "bytes_at_full_source": "1,228 bytes (1.87% of the cap; headroom 53.4\u00d7); 1,213 at 1/1000", + "measurement_basis": "scratchpad geo_byte_census_measure.py M1: the exact key roster of :263-288 encoded by source._encode(doc, maximum=MAX_RECEIPT_BYTES) at the \u00a73 counts; only integer digits move", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536 (identical on origin/native-row-ceilings)", + "head_value": "65,536", + "reachable_from": "19", + "reach_chain": "reconstruct_atomic_survey_geography:431 qualify_current_survey_geography:262 (encode) and :433/:619 observed_graph._check_projection:138 (len check); CurrentSurveyGeographyKernel.run:217/:218/:259 inside run_graph. 19 via financial:1355 \u2192 atomic:169/:293 and financial:1408; age via survey_age_calibration:447/:457; 45 via financial:1320.", + "classification": "stays", + "notes": "Fixed-shape, so it cannot bind at any survey scale; the consumer-side check at graph_current_survey_geography:138 is a plain len() on the issued bytes, not a re-encode. The kernel's own receipt (:226-234, canonical_json at :234 and :267 `canonical_json(result.receipt) == receipt_bytes`) embeds this document as `source_projection` and is likewise fixed (2,071 B at 1/1000); that :267 re-encode is not a bound and is not a row." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:60 0 < len(raw) <= 128 (NATIVE_KEY)", + "value": "128 characters of raw_native_id per origin row", + "enforced_at": [ + "current_survey_geography.py:59-60 `raw = row[\"raw_native_id\"]; _require(type(raw) is str and 0 < len(raw) <= 128, \"NATIVE_KEY\")` \u2192 ValueError(\"CURRENT_SURVEY_GEOGRAPHY_NATIVE_KEY\"), in _origin_key called per origin at :121 (_project), i.e. once per selected household per qualify" + ], + "refusal_codes": [ + "NATIVE_KEY" + ], + "encodes": "a single token per row: the source literal id inside the draw key `json.dumps([channel, source_year, survey_year, raw])` (:63-68); observed widths 13 (ACS SERIALNO) and 4-5 (ASEC H_SEQ) at 1/1000", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13 characters max observed (ACS); does not move with the fraction (ACS SERIALNO is a fixed 13-char literal per acs_household_archive serialno_widths {13: 1,631,969}; ASEC H_SEQ grows to at most 6 digits)", + "bytes_at_full_source": "13 (ACS), \u22646 (ASEC)", + "measurement_basis": "preparation.json raw_native_id lengths (acs 13/13, asec 4/5) and byte-census-measurements.json acs_household_archive.serialno_widths", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "128 (identical on origin/native-row-ceilings)", + "head_value": "128", + "reachable_from": "19", + "reach_chain": "reconstruct:431 qualify_current_survey_geography:244/:309 _project:121 _origin_key:60; and CurrentSurveyGeographyKernel.run:217 inside run_graph (19 via financial:1355; age :447/:457; 45 financial:1320).", + "classification": "must-not-move", + "notes": "Protects a token; it is what keeps the :202 per-row encode (next row) under 4096 even in the worst escaping case (960 B)." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:202 source._encode(values, maximum=4096)", + "value": "4096 bytes per encoded row [household_id, origin_key, state, puma]", + "enforced_at": [ + "current_survey_geography.py:200-204 _projection_digest: `for row in household.itertuples(...): values = [int(row[0]), *(None if pd.isna(v) else v ...)]; encoded = source._encode(values, maximum=4096); digest.update(len(encoded).to_bytes(4, \"big\")); digest.update(encoded)` \u2192 survey_population_preparation.py:120 \"PAYLOAD_LIMIT\" SurveyPopulationPreparationError", + "callers: current_survey_geography.py:261 and :317 (qualify, twice); graph_current_survey_geography.py:148-149 (_check_projection, `document[\"projection_sha256\"] == qualifier._projection_digest(projection.household)`) and :274 (FINAL_OUTPUT)" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT" + ], + "encodes": "one row at a time of the stacked-household projection (survey_geography_origin_key, survey_observed_state, survey_observed_puma keyed by household_id) streamed into a sha256; the roster is stacked households (ACS 1,531,614 + ASEC 55,762 at full source) but the bound is per row", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "per row 68 B (ACS, 7-digit household_id) / 56 B (ASEC); worst case 960 B for a 128-char raw id of control characters; streamed total per digest pass 10,727,204 B at 1/10", + "bytes_at_full_source": "same row widths (69 B with an 8-digit id); streamed total per pass 107,272,424 B; no whole copy exists", + "measurement_basis": "scratchpad geo_byte_census_measure.py M2: source._encode([hid, key, state, puma], maximum=4096) + 4-byte prefix per row on rows shaped from preparation.json (13-char SERIALNO, 5-char H_SEQ), and geo._projection_digest run on a 2-row table to prove the path", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4096 (identical on origin/native-row-ceilings)", + "head_value": "4096", + "reachable_from": "19", + "reach_chain": "reconstruct:431 qualify:261/:317 \u2192 _projection_digest:202; reconstruct:433/:619 observed_graph._check_projection:149; CurrentSurveyGeographyKernel.run:218/:259 (_check_projection) and :274 inside run_graph. 19 via financial:1355 \u2192 atomic:169/:293 and financial:1408; age :447/:457; 45 financial:1320.", + "classification": "must-not-move", + "notes": "docs/us-native-row-ceilings.md:228 already recorded this site as streaming; measured here: the widest possible row is 960 B (128 control chars escaped to \u0001 then re-escaped), 4.3\u00d7 under the cap. Each reconstruct chain runs the digest at least six times (two per qualify, one per _check_projection \u00d72, plus the kernel's three), each pass streaming ~107 MB at full source." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:203 len(encoded).to_bytes(4, \"big\")", + "value": "4-byte big-endian length prefix: 2**32 - 1 = 4,294,967,295 per row", + "enforced_at": [ + "current_survey_geography.py:203 in _projection_digest (OverflowError from int.to_bytes if exceeded; unreachable because :202 caps the row at 4096)" + ], + "refusal_codes": [ + "(none; OverflowError, unreachable behind the 4096 cap)" + ], + "encodes": "an encoder width on the per-row length prefix of the streamed projection digest", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "960 B maximum row (see the :202 row); the width admits 4,294,967,295", + "bytes_at_full_source": "same", + "measurement_basis": "scratchpad M2 widest row", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4 bytes (identical on origin/native-row-ceilings)", + "head_value": "4 bytes", + "reachable_from": "19", + "reach_chain": "same as the :202 row (every _projection_digest call).", + "classification": "must-not-move", + "notes": "Protects an encoder width; moving the 4096 cap above 2**32 - 1 would be the only way to reach it." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:213 _sha(values.household.reset_index(drop=True).to_json(orient=\"table\").encode()) == document.get(\"household_projection_sha256\")", + "value": "none (pandas to_json has no cap; the producer's encode at current_asec_demographics.py:362-364 is the same expression under source._encode's 64 MiB receipt cap for the surrounding receipt only)", + "enforced_at": [ + "current_survey_geography.py:208-216 _check_asec_projection: `_require(type(receipt) is bytes and values.receipt == receipt, \"ASEC_RECEIPT\")`; `_require(_sha(values.household.reset_index(drop=True).to_json(orient=\"table\").encode()) == document.get(\"household_projection_sha256\"), \"ASEC_PROJECTION_DIGEST\")` \u2192 ValueError(\"CURRENT_SURVEY_GEOGRAPHY_ASEC_PROJECTION_DIGEST\")", + "called at current_survey_geography.py:243 and :307 (twice per qualify)", + "producer: current_asec_demographics.py:269-281 builds htable (household_id, native_household_id, H_SEQ_integer, GESTFIPS, state_fips Int64, state_status, state_known) and :362-364 digests the identical to_json(orient=\"table\") bytes" + ], + "refusal_codes": [ + "ASEC_PROJECTION_DIGEST", + "ASEC_RECEIPT" + ], + "encodes": "a per-row roster document: the ASEC household demographic projection (ASEC households), re-encoded whole by pandas to_json(orient=\"table\") to recompute its digest", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "969,567 bytes at 5,576 ASEC households (174.0 B/row)", + "bytes_at_full_source": "9,747,693 bytes at 55,762 ASEC households; 9,903 at 1/1000", + "measurement_basis": "scratchpad geo_byte_census_measure.py M3: htable built exactly as current_asec_demographics.py:269-281 on invented rows (7-digit ids, 5-digit H_SEQ, state 06/in_printed_range), pandas to_json(orient=\"table\").encode(); per-row from 1,000 vs 2,000 rows and direct totals at the \u00a73 counts", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (identical on origin/native-row-ceilings)", + "head_value": "none", + "reachable_from": "19", + "reach_chain": "reconstruct:431 qualify_current_survey_geography:241 demographics.qualify_current_asec_demographics \u2192 :243/:307 _check_asec_projection:213; and CurrentSurveyGeographyKernel.run:217 inside run_graph. 19 via financial:1355 \u2192 atomic:169/:293 and financial:1408; age :447/:457; 45 financial:1320.", + "classification": "stays", + "notes": "A consumer-side re-encoding with no cap, held whole (9.7 MB at full source) twice per qualify; the roster is the ASEC channel only (3.5% of stacked households), so it is small. The producer (another family) materialises the same document; its receipt (which carries selected_household_ids and per-household state tokens, :339-358) is that family's transport." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py:63 0 < len(population) <= 256 (and :65 0 < len(node_id) <= 256)", + "value": "256 characters per node/population name", + "enforced_at": [ + "graph_current_survey_geography.py:61-68 current_survey_geography_node: `_require(type(population) is str and 0 < len(population) <= 256 and type(node_id) is str and 0 < len(node_id) <= 256 and population != node_id, \"NODE_NAMES\")` \u2192 ValueError(\"SURVEY_GEOGRAPHY_PROJECTION_NODE_NAMES\") (_require :41-43)" + ], + "refusal_codes": [ + "NODE_NAMES" + ], + "encodes": "two scalars: the declared population version name and node id (\"survey_population.allocate\" = 26 chars, \"survey_population.observed_geography\" = 36 chars)", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "36 characters; does not move with the fraction", + "bytes_at_full_source": "36", + "measurement_basis": "the literals at graph_current_survey_geography.py:31 and graph_survey_population.ALLOCATION_NODE as used at survey_atomic_geography.py:444", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "256 (identical on origin/native-row-ceilings)", + "head_value": "256", + "reachable_from": "19", + "reach_chain": "reconstruct_atomic_survey_geography:441 observed_graph.current_survey_geography_node; graph_current_survey_geography._check_context:95 (called from run:210/:253) inside run_graph. 19 via financial:1355; age :447/:457; 45 financial:1320.", + "classification": "must-not-move", + "notes": "Protects tokens; listed for completeness. graph_geography.py (PUMA-ladder kernels) defines no byte bound of its own: its only byte site is load_source_bytes(\"raw-bytes-v1\") at :124-129 (recorded under codecs.py:78) and its canonical_json(document) at :249 re-encodes the frame_context, which graph_context._row_identity (:104-120) keeps fixed-shape (rows, id_dtype, ordered_ids_sha256); reachable_from none (only graph_composed_population:711-727 and graph_composed_asec_measures:614-619 use us_geography_nodes)." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/geo/geo_byte_census_measure.py (run with /Users/maxghenis/PolicyEngine/_worktrees/microcosm-native-byte-transports/.venv/bin/python 3.14.4, sys.path = packages/*/src; every imported module resolved inside the worktree)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/geo/geo-byte-census-measurements.json (receipt: receipt 1,213/1,223/1,228 B; projection rows 68/56/960 B, streamed 10,727,204 / 107,272,424; ASEC projection 174.0 B/row, 969,567 / 9,747,693; lineage 17.0/19.0 B per pair, 45,052,091 / 503,523,275; origin-key store column 42.0/35.0 B per row, 6,627,930 / 66,279,466 stacked and 13,255,852 / 132,558,924 clone)" + ], + "open_questions": [ + "Uncapped whole-roster materialisation of the combined-clone EXPAND lineage receipt (about 0.50 GB at full source, 26.5M pairs incl. four SCALED entities) at four sites in this family with encoder cap none \u2014 survey_atomic_geography.py:542 (three reconstruct calls per 19-node run), store.put_json:1099/:1102 (every write and every cache-hit load_json), RunManifest.key:661 and to_json:731 (held twice in one document; graph_atomic_survey_financial:1374 calls to_json_bytes on the 19/45 path) \u2014 plus two outside it (graph_combined_clone lineage digest :217; survey_age_calibration._node_identity:80-98 on the age path). No refusal at any fraction, so no `binds`; whether the lane wants the segmented shape for an uncapped document this size, or an artifact instead of a receipt, is its call. The 1/1000 manifest corroborates the shape (198,514 B clone receipt, 13,129 pairs) \u2014 I read manifest.json's receipt sizes and lineage counts from the recovered run directory although \u00a74 lists only preparation.json there.", + "Lineage byte cost assumes puf_support's decimal remap (puf_support.py:2324-2345, id + clone_index * 10**digits(max_id)) yields 8-digit clone targets over 7-digit natives at full source and 7 over 6 at 1/10; the stacked household id scheme (ASEC receiving ids at 55,002 in the 1/1000 artifact) may widen ids by a digit, adding at most 1 B per pair.", + "executor.py:111 _normal_json_mapping re-encodes every kernel receipt with no cap; on the 19-node path the only receipt that may be roster-shaped at that point is survey_predictors.asec_design_donor (FILTER) whose `selection` key is 12,876 B at 1/1000 \u2014 owned by the predictors family, not measured here.", + "keys.py source identity (source_content_key:64, _directory_identity:45) reads each upstream survey-source member whole with read_bytes and no cap, at run start, after each source-declaring node, and at run end (executor.py:2638/:2824/:3031); the ACS/ASEC archive sizes were not measured (brief \u00a74 forbids the gated data) and the transport is upstream-sized, not roster-sized.", + "graph_geography.py (assign_us_puma_ladder.* nodes) is reachable from none of the three runners (only graph_composed_population and graph_composed_asec_measures declare us_geography_nodes); its two raw-bytes sources (us_puma_ladder_2020 NPZ, us_cd_crosswalk_117_119 CSV) were not sized. congressional_district_geography, puma_ladder, geography_ladder, atomic_block_support, canonical, serialize contain no byte bound (value-range checks such as `< 2**53`, `< 2**63`, block/puma/district integer ranges and NPZ np.load were read and excluded as not byte bounds).", + "experiments/native-row-ceilings/census.json is stale for current_survey_geography.py:36 MAX_HOUSEHOLDS (records 524,288 and binds_at_full_source true; base and head are both 7,000,000, docs/us-native-row-ceilings.md:112) and cites the survey_atomic_geography _require helper at :67-69 (it is :68-70); no geography document appears in byte-census-measurements.json, so every byte figure here is from the scratchpad script.", + "store.put_frame (executor.py:1628) writes the whole clone-version frame with no byte cap (docs/us-native-scale-transport.md:79); only its string-column int64-offset width was measured (origin key 132.6 MB at clone households); numeric columns are exact numpy buffers (rows \u00d7 dtype width) and were not enumerated column by column." + ] + }, + { + "family": "Preparation and population graph", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py (2195 lines, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py (1399 lines, full; working tree differs from HEAD 77bafbb3a: _json_matches added, _json_sha256 signature changed)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_domains.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_sources.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/support_provenance.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py (14231 lines, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py (full)", + "partial, for reachability: graph_atomic_survey_financial.py:1275-1420 and grep; graph_survey_completion_host.py grep; survey_age_calibration.py:394-560; graph_puf_diagnostic_consumer.py:370-410; current_survey_predictors.py:500-530; current_survey_amounts.py:355-375", + "partial, for encoder caps: packages/microcosm-graph/src/microcosm/graph/canonical.py (full); graph/store.py grep (put_bytes/load_bytes carry no byte cap); operator_column_contracts.py:55-70; puf_support.py:2316-2328", + "inputs: experiments/native-byte-transports/census-brief.md, byte-census-measurements.json, byte_census_measure.py; experiments/native-row-ceilings/census.json, roster-census.json; experiments/native-scale-transport/ceiling-receipt.json; docs/us-native-scale-transport.md section 2; docs/us-native-row-ceilings.md; git diff origin/native-row-ceilings and git diff HEAD for the family" + ], + "rows": [ + { + "constant": "survey_population_preparation.py:49 MAX_REQUEST_BYTES", + "value": "4096 (literal) = 4,096", + "enforced_at": [ + "survey_population_preparation.py:456 REQUEST_FILE SurveyPopulationPreparationError(ValueError) (lstat st_size <= 4096 before open)", + "survey_population_preparation.py:458-459 read cap: _regular_reader(path, MAX_REQUEST_BYTES) then stream.read(MAX_REQUEST_BYTES + 1); _regular_reader refuses st_size > maximum at :426 SOURCE_REGULAR_FILE SurveyPopulationPreparationError", + "survey_population_preparation.py:460 REQUEST_LIMIT SurveyPopulationPreparationError", + "survey_population_preparation.py:487 REQUEST_CANONICAL SurveyPopulationPreparationError: consumer re-encoding _encode(document, MAX_REQUEST_BYTES) == payload, whose own refusal is :120 PAYLOAD_LIMIT at the same 4,096", + "survey_population_preparation.py:512 as the _file_limits entry for selection-request.json, enforced by _source_files at :561 SOURCE_FILE_LIMIT and through _regular_reader at :426 SOURCE_REGULAR_FILE" + ], + "refusal_codes": [ + "REQUEST_FILE", + "REQUEST_LIMIT", + "REQUEST_CANONICAL", + "PAYLOAD_LIMIT", + "SOURCE_REGULAR_FILE", + "SOURCE_FILE_LIMIT" + ], + "encodes": "an upstream file: /selection-request.json, a four-field JSON document {protocol, declaration, fraction:[num,den], seed}; also its canonical re-encoding", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "~168 B (the recovered artifact's source_files[0] records 168 B per the row-ceilings census; the artifact is absent from this worktree so not re-read here). Only the fraction pair's digits change with the fraction.", + "bytes_at_full_source": "~168 B (fraction [1,1] is shorter than [1,10])", + "measurement_basis": "row-ceilings census reading of _recovered preparation.json source_files[0]; document shape read from _request :461-486 (four fields, fraction pair of two ints, seed < 2**64)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 -> graph_atomic_survey_population.run_atomic_survey_population:157 -> graph_survey_population.run_authenticated_survey_population:1012 -> survey_population_preparation.prepare_authenticated_survey_population:1901 -> _request:451; also survey_population_source_codecs.load:745 read_survey_population_request:491 -> _request. Same prefix serves 45 (graph_survey_completion_host.extend over run_atomic_survey_financial:1319) and age (survey_age_calibration:427/:444).", + "classification": "must-not-move", + "notes": "Protects a fixed-shape upstream file with >24x headroom. Quoted refusal: `_require(len(payload) <= MAX_REQUEST_BYTES, \"REQUEST_LIMIT\")` (:460). Also sealed into the producer identity at :685 (identity only, not an enforcement). Base value = head value (unchanged on this branch)." + }, + { + "constant": "survey_population_preparation.py:50 MAX_PAYLOAD_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "survey_population_preparation.py:116/:120 PAYLOAD_LIMIT SurveyPopulationPreparationError: default `maximum` of _encode, reached for every non-roster document and per-row encode: _producer() (:670, :1920, :1795, :1734), _authority() (:700), spill header.json (:237), each _bounded_append row (:1151), _frame_identity header lists (:986-1044), _nested_seals snapshot list (:1650), _request canonical (:487 with 4096)", + "survey_population_preparation.py:751/:787 PAYLOAD_LIMIT SurveyPopulationPreparationError: _frame_cell_encode, one normalized Frame cell (fast path bound; slow path :769 is _encode(value, maximum))", + "current_survey_predictors.py:522 (other family, 19-node node survey_predictors.source_projection): passed as `segment=source.MAX_PAYLOAD_BYTES` to host.survey_graph._segmented_json, i.e. the per-segment ceiling of the predictor projection; refuses TRANSPORT_LIMIT in graph_survey_population.py:331 SurveyPopulationGraphError", + "current_survey_amounts.py:369 (not on either path): the _bounded_json limit of the amounts projection" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT", + "TRANSPORT_LIMIT" + ], + "encodes": "in this module: single fixed documents (the ~21 KB _producer() record, the ~2 KB spill header, the 168 B request), single rows (24-247 B) and single cells (<= 6*4096+2 B); out of family: the per-segment width of the predictor projection whose roster is stacked persons (24.04 B per origin row per byte-census-measurements.json predictor_projection)", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "largest document this module encodes under it: _producer() ~21,135 B (row-ceilings census); as the predictor projection's segment: 8,570,284 B total at 1/10 (byte-census-measurements.json predictor_projection.tenth_origins_bytes), one segment", + "bytes_at_full_source": "same fixed documents; as the predictor projection's segment: 85,702,912 B total at full source (byte-census-measurements.json), i.e. two segments; no single accumulation exceeds 64 MiB", + "measurement_basis": "byte-census-measurements.json predictor_projection (per_person_origin_row_bytes 24.04 through graph._bounded_json); row-ceilings census for _producer(); prep-measurements.json per-row encodes (24-247 B)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1892 -> _encode(_producer()):1920 etc.; _frame_identity:980 -> _cells_blob:949/_frame_cell_encode:751 from :2012 and _pure_final:1760; out of family: run_atomic_survey_financial -> current_survey_predictors.qualify_current_survey_predictors -> _segmented_json(segment=source.MAX_PAYLOAD_BYTES) :512-523", + "classification": "must-not-move", + "notes": "This is the encoder's accumulation width; the whole-roster preparation receipt no longer passes through _encode (it goes through _roster_segments under MAX_SEGMENT_BYTES, the alias at :57). Sealed at :686. The predictor projection that meets 64 MiB at full source is that family's segment row; this row records the enforcement site. Base value = head value." + }, + { + "constant": "survey_population_preparation.py:57 MAX_SEGMENT_BYTES", + "value": "MAX_SEGMENT_BYTES = MAX_PAYLOAD_BYTES = 67,108,864", + "enforced_at": [ + "survey_population_preparation.py:163-164 segment close (`if current and len(current) + len(encoded) > segment: close()`), then :165 PAYLOAD_LIMIT SurveyPopulationPreparationError on a single token larger than one whole segment; reached from _roster_payload:241-253 at :2013 (preparation receipt, spill=snapshots) and :2180 (wage projection)" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT" + ], + "encodes": "one segment of the whole-roster preparation receipt: rosters selection.selected (stacked households), origins.households (stacked households), origins.persons.rows (stacked persons), origins.entities (household, person, and SCALED tax_unit/family/marital_unit/spm_unit rows); fixed blocks selection.excluded (all catalogue exclusions), selection.cells, storage_transitions, producer, catalogues, native, source_files, request, digests. Also one segment of the ASEC wage projection (rows = ASEC persons).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "receipt 126,660,764 B at full-source id widths (this lane; 2 segments) = 1.89x one segment; transport lane's replicated 1/1000-width artifact 109,804,304 B (ceiling-receipt.json, 2 segments). Wage projection 667,964 B (14,212 rows x 47 B), one segment.", + "bytes_at_full_source": "receipt 1,251,912,520 B (this lane; 18.65x, 19 segments); transport lane 1,099,892,722 B (17 segments). Per selected household 788.6 B at full widths (selected 238/235 B, households 179/174 B, persons 76/86 B x 2.19, entity rows 24 B x 6.9). Wage projection 6,679,875 B (142,125 x 47 B).", + "measurement_basis": "scratchpad/census/prep/prep-measurements.json: per-row bytes through prep._encode and prep._plan_document on invented rows at full-source widths (7-digit stacked/native ids, 13-char SERIALNO, 22-char PERIDNUM, 4-digit WGTP, unreduced 1/10 inclusion fractions), marginal check by difference of two row counts (matches row+comma), roster totals at the roster-census counts, plus the row-ceilings census's 45,922 B fixed preamble; _roster_segments run on a 20,000-row synthetic document with segment=1 MiB: stream total == len(_encode), digest == sha256(_encode) == _digest, joined == single, every segment under the segment size. Cross-checked against ceiling-receipt.json.", + "binds_at_tenth": true, + "binds_at_full_source": true, + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 -> run_atomic_survey_population:157 -> run_authenticated_survey_population:1012 -> prepare_authenticated_survey_population:2013 -> _roster_payload:241 -> _roster_segments:132 (:165). Wage projection: only graph_puf_diagnostic_consumer.qualify_current_survey_host:393 (callers graph_current_survey_puf_transfer:224/:380 and graph_puf_diagnostic_consumer:569/:751), reachable_from none.", + "classification": "segment", + "notes": "'Binds' here means the document exceeds one segment at both scales (the brief's definition); the site closes a segment rather than refusing, so PAYLOAD_LIMIT at :165 can only fire on one token larger than 64 MiB, which _check_scalars (:97, 1 MiB per string) makes unreachable. Quoted: `_require(len(current) + len(encoded) <= segment, \"PAYLOAD_LIMIT\")`. The segmented shape already shipped on the base branch (base :129 _roster_segments), so base value = head value. Sealed at :687." + }, + { + "constant": "survey_population_preparation.py:58 MAX_ROSTER_BYTES", + "value": "64 * MAX_SEGMENT_BYTES = 4,294,967,296", + "enforced_at": [ + "survey_population_preparation.py:166 ROSTER_LIMIT SurveyPopulationPreparationError (running stream total in _roster_segments)", + "survey_population_preparation.py:262 ROSTER_LIMIT SurveyPopulationPreparationError (`len(payload) == total <= maximum` on the joined payload in _roster_payload)", + "survey_population_preparation.py:1152 ORIGIN_LIMIT SurveyPopulationPreparationError (_bounded_append: three independent budgets, each `budget[0] + len(_encode(row)) + 1`: _plan_document :387-398 selected/excluded/cells; _origins :1157 entities :1204, households :1280, persons :1299; _current_survey_wage_projection :2148)", + "survey_population_preparation.py:1899 CANDIDATE_LIMIT SurveyPopulationPreparationError (caller-supplied candidate copy of the receipt)", + "current_survey_predictors.py:523 (other family, 19-node path): `maximum=source.MAX_ROSTER_BYTES` of the segmented predictor projection; refuses TRANSPORT_ROSTER_LIMIT in graph_survey_population.py:332/:337 SurveyPopulationGraphError" + ], + "refusal_codes": [ + "ROSTER_LIMIT", + "ORIGIN_LIMIT", + "CANDIDATE_LIMIT", + "TRANSPORT_ROSTER_LIMIT" + ], + "encodes": "the whole preparation receipt (same rosters as MAX_SEGMENT_BYTES); the Python row lists behind it (ORIGIN_LIMIT); a candidate copy; out of family, the predictor projection total (stacked persons)", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "receipt 126,660,764 B (0.0295 x); ORIGIN_LIMIT budgets: _origins 87,423,531 B, _plan_document 39,191,311 B, wage rows 667,964 B; predictor projection 8,570,284 B", + "bytes_at_full_source": "receipt 1,251,912,520 B (0.291 x; transport lane 1,099,892,722 B = 0.256 x); ORIGIN_LIMIT budgets: _origins 874,238,396 B, _plan_document 377,628,202 B, wage rows 6,679,875 B; predictor projection 85,702,912 B. The 4 GiB roster admits ~5,445,991 selected households (3.43 x source) at 788.6 B each.", + "measurement_basis": "prep-measurements.json (this lane, full widths, module encoders); ceiling-receipt.json (transport lane); byte-census-measurements.json predictor_projection", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1892 -> _plan_document:2032/:2061, _origins:1986, _roster_payload:2013 (:166, :262), candidate :1899; also survey_population_source_codecs.load:746. 45 and age via the same prefix.", + "classification": "stays", + "notes": "Headroom 3.4x at full source (3.9x on the transport lane's narrower-id figure). Quoted: `_require(budget[0] + size <= MAX_ROSTER_BYTES, \"ORIGIN_LIMIT\")` (:1152) with the comment at :1148-1150 calling it 'A bound on the Python row list, not on a transport'. Base value = head value (already 64 x 64 MiB on origin/native-row-ceilings). Sealed at :688." + }, + { + "constant": "survey_population_preparation.py:60 _MAX_SCALAR_BYTES", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "survey_population_preparation.py:97 SCALAR_LIMIT SurveyPopulationPreparationError in _check_scalars, called before every encode path: _chunks :102, _catalogue_chunks :307, _frame_cell_encode :778" + ], + "refusal_codes": [ + "SCALAR_LIMIT" + ], + "encodes": "one string scalar anywhere in an encoded document (a token), checked in characters before encoding", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "<= 128 characters for every source literal (survey_population_domains.MAX_TOKEN_CHARS); real maxima 64-hex digests, 22-char PERIDNUM, 13-char SERIALNO; identical at every fraction", + "bytes_at_full_source": "same; widths do not scale with the roster", + "measurement_basis": "code: domains._token :187 bounds source literals at 128; _check_scalars walks every string (:96-97)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "every _encode/_digest/_roster_segments call in prepare_authenticated_survey_population (via _chunks:99-102), and _frame_identity via _frame_cell_encode:778", + "classification": "must-not-move", + "notes": "It is what makes the segment refusal at :165 unreachable: no token can exceed 1 MiB, 64x below a segment (comment :159-162). Sealed at :689." + }, + { + "constant": "survey_population_preparation.py:61 _FRAME_FAST_STRING_CHARS", + "value": "4096 (literal) = 4,096", + "enforced_at": [ + "survey_population_preparation.py:765 (fast-path eligibility test `kind is str and len(value) <= _FRAME_FAST_STRING_CHARS` in _frame_cell_encode; an ineligible string takes _encode(value, maximum) at :769 under MAX_PAYLOAD_BYTES; no refusal code of its own)" + ], + "refusal_codes": [], + "encodes": "one string Frame cell's canonical spelling (comment :781-783: at most 6 * 4096 + 2 encoded bytes on the fast path)", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "per cell: string columns of the stacked frame are SERIALNO (13), channel labels, PERIDNUM (22); no scaling with rows", + "bytes_at_full_source": "same", + "measurement_basis": "code reading; no roster document behind it", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:2012 _frame_identity -> _cells_blob:949 -> _string_cells (encodes each distinct value once via _frame_cell_encode) / cell-at-a-time fallback :976; again in _pure_final:1760 and verify_materialized_survey_population:1886", + "classification": "must-not-move", + "notes": "A width threshold, not a ceiling: exceeding it selects the slow path, which is still bounded by MAX_PAYLOAD_BYTES per cell. Sealed at :691. Unchanged on this branch." + }, + { + "constant": "survey_population_preparation.py:793 _FLOAT_WIDTH", + "value": "37 (literal) = 37 bytes per padded float64 cell spelling", + "enforced_at": [ + "survey_population_preparation.py:849-850 (`cells = np.zeros((count, _FLOAT_WIDTH), dtype=np.uint8)`; `keep = np.zeros((count, _FLOAT_WIDTH), dtype=bool)` in _float_cells; no refusal: the only refusal in the function is :834 FRAME_NONFINITE SurveyPopulationPreparationError, a value check)" + ], + "refusal_codes": [ + "FRAME_NONFINITE" + ], + "encodes": "a fixed-width encoder matrix: one float64 column of a Frame (person, household, group or strata/index column) laid out at 37 byte slots per cell plus a 37-slot keep mask, cut to the exact bytes a cell-at-a-time walk would digest", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "transient per float column: 37+37 B x rows -> 26.4 MB for a stacked-person column at 356,501 rows (52.8 MB for a clone-person column at 713,002); the resulting bytes (<= 37 x rows) are then digested. No cap.", + "bytes_at_full_source": "transient per float column: 263.8 MB for a stacked-person column at 3,565,013 rows (527.6 MB at clone persons 7,130,026, reached only from graph_puf_diagnostic_consumer:399, off-path). No cap.", + "measurement_basis": "arithmetic from the code (:849-850, :898 `cells[keep].tobytes()`) at the roster-census counts", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:2012 _frame_identity:980 -> _cells_blob:949 -> _float_cells:814; _pure_final:1760; verify_materialized_survey_population:1886", + "classification": "must-not-move", + "notes": "An encoder width: moving it changes the digest every sealed identity depends on. Recorded because the brief lists struct widths; there is no byte ceiling on the column blob it produces (see open questions)." + }, + { + "constant": "survey_population_preparation.py:289 characters > 65_536 (with :278 len(people) > 20, :274 len(record) != 7, :280 len(person) != 9)", + "value": "65_536 ASCII characters per ACS catalogue record (fast-path eligibility); comment :296-297: 'At most 186 leaves and 65,536 ASCII characters bound this individual JSON record below 400 KiB'", + "enforced_at": [ + "survey_population_preparation.py:289 (returns False from _catalogue_fast_path; the value then takes the generic _chunks path at :302-304; no refusal code)" + ], + "refusal_codes": [], + "encodes": "one raw ACS catalogue record (a 7-tuple with <= 20 person 9-tuples) inside the (records, vacancies) tuple whose digest _nested_seals carries; the tuple covers the whole ACS catalogue (1,631,969 records, 374,174,650 canonical bytes per roster-census.json), fraction-invariant", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "per record a few hundred bytes (<= 20 persons x 9 short literals); the whole catalogue digest is 374,174,650 B at every fraction, streamed record by record into sha256", + "bytes_at_full_source": "same (the catalogue is the whole upstream archive)", + "measurement_basis": "roster-census.json catalogues.acs.canonical_record_bytes; code :267-318", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:2040 _nested_seals -> :1650-1655 _memoized_catalogue_digest -> _catalogue_digest -> _catalogue_chunks:299 -> _catalogue_fast_path:267; again in _pure_final:1762 and _catalogue_memo:2043", + "classification": "must-not-move", + "notes": "Never refuses; failing the test only removes the fast path. Included for completeness as a per-record character cap." + }, + { + "constant": "acs_population_catalogue.py:25 _ARCHIVE_BYTES (defined in the ACS source family; enforced here)", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "survey_population_preparation.py:513-514 (_file_limits entry for acs/csv_hus.zip and acs/csv_pus.zip) -> _source_files:558 _regular_reader(path, limits[name]) -> :426 SOURCE_REGULAR_FILE SurveyPopulationPreparationError (st_size <= maximum) and :561 SOURCE_FILE_LIMIT SurveyPopulationPreparationError (running size while hashing in 1 MiB blocks :559)", + "survey_population_preparation.py:547-552 ARCHIVE_LIMIT SurveyPopulationPreparationError (sum of both archives' lstat sizes <= _ARCHIVE_BYTES)", + "acs_population_catalogue.py:481 ARCHIVE_LIMIT (its own family) and acs_native_coverage_binding.py:568 MAX_ARCHIVE_BYTES (same value, same quantity)" + ], + "refusal_codes": [ + "SOURCE_REGULAR_FILE", + "SOURCE_FILE_LIMIT", + "ARCHIVE_LIMIT" + ], + "encodes": "two upstream files: the captured ACS PUMS archives csv_hus.zip and csv_pus.zip", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "251,500,587 + 602,847,146 = 854,347,733 B (acs_2024_1yr_sources.json, confirmed present in this tree), identical at every fraction", + "bytes_at_full_source": "854,347,733 B (9.9% of the ceiling)", + "measurement_basis": "acs_2024_1yr_sources.json sizes; row-ceilings census confirmed them on the pilot's captured files", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1919 _source_files(root) and _validate:1791 _source_files(state.root) (every full validation, including verification-epoch closes)", + "classification": "must-not-move", + "notes": "Recorded here for its enforcement sites in this family; keyed by its definition in the ACS family. Quoted: `_require(size <= limits[name], \"SOURCE_FILE_LIMIT\")` (:561)." + }, + { + "constant": "asec_2024_native_population.py:42 _MAX_CHECKPOINT_BYTES (defined in the ASEC family; enforced here)", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "survey_population_preparation.py:506 (_file_limits default for every _SOURCE_ROSTER entry; it remains the limit for asec/parent.h5, asec/household-attachment.h5 and asec/person-income-attachment.h5) -> :426 SOURCE_REGULAR_FILE and :561 SOURCE_FILE_LIMIT SurveyPopulationPreparationError", + "asec_2024_native_population.py:163 SOURCE_FILE_BOUNDS (its own family)" + ], + "refusal_codes": [ + "SOURCE_REGULAR_FILE", + "SOURCE_FILE_LIMIT" + ], + "encodes": "three upstream HDF5 files: the ASEC parent and its two attachments", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "person-income-attachment.h5 666,333,922 B (row-ceilings census); parent.h5 and household-attachment.h5 sizes not established in either census; identical at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "row-ceilings census (asec_2024_native_population.py:42 row); no size pin for the other two files exists in this tree", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1919 / _validate:1791 -> _source_files:518 -> _regular_reader:420", + "classification": "must-not-move", + "notes": "An upstream file's real size; fraction-invariant. The two unestablished sizes are an open question." + }, + { + "constant": "asec_person_coverage_source.py:33 MAX_MEMBER_BYTES (defined in the ASEC family; enforced here as asec_native.coverage_owner.literal.MAX_MEMBER_BYTES)", + "value": "1_000_000_000", + "enforced_at": [ + "survey_population_preparation.py:507-509 (_file_limits entries for asec/pppub23.csv, pppub24.csv, pppub25.csv) -> :426 SOURCE_REGULAR_FILE and :561 SOURCE_FILE_LIMIT SurveyPopulationPreparationError", + "asec_person_coverage_source.py:168/:181/:191 MEMBER_BYTES and asec_coverage_authentication.py:315-318 COVERAGE_MEMBER_BYTES (own family)" + ], + "refusal_codes": [ + "SOURCE_REGULAR_FILE", + "SOURCE_FILE_LIMIT" + ], + "encodes": "three upstream CSV files: the pinned ASEC person members", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "largest member 281,065,733 B (row-ceilings census; the figure appears in packages/microcosm-build/src/microcosm/build/us/source_stages.json), identical at every fraction", + "bytes_at_full_source": "same (28% of the ceiling)", + "measurement_basis": "row-ceilings census; source_stages.json", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1919 / _validate:1791 -> _source_files:518", + "classification": "must-not-move", + "notes": "asec_coverage_authentication.py:28 binds `literal` to asec_person_coverage_source, so :508 resolves to this constant." + }, + { + "constant": "asec_original_household_weights.py:41 _MAX_MEMBER_BYTES (defined in the ASEC family; enforced here as asec_native.anchor_owner._MAX_MEMBER_BYTES)", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "survey_population_preparation.py:511 (_file_limits entry for asec/hhpub25.csv) -> :426 SOURCE_REGULAR_FILE and :561 SOURCE_FILE_LIMIT SurveyPopulationPreparationError", + "asec_original_household_weights.py:174 (own family registry pin check)" + ], + "refusal_codes": [ + "SOURCE_REGULAR_FILE", + "SOURCE_FILE_LIMIT" + ], + "encodes": "one upstream CSV file: the pinned ASEC 2025 household member hhpub25.csv", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "33,298,479 B (native_household_origin.py:164 pin `size_bytes=33_298_479`), identical at every fraction", + "bytes_at_full_source": "33,298,479 B (2.0x headroom)", + "measurement_basis": "the pin in native_household_origin.py:158-166 (this tree); row-ceilings census agrees", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1919 / _validate:1791 -> _source_files:518", + "classification": "must-not-move", + "notes": "asec_2024_native_population.py:34 binds anchor_owner to asec_original_household_weights." + }, + { + "constant": "graph_survey_population.py:61 PREPARATION_MAX_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "graph_survey_population.py:403-404 CONTEXT_BYTES SurveyPopulationGraphError (`0 < len(context) <= PREPARATION_MAX_BYTES` on view.context in _checked_preparation)", + "graph_survey_population.py:564 TRANSPORT_LIMIT SurveyPopulationGraphError (`new_context = _bounded_json(context, PREPARATION_MAX_BYTES)` in _allocation_output, the outgoing allocation frame context)", + "graph_survey_population.py:676-677 CONTEXT_BYTES SurveyPopulationGraphError (SurveyPopulationAllocationKernel.__init__ on context_bytes)", + "BASE ONLY (origin/native-row-ceilings): base:305 and base:577 PREPARATION_BYTES on the preparation payload itself; on this branch those two sites moved to PREPARATION_ROSTER_BYTES (head :399-400, :671-672)" + ], + "refusal_codes": [ + "CONTEXT_BYTES", + "TRANSPORT_LIMIT", + "PREPARATION_BYTES (base only)" + ], + "encodes": "the US frame-context document (graph_context.encode_us_frame_context: per entity the column list, row count, id dtype and one 64-hex ordered-id digest; normative metadata; mass_log; weight_sources), a fixed-shape header that does not grow with rows", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "5,442 B measured through graph_context.encode_us_frame_context on an invented six-entity Frame with 40 columns per entity (prep-measurements.json frame_context); the real inventory's width differs but no term grows with rows", + "bytes_at_full_source": "same order (a few KB); at the BASE value the payload sites also saw the receipt: 126,660,764 B and 1,251,912,520 B, both over 64 MiB", + "measurement_basis": "prep-measurements.json frame_context; code graph_context.py:127-160 (only columns lists and one digest per entity vary)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "67,108,864, and at base also enforced on the preparation payload (base:305 PREPARATION_BYTES, base:577), where the receipt exceeded it at 1/10 (1.64-1.89x) and full source (16-19x)", + "head_value": "67,108,864, frame-context sites only (:403-404, :564, :676-677)", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 -> run_atomic_survey_population:157 -> run_authenticated_survey_population:1015 _checked_preparation:390 (:403) and :1028 _allocation_output:543 (:564) and :1034 SurveyPopulationAllocationKernel.__init__:662 (:676); run_graph -> kernel.run:726; graph_atomic_survey_population:172/:176; survey_age_calibration:467/:473", + "classification": "must-not-move", + "notes": "The bound now sits only in front of a fixed-shape document at the encoder's own 64 MiB. The binds flags are false for the document it bounds at head; the receipt's binding at the base payload sites is carried by the PREPARATION_ROSTER_BYTES row. graph_context itself has no constant: encode_us_frame_context is canonical_json (no cap) and _row_identity :113-115 digests `ids.to_numpy(' run_atomic_survey_population:157 -> run_authenticated_survey_population:1015 -> _checked_preparation:390 (:399); :1034 -> SurveyPopulationAllocationKernel.__init__:662 (:671); :1284 final _checked_preparation; graph_atomic_survey_population:172; survey_age_calibration:467. 45 via the same prefix under graph_survey_completion_host.extend.", + "classification": "consumer", + "notes": "The binds flags follow the brief: evaluated at the base value (64 MiB) the receipt exceeded the cap at both scales; at the head value it is 0.29x of the ceiling and equals the producer's own MAX_ROSTER_BYTES total. Quoted head comment :69-75: 'this consumer re-checked the same bytes against PREPARATION_MAX_BYTES ... The consumer's ceiling on the receipt is the producer's own.' Two further whole copies of the receipt exist without any bound: _final_artifact:899-900 (`actual = store.load_bytes(key); _require(... actual == payload, \"FINAL_ARTIFACT_BYTES\")`) and graph_atomic_survey_population:264 (ATOMIC_PREFIX_ARTIFACT_BYTES)." + }, + { + "constant": "graph_survey_population.py:62 ALLOCATION_MAX_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "graph_survey_population.py:478-483 segment close (`if (segments or output) and len(output) + len(separator) + len(raw) + len(tail) > (ALLOCATION_MAX_BYTES): segments.append(bytes(output))`)", + "graph_survey_population.py:484-487 ALLOCATION_LIMIT SurveyPopulationGraphError (one instruction row plus the reserved tail larger than one segment)", + "graph_survey_population.py:495 ALLOCATION_LIMIT SurveyPopulationGraphError (last segment plus tail)" + ], + "refusal_codes": [ + "ALLOCATION_LIMIT" + ], + "encodes": "one segment of the allocation payload `{\"households\":[],\"input_context_sha256\":...,\"output_context_sha256\":...,\"preparation_sha256\":...,\"protocol\":...,\"release_eligible\":false,\"weight_transition\":[...]}`; roster = selected households = stacked households", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "51,879,119 B (158,737 households; 1 segment; 0.77x): 327 B per ACS row and 322 B per ASEC row at 1/10 widths (unreduced inclusion fraction)", + "bytes_at_full_source": "490,164,612 B (8 segments; 7.3x): 309 B per ACS row, 303 B per ASEC row (inclusion [1,1]); docs/us-native-scale-transport.md cites 316.665 B/row -> 502.6 MB. One 64 MiB accumulation admits ~217,180 households (13.7% of source).", + "measurement_basis": "prep-measurements.json allocation_payload: graph._allocation_payload on invented AllocationInstruction tuples at full widths, marginal by difference (2 vs 1002 rows); per-row instruction document via graph._bounded_json(graph._instruction_document(row), 4096); payload bytes proved equal to the canonical whole-tree _bounded_json of the decoded document (payload_equals_canonical_whole_tree true)", + "binds_at_tenth": false, + "binds_at_full_source": true, + "reachable_from": "19", + "reach_chain": "run_authenticated_survey_population:1028 -> _allocation_output:543 -> _allocation_payload:565 (:447); again in SurveyPopulationAllocationKernel.run:726 during run_graph; graph_atomic_survey_population:176; survey_age_calibration:473", + "classification": "segment", + "notes": "Already segmented on the base branch (base:388/:404), so base value = head value; at full source it closes seven segments and never refuses. 'Binds at full source' means the roster exceeds one segment. Quoted: `_require(len(output) + len(separator) + len(raw) + len(tail) <= ALLOCATION_MAX_BYTES, \"ALLOCATION_LIMIT\")` (:484-487)." + }, + { + "constant": "graph_survey_population.py:68 ALLOCATION_ROSTER_BYTES", + "value": "64 * ALLOCATION_MAX_BYTES = 4,294,967,296", + "enforced_at": [ + "graph_survey_population.py:170 ALLOCATION_LIMIT SurveyPopulationGraphError as a row count: `len(plan.selected) <= ALLOCATION_ROSTER_BYTES // 128` = 33,554,432 selected households", + "graph_survey_population.py:488-491 ALLOCATION_ROSTER_LIMIT SurveyPopulationGraphError (running total in _allocation_payload)", + "graph_survey_population.py:499 ALLOCATION_ROSTER_LIMIT (total after the tail)", + "graph_survey_population.py:501 ALLOCATION_ROSTER_LIMIT (`len(payload) == total` on the joined payload)" + ], + "refusal_codes": [ + "ALLOCATION_LIMIT", + "ALLOCATION_ROSTER_LIMIT" + ], + "encodes": "the whole allocation payload (selected households); the :170 arm bounds the count of selected households before the two maps are built", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "51,879,119 B (0.012x); 158,737 households against 33,554,432", + "bytes_at_full_source": "490,164,612 B (0.114x); the byte total admits ~13,899,570 households (8.8x source) at 309 B; 1,587,376 against the 33,554,432 row pre-check (21x)", + "measurement_basis": "prep-measurements.json allocation_payload", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "run_authenticated_survey_population:1020 allocation_instructions:154 (:170); :1028 -> _allocation_payload (:488-501); graph_atomic_survey_population:173/:176; survey_age_calibration:470/:473", + "classification": "stays", + "notes": "Headroom 8.8x on bytes. The :170 arm is a row count wearing a byte name (rule: ALLOCATION_ROSTER_BYTES // 128; the comment at :165-169 says the implied 128-byte row budget is below the payload's structural overhead so it never binds the payload) and is not-a-transport by itself. Base value = head value." + }, + { + "constant": "graph_survey_population.py:472 4096 (tail) and :477 4096 (per-instruction document)", + "value": "4096 = 4,096 bytes per _bounded_json call", + "enforced_at": [ + "graph_survey_population.py:472 TRANSPORT_LIMIT SurveyPopulationGraphError via `_bounded_json(metadata, 4096)` (the six-field metadata tail with three 64-hex digests)", + "graph_survey_population.py:477 TRANSPORT_LIMIT SurveyPopulationGraphError via `_bounded_json(_instruction_document(row), 4096)` (one instruction document per selected household)" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT" + ], + "encodes": "one instruction document (11 fields: ids, key, four fraction pairs, a float64 hex) or the payload tail", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "instruction document 326 B (ACS) / 321 B (ASEC) at 1/10 widths; tail ~300 B", + "bytes_at_full_source": "308 B / 302 B (inclusion [1,1]); tail unchanged. 13x headroom.", + "measurement_basis": "prep-measurements.json allocation_payload.rows.instruction_document_bytes through graph._bounded_json(graph._instruction_document(row), 4096)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "as ALLOCATION_MAX_BYTES: _allocation_output:565 -> _allocation_payload:447", + "classification": "must-not-move", + "notes": "The only variable-width field is raw_native_id, bounded at 128 chars by :184 SELECTED_KEY; every other member is an integer or a two-integer pair." + }, + { + "constant": "graph_survey_population.py:184 0 < len(row.key.native_id) <= 128", + "value": "128 characters", + "enforced_at": [ + "graph_survey_population.py:184-185 SELECTED_KEY SurveyPopulationGraphError (allocation_instructions, per selected household)" + ], + "refusal_codes": [ + "SELECTED_KEY" + ], + "encodes": "one raw native household id token (ACS SERIALNO, 13 chars by survey_population_domains.py:208; ASEC H_SEQ <= 5 digits by :219)", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13 / <= 5 characters; identical at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "code: domains regexes :208 and :219 fix the widths", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "run_authenticated_survey_population:1020 -> allocation_instructions:154; graph_atomic_survey_population:173; survey_age_calibration:470", + "classification": "must-not-move", + "notes": "Same value as survey_population_domains.MAX_TOKEN_CHARS; the comment at :279-280 relies on it to justify the pre-encoding TRANSPORT_LIMIT check." + }, + { + "constant": "graph_survey_population.py:272 0 < limit <= 64 * 1024**2 (_bounded_json) and :294 (_canonical_pieces segment)", + "value": "64 * 1024**2 = 67,108,864: the shared encoder's ceiling on any limit/segment argument", + "enforced_at": [ + "graph_survey_population.py:272 TRANSPORT_LIMIT SurveyPopulationGraphError (refuses a limit above 64 MiB before encoding a byte); per piece :281 (characters) and :283 (bytes) TRANSPORT_LIMIT", + "graph_survey_population.py:294 TRANSPORT_LIMIT (segment argument of _canonical_pieces); per piece :303/:305 TRANSPORT_LIMIT; used by _segmented_json:313 (:331 per segment), _json_sha256:341, _json_matches:356", + "callers passing a limit <= 64 MiB: this module :472/:477 (4096), :564 (PREPARATION_MAX_BYTES); survey_origin_budget.py:93 _json (MAX_PAYLOAD_BYTES) and :629-632; graph_survey_budget.py:146-158 (segment=numeric.MAX_BYTES); current_survey_predictors.py:512-523 (segment=survey_population_preparation.MAX_PAYLOAD_BYTES); graph_puf_diagnostic_consumer.py:403 (CURRENT_SURVEY_MAX_BYTES); current_survey_amounts.py:362-369" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT", + "TRANSPORT_ENCODING" + ], + "encodes": "an encoder width: the largest single accumulation (or single segment) any canonical JSON document may occupy through the shared encoder", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "largest single accumulation on the 19-node path through _bounded_json proper: the allocation frame context (~5.4 KB); largest segment through _canonical_pieces: one 64 MiB segment of the allocation payload / predictor projection", + "bytes_at_full_source": "same by construction; every whole-roster document that exceeds it is segmented by its owner (allocation payload 8 segments, predictor projection 2, origin budget ~18)", + "measurement_basis": "code; prep-measurements.json and byte-census-measurements.json for the callers' totals", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "every _bounded_json/_segmented_json call in the runners' prefix: run_authenticated_survey_population:1028 -> _allocation_output:564/:565; current_survey_predictors:512", + "classification": "must-not-move", + "notes": "docs/us-native-row-ceilings.md section 5c: 'so the shared encoder refuses any cap above 64 MiB before it encodes a byte ... The 64 MiB is the encoder's, and moving it is the transport change.' Quoted: `_require(type(limit) is int and 0 < limit <= 64 * 1024**2, \"TRANSPORT_LIMIT\")` (:272)." + }, + { + "constant": "graph_survey_population.py:349 64 * 1024**2 (_json_sha256 token cap) and :369 (_json_matches token cap)", + "value": "64 * 1024**2 = 67,108,864 per JSON token; the total is the caller's `maximum`", + "enforced_at": [ + "graph_survey_population.py:349 -> _canonical_pieces(value, 64 * 1024**2) -> :303/:305 TRANSPORT_LIMIT SurveyPopulationGraphError (one token); :350 TRANSPORT_ROSTER_LIMIT (caller total) in _json_sha256", + "graph_survey_population.py:369 -> _canonical_pieces(value, 64 * 1024**2) -> :303/:305 TRANSPORT_LIMIT; :371 TRANSPORT_ROSTER_LIMIT; :367 TRANSPORT_ENCODING (payload not bytes) in _json_matches", + "callers: survey_origin_budget.py:605 selection_sha256 = graph._json_sha256(view.receipt['selection'], maximum=MAX_ROSTER_BYTES); survey_origin_budget.py:935 graph._json_matches(document, entry[1], maximum=MAX_ROSTER_BYTES) FINAL_BUDGET_VIEW_DOCUMENT" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT", + "TRANSPORT_ROSTER_LIMIT", + "TRANSPORT_ENCODING" + ], + "encodes": "a per-token width inside a streamed digest / streamed comparison: the selection block (stacked households, 243 B/row SCALED: 38,594,135 B at 1/10, 385,942,815 B full, byte-census-measurements.json selection_block) and the origin-budget document (742 B/group: 117,782,854 B at 1/10, 1,177,832,992 B full, byte-census-measurements.json origin_budget); largest token a 64-hex digest or 22-char PERIDNUM", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "per token <= 64 chars; the streamed documents 38.6 MB (selection) and 117.8 MB (budget), no accumulation", + "bytes_at_full_source": "per token unchanged; streamed documents 386 MB and 1.18 GB", + "measurement_basis": "byte-census-measurements.json selection_block and origin_budget; code :341-374", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration:529 budgets.freeze_survey_origin_budget -> survey_origin_budget:605 _json_sha256; :535 budget.checked_view -> survey_origin_budget:935 _json_matches. Not reached by the 19- or 45-node graphs (neither imports survey_origin_budget on its prefix).", + "classification": "must-not-move", + "notes": "_json_matches and the segment-less _json_sha256 signature are uncommitted working-tree changes (git diff HEAD); the cited lines are working-tree lines. The token cap is the encoder width; the totals are the origin-budget family's MAX_ROSTER_BYTES rows." + }, + { + "constant": "survey_catalogue_selection.py:19 _BATCH_HOUSEHOLDS", + "value": "10_000 = 10,000 households per classification batch", + "enforced_at": [ + "survey_catalogue_selection.py:127-132 BATCH_LIMITS CatalogueSelectionError(ValueError) (static self-consistency: 0 < _BATCH_HOUSEHOLDS <= min(10_000, domains.MAX_HOUSEHOLDS))", + "survey_catalogue_selection.py:196-201 (the flush `if batch and (len(batch) == _BATCH_HOUSEHOLDS or ...): consume()`; no data refusal)" + ], + "refusal_codes": [ + "BATCH_LIMITS" + ], + "encodes": "no document: the number of catalogue households held between two classify_households calls", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a (row count); <= 10,000 households per batch at every fraction; the whole catalogue (1,587,376 supplied households) is classified in ~159 batches regardless of fraction because the draw follows classification", + "bytes_at_full_source": "n/a; same", + "measurement_basis": "code :139-202", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1943 plan_catalogue_selection:112", + "classification": "not-a-transport", + "notes": "A memory ceiling with no document behind it; it is what keeps survey_population_domains.MAX_HOUSEHOLDS dead. Sealed into the producer identity at :697." + }, + { + "constant": "survey_catalogue_selection.py:20 _BATCH_PEOPLE", + "value": "100_000 = 100,000 persons per classification batch", + "enforced_at": [ + "survey_catalogue_selection.py:127-132 BATCH_LIMITS CatalogueSelectionError (static: <= min(100_000, domains.MAX_TOTAL_MEMBERS))", + "survey_catalogue_selection.py:186-189 HOUSEHOLD_MEMBER_BOUND CatalogueSelectionError (`len(row.persons) <= min(domains.MAX_MEMBERS, _BATCH_PEOPLE)` = 20)", + "survey_catalogue_selection.py:196-201 flush when batch_people + len(row.persons) > _BATCH_PEOPLE" + ], + "refusal_codes": [ + "BATCH_LIMITS", + "HOUSEHOLD_MEMBER_BOUND" + ], + "encodes": "no document: persons held between two classify_households calls (the whole catalogue's 3,565,013 persons stream through)", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a (row count); <= 100,000 per batch", + "bytes_at_full_source": "n/a; same", + "measurement_basis": "code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1943 -> plan_catalogue_selection", + "classification": "not-a-transport", + "notes": "Keeps survey_population_domains.MAX_TOTAL_MEMBERS dead. Sealed at :698." + }, + { + "constant": "survey_population_domains.py:17 MAX_TOKEN_CHARS", + "value": "128 characters", + "enforced_at": [ + "survey_population_domains.py:187 LITERAL_TYPE_SIZE DomainInputError(ValueError) via _token, from _key:204, _unsigned:192, _person:291/:296/:329, classify_households:569", + "survey_population_domains.py:219 (digit width for the ASEC H_SEQ parse) and :500 (digit width for HSUP_WGT)", + "fixed-width companions in the same module: :208 SERIALNO regex `2024(?:HU|GQ)[0-9]{7}` (13 chars), :381 and :571 PERIDNUM `[0-9]{22}`" + ], + "refusal_codes": [ + "LITERAL_TYPE_SIZE", + "HOUSEHOLD_KEY", + "MEMBER_PERSON_KEY" + ], + "encodes": "one raw source literal (SERIALNO, H_SEQ, PERIDNUM, AGEP, WGTP, HSUP_WGT, ...)", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "<= 22 characters in practice (PERIDNUM 22, SERIALNO 13, H_SEQ <= 5); identical at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "code regexes", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "plan_catalogue_selection:139 consume -> domains.classify_households:545 -> classify_household -> _key/_members/_person", + "classification": "must-not-move", + "notes": "The upstream bound behind _MAX_SCALAR_BYTES and SELECTED_KEY." + }, + { + "constant": "survey_population_domains.py:18 MAX_MEMBERS", + "value": "20 persons per household", + "enforced_at": [ + "survey_population_domains.py:244 QUALIFIER_ROSTER DomainInputError", + "survey_population_domains.py:358-360 MEMBER_TUPLE_BOUND DomainInputError (_members)", + "survey_population_domains.py:558-560 MEMBER_TUPLE_BOUND (classify_households)", + "survey_catalogue_selection.py:186-188 HOUSEHOLD_MEMBER_BOUND as min(MAX_MEMBERS, _BATCH_PEOPLE)" + ], + "refusal_codes": [ + "QUALIFIER_ROSTER", + "MEMBER_TUPLE_BOUND", + "HOUSEHOLD_MEMBER_BOUND" + ], + "encodes": "no document: persons in one household (the publisher's NP/H_NUMPER top code, parsed at :459-464 with maxima 20/16)", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a (row count), <= 20", + "bytes_at_full_source": "n/a", + "measurement_basis": "code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "as MAX_TOKEN_CHARS", + "classification": "not-a-transport", + "notes": "A structural invariant of the source, not a scale bound. Sealed at survey_population_preparation.py:694." + }, + { + "constant": "survey_population_domains.py:19 MAX_HOUSEHOLDS", + "value": "100_000 households per classify_households batch", + "enforced_at": [ + "survey_population_domains.py:551 HOUSEHOLD_BATCH DomainInputError", + "survey_catalogue_selection.py:129 BATCH_LIMITS (static upper bound on _BATCH_HOUSEHOLDS)" + ], + "refusal_codes": [ + "HOUSEHOLD_BATCH", + "BATCH_LIMITS" + ], + "encodes": "no document: one batch's household count", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a; batches are capped at 10,000 by _BATCH_HOUSEHOLDS, so this is dead at any fraction", + "bytes_at_full_source": "n/a", + "measurement_basis": "code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "as MAX_TOKEN_CHARS", + "classification": "not-a-transport", + "notes": "Dead behind survey_catalogue_selection._BATCH_HOUSEHOLDS (row-count rule: batch flush at 10,000). Sealed at :695." + }, + { + "constant": "survey_population_domains.py:20 MAX_TOTAL_MEMBERS", + "value": "1_000_000 persons per classify_households batch", + "enforced_at": [ + "survey_population_domains.py:563 BATCH_MEMBER_BOUND DomainInputError", + "survey_catalogue_selection.py:131 BATCH_LIMITS (static upper bound on _BATCH_PEOPLE)" + ], + "refusal_codes": [ + "BATCH_MEMBER_BOUND", + "BATCH_LIMITS" + ], + "encodes": "no document: one batch's person count", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a; dead behind _BATCH_PEOPLE (100,000)", + "bytes_at_full_source": "n/a", + "measurement_basis": "code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "as MAX_TOKEN_CHARS", + "classification": "not-a-transport", + "notes": "Sealed at :696." + }, + { + "constant": "survey_observed_age.py:19 MAX_ROWS", + "value": "14_000_000 (comment :14-18: four times the 3,422,888 ACS persons, rounded up)", + "enforced_at": [ + "survey_observed_age.py:52 ROW_BOUND, raised as ValueError('SURVEY_OBSERVED_AGE_ROW_BOUND') (:37-39)" + ], + "refusal_codes": [ + "SURVEY_OBSERVED_AGE_ROW_BOUND" + ], + "encodes": "no document: len(raw_age) for one channel's person rows at a time (_normalized_source_copy :1079-1083 normalizes ACS and ASEC separately)", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a (row count): ACS 342,288 / ASEC 14,212 persons", + "bytes_at_full_source": "n/a: ACS 3,422,888 (4.09x headroom) / ASEC 142,125", + "measurement_basis": "roster-census.json counts; code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1974 _normalized_source_copy:1079 -> normalize_observed_age:44; also _verify_normalized_copy:1140", + "classification": "not-a-transport", + "notes": "Base value = head value (14,000,000 already on origin/native-row-ceilings). Sealed at survey_population_preparation.py:701." + }, + { + "constant": "survey_observed_age.py:20 MAX_EXACT_FLOAT64_INTEGER", + "value": "2**53 = 9,007,199,254,740,992", + "enforced_at": [ + "survey_observed_age.py:60 RAW_REPRESENTATION ValueError('SURVEY_OBSERVED_AGE_RAW_REPRESENTATION')", + "survey_observed_age.py:81 INCUMBENT_REPRESENTATION ValueError" + ], + "refusal_codes": [ + "SURVEY_OBSERVED_AGE_RAW_REPRESENTATION", + "SURVEY_OBSERVED_AGE_INCUMBENT_REPRESENTATION" + ], + "encodes": "a fixed-width field: the float64 exact-integer range of the derived `age` column (docstring :49-51: 'The float bound is a representation limit')", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "8 bytes per cell; values <= 99; independent of the roster", + "bytes_at_full_source": "same", + "measurement_basis": "code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "as MAX_ROWS", + "classification": "must-not-move", + "notes": "docs/us-native-row-ceilings.md line 188 lists it under 'A fixed-width encoding'. Sealed at survey_population_preparation.py:702." + }, + { + "constant": "operator_column_contracts.py:64 PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID (defined outside this family; enforced in spine_assembly)", + "value": "10**15 - 1 = 999,999,999,999,999", + "enforced_at": [ + "spine_assembly.py:630-640 ValueError (every entity's source ids per channel in _validate_source_frame)", + "spine_assembly.py:763-767 ValueError (after collision remap in _id_offsets); companion int64 overflow check :756 (`int(ids.max()) + offset > int(info.max)`)", + "puf_support._remap_ids re-check (other family; comment puf_support.py:2324-2328)" + ], + "refusal_codes": [ + "ValueError (no static code)" + ], + "encodes": "a fixed-width field: the magnitude of an entity id so the clone stage's decimal remap `id + clone_index * 10**digits(max_id)` stays inside int64", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "stacked ids <= 356,501 persons (6 digits), far below 10**15; 8 bytes per id", + "bytes_at_full_source": "stacked ids <= 3,565,013 persons (7 digits); clone ids <= 7 + 10**7 digits", + "measurement_basis": "roster-census.json counts; code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1977 spine_assembly.stack_survey_spines:322 -> _stack_source_tables:213 -> _validate_source_frame:596 / _id_offsets:732", + "classification": "must-not-move", + "notes": "spine_assembly has no byte constant of its own; its digests (_numeric_digest :170, _channel_digest :175) stream contiguous copies (8 B x households) with no cap." + }, + { + "constant": "native_household_origin.py:48 MAX_SOURCE_BYTES", + "value": "512 * 1024**2 = 536,870,912", + "enforced_at": [ + "native_household_origin.py:125-130 ASEC_MEMBER_BOUNDS NativeOriginError(ValueError) (`0 < self.size_bytes <= MAX_SOURCE_BYTES` on each pinned hhpub member)", + "native_household_origin.py:196 SOURCE_SIZE NativeOriginError (`len(payload) <= MAX_SOURCE_BYTES` in _source_document, before :198 json.loads and :199 SOURCE_CANONICAL `canonical_json(value) == payload`, a consumer re-encoding whose encoder cap is this same 512 MiB by order of checks); reached from _issue:241-244, AuthenticatedNativeOriginSource.document:189, graph_native_household_origin.py:180 and :190" + ], + "refusal_codes": [ + "ASEC_MEMBER_BOUNDS", + "SOURCE_SIZE", + "SOURCE_CANONICAL" + ], + "encodes": "(a) three upstream files hhpub23/24/25.csv; (b) the origin source document: one `[period, native_id, member_index, origin_key]` record per ACS housing record (all 1,631,969, vacancies included: produce_acs_native_origins :279-283 iterates the whole housing source) or per ASEC hhpub row (267,383 across three cohorts), fraction-invariant", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "(a) 30,259,450 / 30,448,089 / 33,298,479 B (pins :150-176); (b) ACS 92 B x 1,631,969 = 150,141,148 B (0.280 x), ASEC 82 B x 267,383 = 21,925,406 B; identical at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "prep-measurements.json native_household_origin.source_document (canonical_json on invented records at 13-char SERIALNO / 5-digit H_SEQ / 64-hex key); counts from roster-census.json catalogues.acs.households and the pins' rows", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Only graph_native_household_origin.NativeACSOriginKernel.run:69-82 / NativeASECOriginKernel.run:89-104 / verify_materialized_population_origins:325; those nodes are declared by native_origin_source_nodes:239 and appear in neither the 19-node nor the 45-node roster (brief section 2) and no runner in this family imports graph_native_household_origin.", + "classification": "must-not-move", + "notes": "Base value = head value. The :199 re-encoding holds the source document twice (payload and canonical_json copy)." + }, + { + "constant": "native_household_origin.py:129 0 < self.rows <= 1_000_000", + "value": "1_000_000 rows per pinned hhpub member", + "enforced_at": [ + "native_household_origin.py:126-130 ASEC_MEMBER_BOUNDS NativeOriginError; the pin's rows is then enforced on the file at :350 and :352 ASEC_MEMBER_ROWS" + ], + "refusal_codes": [ + "ASEC_MEMBER_BOUNDS", + "ASEC_MEMBER_ROWS" + ], + "encodes": "no document: an upstream file's row count (88,978 / 89,473 / 88,932 pinned)", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a (row count); fraction-invariant", + "bytes_at_full_source": "n/a", + "measurement_basis": "pins :150-176", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "as MAX_SOURCE_BYTES", + "classification": "not-a-transport", + "notes": "A row count on an upstream file; must-not-move by nature though classified as the row-count rule the brief gives." + }, + { + "constant": "native_household_origin.py:81 [1-9][0-9]{0,17} and :335 [0-9]{1,18}", + "value": "<= 18 decimal digits for an ASEC H_SEQ token", + "enforced_at": [ + "native_household_origin.py:80-83 CARRIED_NATIVE_INTEGER NativeOriginError (_carried_native_integer on a carried asec_H_SEQ cell)", + "native_household_origin.py:334-338 ASEC_MEMBER_NATIVE_KEY NativeOriginError (produce_asec_native_origins per hhpub row)" + ], + "refusal_codes": [ + "CARRIED_NATIVE_INTEGER", + "ASEC_MEMBER_NATIVE_KEY" + ], + "encodes": "one native id token (real width <= 5 digits)", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "<= 5 characters; fraction-invariant", + "bytes_at_full_source": "same", + "measurement_basis": "code; survey_population_domains.py:219 (H_SEQ <= 99999)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "as MAX_SOURCE_BYTES; :81 also from _bind_population_origins:530", + "classification": "must-not-move", + "notes": "Token width only." + }, + { + "constant": "graph_native_household_origin.py:210 canonical_json(doc) == value.payload and :408 canonical_json(document) == artifact.payload (consumer re-encodings of the population-origin binding; encoder cap: none)", + "value": "no cap: microcosm.graph.canonical.canonical_json is json.dumps with no size bound (canonical.py:43-56); native_household_origin.py:636 produces the binding with the same encoder and :394 json.loads decodes it whole", + "enforced_at": [ + "graph_native_household_origin.py:208-212 ORIGIN_PARENT_PAYLOAD NativeOriginError (parent binding: json.loads then canonical_json == payload)", + "graph_native_household_origin.py:404-409 ORIGIN_MATERIALIZED_LINEAGE NativeOriginError (after a full bind_population_origins rebuild: `document == expected and canonical_json(document) == artifact.payload`)", + "native_household_origin.py:636-650 (producer canonical_json of the whole document, uncapped); :394 PopulationOriginBinding.document json.loads (uncapped)" + ], + "refusal_codes": [ + "ORIGIN_PARENT_PAYLOAD", + "ORIGIN_MATERIALIZED_LINEAGE" + ], + "encodes": "the population-origin binding document: one row per household of the bound population (202 B) plus one row per entity row of all six entities (229 B; 353 B per person with membership_source_ids) plus per-entity ordered-id digests; roster = the bound population's households/persons/groups (stacked, or clone when bound after the EXPAND)", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "379,709,051 B at the stacked 1/10 population (158,737 households, 356,501 persons, groups SCALED) through canonical_json at full-width invented rows; a second whole copy on every re-encode", + "bytes_at_full_source": "3,797,101,483 B at the stacked full population; 7,594,203,653 B at the clone population (3,174,752 households, 7,130,026 persons); no cap anywhere", + "measurement_basis": "prep-measurements.json native_household_origin.binding_document (canonical_json per row + comma, counts from roster-census.json, groups SCALED by the 1/1000 ratios)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "PopulationOriginBindingKernel.run:146-244 and verify_materialized_population_origins:325-413 only; not declared in either pilot graph (brief section 2)", + "classification": "consumer", + "notes": "Recorded because the brief counts a consumer-side re-encoding as a bound even without an explicit cap; the encoder's cap is 'none'. The binds flags are false only because there is no value to exceed; at full source the document is 3.8-7.6 GB held at least twice (:404-409 holds document, expected, canonical_json(document) and artifact.payload). graph_combined_clone's lineage digest (:216-217, :310-318) uses the same uncapped canonical_json over [[source_id, 1, target_id]] per base row and IS on the 19-node path (see open questions)." + }, + { + "constant": "graph_sources.py:441-442 canonical_json(preparation) != prepared.payload (consumer re-encoding of the stacked-spine preparation receipt; encoder cap: none)", + "value": "no cap: canonical_json; the producer USAssemblyCreateKernel.run:395 emits `canonical_json(_json_data(result.receipt))` and the consumer USSpineHarmonizeKernel.run:441 json.loads then re-encodes", + "enforced_at": [ + "graph_sources.py:441-443 ValueError('US spine preparation must be canonical JSON.')" + ], + "refusal_codes": [ + "ValueError (no static code)" + ], + "encodes": "stacked_spine.prepare_stacked_spine's receipt (:1005-1021): sampling receipts, assembly_context (spine_assembly:223-248: counts, digests, `household_order` = one int per stacked household when the concatenated household ids are not already sorted, else None; source_mass_logs), assembly_metadata, acs_native_group_quarters digests. Roster-shaped only through household_order.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "not measured (not on any path): fixed part ~ a few KB; household_order, when present, ~8 B per stacked household (1.3 MB at 1/10)", + "bytes_at_full_source": "not measured; household_order ~12.7 MB at 1,587,376 households when present", + "measurement_basis": "code reading; the legacy assembly graph is outside the pilot rosters so no measurement effort was spent (brief section 2)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "us_assembly_graph:483 / us_assembly_registry:562 only; graph_sources' star exports and _stacked_alias:171 are how stacked_spine is reached, and survey_population_preparation imports graph_sources only for _canonical_assembly:1979 and frame_column_declarations", + "classification": "consumer", + "notes": "graph_sources also emits the frame context at :470 via canonical_json (fixed). stacked_spine.py carries no byte constant at all (14,231 lines read): its json.dumps materialisations are _integer_rows_sha256 (:1987-1995, every native ACS household and person lineage as 4- and 5-int rows) and _canonical_sha256 (:2712-2719), both uncapped and off both paths." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prep/measure_prep.py", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prep/prep-measurements.json (receipt)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prep/run.log", + "cited: experiments/native-byte-transports/byte-census-measurements.json (predictor_projection, selection_block, origin_budget); experiments/native-scale-transport/ceiling-receipt.json (replicated preparation receipt 109,804,304 B at 1/10 and 1,099,892,722 B at full)" + ], + "open_questions": [ + "The recovered 1/1000 artifact (_recovered/pilot-runs/native19-required-20260912/... and native45-v5/) is not present in this worktree or at the path the brief gives, so the 45,922 B fixed preamble, the 168 B request size and the graph.json node rosters were taken from the row-ceilings census, roster-census.json and the brief rather than re-read; the per-row and roster bytes above are this lane's own measurements through the module encoders on invented full-width rows and sit 12-15% above the transport lane's replicated-artifact figures because invented ids use maximal 7-digit widths and unreduced 1/10 inclusion fractions.", + "graph_survey_population.py and survey_origin_budget.py differ in the working tree from HEAD 77bafbb3a (git diff HEAD): _json_matches is added, _json_sha256 loses its segment parameter, and survey_origin_budget:935 switches to _json_matches. All cited line numbers are working-tree lines; a verifier reading the committed HEAD will find the _json_sha256/_json_matches rows shifted or absent.", + "Uncapped whole-roster materialisations on the 19-node path that have no bound and therefore no row: (1) graph_combined_clone._digest:216-217 encodes `[[source_id, 1, target_id], ...]` for every base row of every entity via canonical_json (about 25 B per row: stacked persons 3,565,013 -> ~90 MB, all six entities ~250 MB per EXPAND), computed in USCombinedSurveyCloneExpandKernel.run:502 and again in run_authenticated_survey_population.observe:1150-1152 and _verify_cloned_frame; (2) survey_population_preparation._frame_identity column blobs (_float_cells 37+37 B x rows per float column, _string_cells/_integer_cells whole-column bytes) and graph_context._row_identity:113-115 (8 B x rows per entity); (3) graph_survey_population._final_artifact:899-900 and graph_atomic_survey_population:264 read a second whole copy of the preparation receipt and allocation payload from the store for an equality check, and _checked_preparation:397 json.loads the whole receipt into the view on every borrow. None refuses; each is a multiple of full source with no ceiling.", + "survey_population_preparation._file_limits enforces four constants defined in other families (acs_population_catalogue._ARCHIVE_BYTES, asec_2024_native_population._MAX_CHECKPOINT_BYTES, asec_person_coverage_source.MAX_MEMBER_BYTES, asec_original_household_weights._MAX_MEMBER_BYTES); they are recorded here as rows keyed by their definitions so the owning families' rows can absorb the enforcement sites (:426, :506-514, :551-552, :561). The real sizes of asec/parent.h5 and asec/household-attachment.h5 are not pinned anywhere in this tree and neither census established them.", + "The wage projection (_current_survey_wage_projection:2097, rows = ASEC persons, 47 B/row, 6.68 MB at full source) is a whole-roster receipt through _roster_payload but is reached only from graph_puf_diagnostic_consumer.qualify_current_survey_host:393, whose callers (graph_current_survey_puf_transfer, graph_puf_diagnostic_consumer) the brief lists as off-path; it is folded into the MAX_SEGMENT_BYTES/MAX_ROSTER_BYTES rows as reachable_from none.", + "The binds_at_* flags for PREPARATION_ROSTER_BYTES and the segment rows follow the brief's literal rule (measured bytes exceed the base value / the document exceeds one segment); a reader who wants 'refuses at head' should use the head_value column and the classification.", + "stacked_spine.py (14,231 lines) and graph_implementation.py, support_provenance.py, survey_population_replay.py, graph_atomic_survey_clone.py and graph_context.py contain no byte constant, encoder limit argument, len(payload) check, read cap or struct-width product; their canonical_json/json.dumps uses are either fixed-shape (replay metadata/mass_log, implementation manifests, frame context) or uncapped off-path materialisations (stacked_spine _integer_rows_sha256, _canonical_sha256)." + ] + }, + { + "family": "Property income, property tax and completion", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_property.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income_receipts.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_property_income_sources.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/property_income_constants.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_completion.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_completion_host.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_child_property_income.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_child_property_income_source.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_property_completion_routing.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_dividend_source.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py", + "packages/microcosm-fit/src/microcosm/fit/graph_joint_empirical.py", + "packages/microcosm-fit/src/microcosm/fit/joint_empirical.py", + "packages/microcosm-fit/src/microcosm/fit/_graph_legacy_qrf.py (encode_json/decode_json/artifact/encode_raw_target/read_raw_target, :18-200)", + "packages/microcosm-fit/src/microcosm/fit/model_input.py (:1-140)", + "packages/microcosm-fit/src/microcosm/fit/qrf.py (_IndexIdentity :577-638)", + "packages/microcosm-fit/src/microcosm/fit/graph_signed_reconciliation.py (:212-240)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py (:30-45, :74-76, :225-345)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py (:90-155)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py (:110-175)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py (:355-480, :820-900, :1275-1360, :1460-1520, :1575-1600, :1685-1900)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py (_states :79-135)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py (_check_node_states :903-925)", + "packages/microcosm-graph/src/microcosm/graph/canonical.py (:43-58)", + "packages/microcosm-graph/src/microcosm/graph/manifest.py (receipt freezing, to_json_bytes :733)", + "experiments/native-byte-transports/census-brief.md", + "experiments/native-byte-transports/byte-census-measurements.json", + "experiments/native-byte-transports/byte_census_measure.py", + "experiments/native-row-ceilings/census.json (family entries)", + "experiments/native-row-ceilings/roster-census.json", + "docs/us-native-row-ceilings.md (:220-240)", + "/Users/maxghenis/PolicyEngine/_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json (origins shape)", + "/Users/maxghenis/PolicyEngine/_recovered/pilot-runs/native45-v5/run/execution-config.proposed.json, PROGRESS.md, out.md (options only; no manifest was exported)", + "git diff origin/native-row-ceilings for graph_child_property_income.py, current_child_property_income_source.py, graph_joint_empirical.py" + ], + "rows": [ + { + "constant": "graph_child_property_income.py:79 MAX_ARTIFACT_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "graph_child_property_income.py:110 (_json, per-segment accumulation: `require(len(current) + len(encoded) <= MAX_ARTIFACT_BYTES, \"GRAPH_ARTIFACT_SIZE\")`; require is child.require, ValueError(\"CHILD_PROPERTY_GRAPH_ARTIFACT_SIZE\"))" + ], + "refusal_codes": [ + "GRAPH_ARTIFACT_SIZE (ValueError CHILD_PROPERTY_GRAPH_ARTIFACT_SIZE)" + ], + "encodes": "Every document this module's _json encodes. Roster-shaped: the child draw document (one row per eligible child: adapter._draw_document rows support_id/coordinate/values/pattern/donor_key, encoded at :691 in _reconstruct) and the completion document (:728, which re-embeds every draw row as original_draws). Fixed-shape: donor/recipient projections (:389/:404 digests+counts), scenario (:542/:760/:945), model metadata (:694), verification (:745), host_pins (:550/:947-948), receipt stamps (:1176/:1190).", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "draw 11.9-19.0 MB (65,952 eligible children ESTIMATE x 180.5-288.5 B + 770 fixed); completion the same + ~1 KB", + "bytes_at_full_source": "draw 119.0-190.3 MB (659,527 eligible children ESTIMATE x 180.5-288.5 B); completion the same + ~1 KB", + "measurement_basis": "Measured through graph_child_property_income._json(adapter._draw_document(...)) at full-source widths (7-digit support ids, 13-char ACS SERIALNO keys, 22-char ASEC PERIDNUM donor keys) as a difference of 2 and 2,002 rows: 288.5 B/row for all-positive (pattern 3) rows, 180.5 B/row for zero-pattern rows (donor_key null), fixed 770 B; asserted byte-identical to adapter._json. byte-census-measurements.json child_draw gives 273.0 B/row with a shorter invented donor key. Eligible-child count is an ESTIMATE (18.5% of stacked persons, the row-count lane's share; no recovered artifact records eligible_child_rows and the 45-node pilot exported no manifest). Scripts: scratchpad/census/prop/prop_measure.py, prop_measure_2.py; receipts prop-measurements*.json, prop-derived-totals.json.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "64 * 1024**2 = 67,108,864 as a whole-document cap: base _json was `payload = adapter._json(value); require(len(payload) <= MAX_ARTIFACT_BYTES, \"GRAPH_ARTIFACT_SIZE\")`", + "head_value": "64 * 1024**2 = 67,108,864 as a per-segment ceiling inside json.JSONEncoder.iterencode accumulation (:102-116); the whole total is bounded by MAX_ROSTER_BYTES", + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> graph_survey_completion_host.extend -> _CompletionHost.__init__ -> child.ChildPropertyBoundary.__init__ -> child_property_nodes -> _projections -> _json; _extend -> _child_expected -> _reconstruct -> _json (:691 draw, :728 completion); host.run_graph -> _ChildKernel.run -> _reconstruct; boundary.verify -> ChildPropertyBoundary.verify_materialized -> _reconstruct", + "classification": "segment", + "notes": "binds_at_full_source is true at the base value's whole-document semantics (119 MB lower bound > 64 MiB; the verdict holds for any pattern mix once eligible children exceed 371,800, i.e. 10.4% of stacked persons). At head the per-segment check cannot refuse a roster (a segment closes at :106 before it exceeds 64 MiB), so it does not bind at any fraction. Segments are joined once at :116 `payload = b\"\".join(segments)`, so the whole payload is still materialised after the check." + }, + { + "constant": "graph_child_property_income.py:86 MAX_ROSTER_BYTES", + "value": "64 * MAX_ARTIFACT_BYTES = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "graph_child_property_income.py:112 (_json running total: `require(total + len(encoded) <= MAX_ROSTER_BYTES, \"GRAPH_ARTIFACT_LIMIT\")`)", + "graph_child_property_income.py:117 (joined payload: `require(len(payload) == total <= MAX_ROSTER_BYTES, \"GRAPH_ARTIFACT_LIMIT\")`)" + ], + "refusal_codes": [ + "GRAPH_ARTIFACT_LIMIT (ValueError CHILD_PROPERTY_GRAPH_ARTIFACT_LIMIT)" + ], + "encodes": "The same documents as MAX_ARTIFACT_BYTES; the roster-shaped ones are the child draw document (eligible children) and the completion document (re-embedded draw rows).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "draw 11.9-19.0 MB; completion the same + ~1 KB (65,952 eligible children ESTIMATE)", + "bytes_at_full_source": "draw 119.0-190.3 MB; completion the same + ~1 KB (659,527 eligible children ESTIMATE)", + "measurement_basis": "Same measurement as the MAX_ARTIFACT_BYTES row (prop_measure.py child_draw, prop_measure_2.py draw_rows): 180.5-288.5 B per eligible child through the module's own _json. At 288.5 B the 4 GiB total admits 14,887,286 children, more than four times the stacked person roster.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none: the base branch had no whole-roster total; its only cap was the whole-document MAX_ARTIFACT_BYTES", + "head_value": "64 * MAX_ARTIFACT_BYTES = 4,294,967,296", + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> graph_survey_completion_host.extend -> _CompletionHost.__init__/_extend -> child._reconstruct -> _json (:691, :728); host.run_graph -> _ChildKernel.run; boundary.verify -> verify_materialized -> _reconstruct", + "classification": "segment", + "notes": ":112 streams (running total during iterencode); :117 is checked on the joined whole payload, so the bytes are materialised once at the end. Headroom at full source is about 22x on the all-positive upper bound. The comment at :85 cites docs/us-native-byte-transports.md, which does not exist in this tree." + }, + { + "constant": "graph_child_property_income.py:89 MAX_ORIGINALS", + "value": "15_000_000", + "enforced_at": [ + "graph_child_property_income.py:340 (_projections: `0 < len(recipients) <= MAX_ORIGINALS`, \"ORIGINAL_RECIPIENT_ROSTER\")" + ], + "refusal_codes": [ + "ORIGINAL_RECIPIENT_ROSTER (ValueError CHILD_PROPERTY_ORIGINAL_RECIPIENT_ROSTER)" + ], + "encodes": "A row count, not bytes: qualified.recipients, the stacked person roster (index-identical to the origins persons table, :337-339).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "not a byte count: 356,501 stacked persons", + "bytes_at_full_source": "not a byte count: 3,565,013 stacked persons", + "measurement_basis": "Brief \u00a73 stacked person counts; the site compares len() of a DataFrame.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "600_000", + "head_value": "15_000_000", + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> extend -> _CompletionHost.__init__ -> ChildPropertyBoundary.__init__ -> child_property_nodes (:957) -> _projections (:539) -> :340", + "classification": "not-a-transport", + "notes": "Row count wearing no byte name; listed because brief \u00a75 records it as moved. binds_at_full_source is true at the base value (3,565,013 > 600,000); false at head. Row-count rule (comment :87-88): four times the measured full-source stacked person roster, rounded up to the next whole million (docs/us-native-row-ceilings.md)." + }, + { + "constant": "graph_child_property_income.py:138 0 < len(self.scenario_id) <= 256", + "value": "256 characters", + "enforced_at": [ + "graph_child_property_income.py:136-144 (ChildPropertyOptions.document, \"SCENARIO_CONTRACT\")" + ], + "refusal_codes": [ + "SCENARIO_CONTRACT (ValueError CHILD_PROPERTY_SCENARIO_CONTRACT)" + ], + "encodes": "A single scalar token: the operator's scenario_id string inside the scenario document.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "38 characters in the 45-node pilot (\"us-native-completion45-v5-candidate\"); fraction-independent", + "bytes_at_full_source": "same 38 characters; fraction-independent", + "measurement_basis": "execution-config.proposed.json child_property_document.scenario_id; a scalar, not a roster.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> graph_survey_completion_host.validate_options -> options.document() (:45); also ChildPropertyBoundary.__init__ :945 and _pure :976", + "classification": "must-not-move", + "notes": "Token cap on a scalar; nothing grows with the fraction." + }, + { + "constant": "graph_child_property_income.py:812 value.payload == expected[edge.producer, edge.artifact]", + "value": "consumer-side byte equality against bytes re-encoded through the segmented _json (encoder cap MAX_ROSTER_BYTES = 4,294,967,296; per segment MAX_ARTIFACT_BYTES = 67,108,864)", + "enforced_at": [ + "graph_child_property_income.py:811-814 (_artifacts, \"RECONSTRUCTED_ARTIFACT_CHANGED\"), reached from :1040 and :1062 (verify_materialized), :1153 (ATTACH/VERIFY kernels); :1130 passes expected={} for DONOR/RECIPIENT so only host pins are checked there" + ], + "refusal_codes": [ + "RECONSTRUCTED_ARTIFACT_CHANGED (ValueError CHILD_PROPERTY_RECONSTRUCTED_ARTIFACT_CHANGED)" + ], + "encodes": "Roster-shaped: the draw document produced in the graph run by microcosm.fit.graph_joint_empirical.JointEmpiricalDrawKernel.run (:567-568, adapter._json, one json.dumps) and the completion document produced by the ATTACH kernel; both are re-produced by _reconstruct (:691, :728) through the segmented _json and compared whole. Fixed-shape: projections, scenario, model, metadata.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "draw 11.9-19.0 MB and completion the same + ~1 KB (65,952 eligible children ESTIMATE)", + "bytes_at_full_source": "draw 119.0-190.3 MB and completion the same + ~1 KB (659,527 eligible children ESTIMATE)", + "measurement_basis": "Same encoder measurement as the MAX_ARTIFACT_BYTES row (180.5-288.5 B per eligible child). The check has no cap of its own; the re-encoder's total is MAX_ROSTER_BYTES.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> extend -> host.run_graph -> _ChildKernel.run (ATTACH_REF/VERIFY_REF) -> _reconstruct -> _artifacts (:1153) -> :812; _extend -> boundary.verify -> ChildPropertyBoundary.verify_materialized -> _artifacts (:1040, :1062)", + "classification": "consumer", + "notes": "The stored draw bytes come from the fit adapter's kernel, an unsegmented json.dumps under MAX_DRAW_BYTES (graph_joint_empirical.py:568-569); the comparison here re-materialises the same bytes through the segmented host encoder. Two whole copies of the draw document exist at the comparison. _reconstruct also does `json.loads(draw_payload)[\"rows\"]` (:737) to re-embed the rows into the completion document." + }, + { + "constant": "graph_joint_empirical.py:48 MAX_DRAW_BYTES", + "value": "64 * 64 * 1024**2 = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "graph_joint_empirical.py:569 (JointEmpiricalDrawKernel.run: `_require(len(payload) <= MAX_DRAW_BYTES, \"DRAW_BYTES\")` on `payload = _json(document)` at :568, a single json.dumps)", + "graph_joint_empirical.py:84 (_document default limit, \"PAYLOAD_BYTES\"): reached for support_json (:108), transport_json (:143) and model_metadata (:542), all fixed-shape; for the draw document only via read_joint_empirical_draw :629, which no runner calls (only packages/microcosm-fit/tests/test_graph_joint_empirical.py:193)" + ], + "refusal_codes": [ + "DRAW_BYTES (ValueError GRAPH_JOINT_EMPIRICAL_DRAW_BYTES)", + "PAYLOAD_BYTES (ValueError GRAPH_JOINT_EMPIRICAL_PAYLOAD_BYTES)" + ], + "encodes": "The child draw document (one row per eligible child) as produced by the draw kernel in the graph run; adapter._json is json.dumps with no segmentation.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "11.9-19.0 MB (65,952 eligible children ESTIMATE x 180.5-288.5 B + 770)", + "bytes_at_full_source": "119.0-190.3 MB (659,527 eligible children ESTIMATE x 180.5-288.5 B)", + "measurement_basis": "prop_measure.py child_draw asserts adapter._json(document) == graph_child_property_income._json(document) byte for byte, so the per-row figures (180.5 zero-pattern, 288.5 all-positive) apply. byte-census-measurements.json child_draw: 273.0 B/row, children_the_64MiB_cap_admits 245,820.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "64 * 1024**2 = 67,108,864", + "head_value": "64 * 64 * 1024**2 = 4,294,967,296", + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> extend -> host.run_graph -> adapter.JointEmpiricalDrawKernel.run -> :568 _json(document) -> :569", + "classification": "consumer", + "notes": "binds_at_full_source true at the base value (119 MB lower bound > 64 MiB). Brief \u00a75 and the comment at :44-47 describe this as the decoder's ceiling on the producer's segmented total, but the reachable enforcement (:569) is the kernel's own producer-side check on an unsegmented json.dumps; the decoder path (`_document(payload)` on a draw) is reachable only through read_joint_empirical_draw, which has no runner caller. The receipt at :572-583 carries only counts and digests." + }, + { + "constant": "graph_joint_empirical.py:87 _json(value) == payload", + "value": "consumer-side canonical re-encoding inside _document (encoder _json has no cap; the preceding length check at :84 uses the limit argument)", + "enforced_at": [ + "graph_joint_empirical.py:83-90 (_document, \"CANONICAL_JSON\"); callers on the runner path: :108 _support (support_json), :143 _transport (transport_json), :371 _metadata (model bytes, limit=empirical.MAX_MODEL_BYTES), :542 draw kernel (model_metadata payload); :629 read_joint_empirical_draw (draw document) has no runner caller" + ], + "refusal_codes": [ + "CANONICAL_JSON (ValueError GRAPH_JOINT_EMPIRICAL_CANONICAL_JSON)", + "PAYLOAD_BYTES (ValueError GRAPH_JOINT_EMPIRICAL_PAYLOAD_BYTES) at :84", + "GRAPH_JOINT_EMPIRICAL_JSON at :89 on decode errors" + ], + "encodes": "On the reachable path the largest document is the joint-empirical model (one donor record per ASEC 15-17 person with a known O/D pair; upstream, fraction-independent); support, transport and metadata are fixed-shape. The roster-shaped draw document only reaches this check through read_joint_empirical_draw (unreachable).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "model ~1.25 MB at an ESTIMATED ~5,400 teen donors (231 B/donor + 437); hard ceiling 32.8 MB if every one of the 142,125 ASEC persons were a donor; identical at every fraction", + "bytes_at_full_source": "same as at 1/10 (the donor roster is the whole 2025 ASEC member cohort, current_child_property_income_source.py:639-640 donor_scope complete_original_current_year_source, donor_sampling_fraction_applied False)", + "measurement_basis": "prop_measure.py model_document: joint_empirical.fit_joint_empirical(...).to_bytes() on invented donors at full widths, 4 vs 1,004 rows: 231.0 B/donor, fixed 437 B. Donor count not established from any artifact (ESTIMATE of the ASEC 15-17 share).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> host.run_graph -> adapter.JointEmpiricalFitKernel.run -> _metadata (:465) -> _document(model.to_bytes(), MAX_MODEL_BYTES) (:371); JointEmpiricalDrawKernel.run -> _metadata (:537), _document(metadata_value.payload) (:542), _support/_transport via _draw_expected (:404-407)", + "classification": "stays", + "notes": "Headroom against MAX_MODEL_BYTES is at least 2.0x even at the impossible maximum donor count and about 54x at the estimate. read_joint_empirical_draw (:587) and its _document(payload) at :629 are `reachable_from: none`; the function that would reach it is only the fit package's test." + }, + { + "constant": "graph_joint_empirical.py:49 MAX_RECIPIENTS", + "value": "1_048_576", + "enforced_at": [ + "graph_joint_empirical.py:530 (JointEmpiricalDrawKernel.run: `_require(len(table) <= MAX_RECIPIENTS, \"RECIPIENT_COUNT\")` on the whole RECIPIENT support table, before the eligibility filter at :552)", + "graph_joint_empirical.py:614 (read_joint_empirical_draw, \"RECIPIENT_COUNT\"; no runner caller)" + ], + "refusal_codes": [ + "RECIPIENT_COUNT (ValueError GRAPH_JOINT_EMPIRICAL_RECIPIENT_COUNT)" + ], + "encodes": "A row count, not bytes: the RECIPIENT support population's person table, which graph_child_property_income._projections builds from every stacked person (:378 `recipient_table = _key_table(tuple(recipients.recipient_key))`, comment :376-377 'A nonempty original-recipient Frame remains valid when no child is eligible. The generic draw declares this bool and selects internally.'), so len(table) is the stacked person roster.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "not a byte count: 356,501 stacked persons", + "bytes_at_full_source": "not a byte count: 3,565,013 stacked persons", + "measurement_basis": "Brief \u00a73 stacked person counts; _table(context) at :339 returns the full projected table and :530 checks it before `table = table.loc[table[eligibility]]` at :552.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "1_048_576 (unchanged on this branch)", + "head_value": "1_048_576", + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> extend -> host.run_graph -> adapter.JointEmpiricalDrawKernel.run -> _table (:528) -> :530", + "classification": "not-a-transport", + "notes": "Binds at full source (3,565,013 > 1,048,576; first binds near fraction 0.294) and was not lifted on this branch, unlike MAX_ORIGINALS and MAX_RECIPIENT_ROWS which guard the same roster upstream and were raised to 15,000,000. It is the first refusal the child lane would meet at full source now that the two upstream row ceilings moved. No row-count rule from docs/us-native-row-ceilings.md has been applied to it." + }, + { + "constant": "graph_joint_empirical.py:67 _hash len(value) == 64", + "value": "64 lowercase hex characters (sha256)", + "enforced_at": [ + "graph_joint_empirical.py:67-72 (_hash), used at :198 (\"SOURCE_HASH\"), :267 (\"DRAW_HASHES\"), :623 (\"READER_HASHES\", unreachable), and graph_child_property_income.py:490 (\"HOST_PIN\", every host pin value)" + ], + "refusal_codes": [ + "SOURCE_HASH", + "DRAW_HASHES", + "HOST_PIN (ValueError CHILD_PROPERTY_HOST_PIN)" + ], + "encodes": "A fixed-width scalar field: sha256 hex digests in node params and host pins.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 bytes per digest; fraction-independent", + "bytes_at_full_source": "64 bytes per digest; fraction-independent", + "measurement_basis": "Fixed-width field by definition.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> ChildPropertyBoundary.__init__ -> child_property_nodes -> _bindings (:520) -> :490; joint_empirical_fit_node/draw_node (:582/:598) -> :198/:267", + "classification": "must-not-move", + "notes": "An equality on a digest width, not a capacity bound." + }, + { + "constant": "joint_empirical.py:24 MAX_MODEL_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "joint_empirical.py:376-379 (JointEmpiricalModel._checked: `0 < len(self.payload) <= MAX_MODEL_BYTES`, \"MODEL_BYTES\"), reached by from_bytes :391, to_bytes :395, sha256 :400, diagnostics :404, draw_joint_empirical :498", + "graph_joint_empirical.py:371 (_metadata: `_document(model.to_bytes(), empirical.MAX_MODEL_BYTES)`, \"PAYLOAD_BYTES\")" + ], + "refusal_codes": [ + "MODEL_BYTES (ValueError JOINT_EMPIRICAL_MODEL_BYTES)", + "PAYLOAD_BYTES (ValueError GRAPH_JOINT_EMPIRICAL_PAYLOAD_BYTES)" + ], + "encodes": "The joint-empirical model payload: protocol, targets, support, weight_kind and one donor record per ASEC 15-17 person with a known O/D pair (key, household, values, weight) \u2014 an upstream roster read whole at every fraction.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "~1.25 MB at an ESTIMATED ~5,400 teen donors (231 B/donor + 437); at most 32.8 MB if all 142,125 ASEC persons were donors; identical at every fraction", + "bytes_at_full_source": "same as at 1/10", + "measurement_basis": "prop_measure.py model_document through joint_empirical.fit_joint_empirical(...).to_bytes(): 231.0 B/donor, 437 B fixed; 64 MiB admits 290,514 donors, more than double the whole ASEC member.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> _child_expected -> child._reconstruct -> empirical.fit_joint_empirical (:667) -> JointEmpiricalModel.from_bytes (:483) -> _checked; host.run_graph -> JointEmpiricalFitKernel.run (:457) and JointEmpiricalDrawKernel.run -> from_bytes (:536)", + "classification": "stays", + "notes": "Headroom at least 2.0x at the impossible maximum, ~54x at the estimate. Every _checked() call re-parses and re-encodes the whole model (see the :385 row)." + }, + { + "constant": "joint_empirical.py:25 MAX_DONORS", + "value": "1_048_576", + "enforced_at": [ + "joint_empirical.py:304 (_prepare: `1 <= len(rows) <= MAX_DONORS`, \"DONOR_COUNT\")", + "joint_empirical.py:457-461 (fit_joint_empirical: `1 <= len(frame) <= MAX_DONORS`, \"DONOR_AXIS\")" + ], + "refusal_codes": [ + "DONOR_COUNT (ValueError JOINT_EMPIRICAL_DONOR_COUNT)", + "DONOR_AXIS (ValueError JOINT_EMPIRICAL_DONOR_AXIS)" + ], + "encodes": "A row count, not bytes: donor records in the model (ASEC 15-17 known-pair cohort, at most the 142,125-row member).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "not a byte count: ~5,400 donors ESTIMATE, at most 142,125; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Upstream member row count from the brief \u00a73 (142,125 ASEC persons).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> child._reconstruct -> fit_joint_empirical (:667) -> :458; JointEmpiricalFitKernel.run (:457); _prepare via _checked", + "classification": "not-a-transport", + "notes": "Row count; cannot bind because the donor roster is bounded by the ASEC member (142,125 < 1,048,576)." + }, + { + "constant": "joint_empirical.py:385 _json(document) == self.payload", + "value": "consumer-side canonical re-encoding of the model payload in _checked (encoder _json has no cap; :377 MAX_MODEL_BYTES precedes it)", + "enforced_at": [ + "joint_empirical.py:385 (JointEmpiricalModel._checked, \"MODEL_CANONICAL_BYTES\"), run on every to_bytes (:395), sha256 (:400), diagnostics (:404), from_bytes (:391) and draw_joint_empirical (:498) call" + ], + "refusal_codes": [ + "MODEL_CANONICAL_BYTES (ValueError JOINT_EMPIRICAL_MODEL_CANONICAL_BYTES)" + ], + "encodes": "The model document (upstream donor roster).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "~1.25 MB ESTIMATE, at most 32.8 MB; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Same as the MAX_MODEL_BYTES row (231 B/donor).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> child._reconstruct -> model.sha256 (:731, :749, :773) and draw_joint_empirical (:684) -> _checked -> :385; JointEmpiricalDrawKernel.run -> from_bytes (:536)", + "classification": "stays", + "notes": "Each call re-parses (json.loads) and re-encodes the whole model, plus _prepare's per-donor _json of keys and households (:318, :321); a fixed cost at every fraction." + }, + { + "constant": "current_child_property_income_source.py:34 MAX_ROWS", + "value": "600_000", + "enforced_at": [ + "current_child_property_income_source.py:192 (_raw_axis: `0 < len(frame) <= MAX_ROWS`, \"FULL_SOURCE_AXIS\"; the whole 2025 ASEC literal frame, 142,125 rows)", + "current_child_property_income_source.py:209 (_catalogue_members: `len(households) <= MAX_ROWS`, \"CATALOGUE_TYPE\"; 55,762 ASEC households)", + "current_child_property_income_source.py:223 (H_NUMPER numeric range via _integer_literal, \"H_NUMPER_RANGE\")" + ], + "refusal_codes": [ + "FULL_SOURCE_AXIS", + "CATALOGUE_TYPE", + "H_NUMPER_RANGE (all ValueError CHILD_PROPERTY_ + code)" + ], + "encodes": "Row counts of the upstream ASEC member and catalogue, not bytes; at head no longer applied to the stacked roster (the :399 site moved to MAX_RECIPIENT_ROWS).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "not a byte count: 142,125 ASEC persons / 55,762 ASEC households; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Brief \u00a73 ASEC counts.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "600_000, also enforced at base :393 on the stacked person roster (\"RECIPIENT_AXIS\")", + "head_value": "600_000, ASEC-source sites only", + "reachable_from": "45", + "reach_chain": "extend -> ChildPropertyBoundary.__init__ -> child.qualify_child_property_sources (:955) -> project_child_property_donors (:611) -> _raw_axis (:274-275), _catalogue_members (:286)", + "classification": "not-a-transport", + "notes": "Row count on an upstream file; headroom 4.2x/10.8x; must not be lowered below the real member sizes. The row-count lane's binding verdict for this constant referred to the base :393 site, which this branch re-keyed to MAX_RECIPIENT_ROWS." + }, + { + "constant": "current_child_property_income_source.py:39 MAX_RECIPIENT_ROWS", + "value": "15_000_000", + "enforced_at": [ + "current_child_property_income_source.py:399 (project_child_property_recipients: `len(table) <= MAX_RECIPIENT_ROWS`, \"RECIPIENT_AXIS\")" + ], + "refusal_codes": [ + "RECIPIENT_AXIS (ValueError CHILD_PROPERTY_RECIPIENT_AXIS)" + ], + "encodes": "A row count, not bytes: origins[\"persons\"][\"rows\"] rebuilt as a DataFrame (:384), the stacked person roster.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "not a byte count: 356,501 stacked persons", + "bytes_at_full_source": "not a byte count: 3,565,013 stacked persons", + "measurement_basis": "Brief \u00a73 stacked person counts.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "none: the site used MAX_ROWS = 600_000", + "head_value": "15_000_000", + "reachable_from": "45", + "reach_chain": "extend -> ChildPropertyBoundary.__init__ -> qualify_child_property_sources (:955) -> project_child_property_recipients (:619) -> :395-401", + "classification": "not-a-transport", + "notes": "binds_at_full_source true at the base value (3,565,013 > 600,000), false at head. Rule (comment :35-38): four times the measured full-source stacked person roster rounded up to the next whole million. Recorded in the _live contract (:114) so its value is part of the implementation seal." + }, + { + "constant": "current_child_property_income_source.py:40 MAX_PROJECTION_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "current_child_property_income_source.py:55 (_json: `require(len(result) <= MAX_PROJECTION_BYTES, \"PROJECTION_SIZE\")`), called per coordinate tuple in _table_stamp (:556) and on the evidence dict (:579 seal, :656 qualify)" + ], + "refusal_codes": [ + "PROJECTION_SIZE (ValueError CHILD_PROPERTY_PROJECTION_SIZE)" + ], + "encodes": "Fixed-shape documents only: one encoded key coordinate at a time (tens of bytes) and the aggregate evidence dict (protocol strings, digests, three nested source-evidence dicts, counts).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "a few KB (evidence dict); fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Read from :631-654: the evidence carries counts and digests, no rows. Not measured.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> ChildPropertyBoundary.__init__ -> qualify_child_property_sources -> _json (:656) and child_property_sources_seal -> _table_stamp -> _json (:556)", + "classification": "not-a-transport", + "notes": "A memory ceiling with no roster document behind it; cannot bind on the fraction. Note _table_stamp encodes every recipient_key coordinate one at a time (3,565,013 small _json calls at full source) before streaming into physical._table_stamp." + }, + { + "constant": "current_child_property_income_source.py:178 len(value) <= 64", + "value": "64 characters per integer literal", + "enforced_at": [ + "current_child_property_income_source.py:173-179 (_integer_literal, name + \"_LITERAL\"), entered from :202 PH_SEQ, :203 A_LINENO, :223 H_NUMPER, :228 H_SEQ, :232 HSUP_WGT, :239 A_LINENO" + ], + "refusal_codes": [ + "PH_SEQ_LITERAL", + "A_LINENO_LITERAL", + "H_NUMPER_LITERAL", + "H_SEQ_LITERAL", + "HSUP_WGT_LITERAL (ValueError CHILD_PROPERTY_ + name + _LITERAL)" + ], + "encodes": "One ASEC source literal token at a time (printed widths 1-9).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "1-9 characters per token; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Printed ASEC field widths (COORDINATE_WIDTHS PH_SEQ 5, A_LINENO 2; HSUP_WGT up to 9 digits).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> qualify_child_property_sources -> project_child_property_donors -> _raw_axis / _catalogue_members -> _integer_literal", + "classification": "must-not-move", + "notes": "Token width guard; the numeric range caps at the same sites (99999, 16, MAX_ROWS, 999999999) are domain checks, not byte bounds." + }, + { + "constant": "current_child_property_income_source.py:197 0 < len(v) <= 64 (PERIDNUM)", + "value": "64 characters", + "enforced_at": [ + "current_child_property_income_source.py:196-200 (_raw_axis, \"PERIDNUM\")" + ], + "refusal_codes": [ + "PERIDNUM (ValueError CHILD_PROPERTY_PERIDNUM)" + ], + "encodes": "One ASEC PERIDNUM literal per member row (22 digits in the source; current_asec_income_routing_source.py:1084 enforces exactly 22 on the read).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "22 characters; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "preparation.json ASEC raw_native_person_id sample '2156659610400131701102' (22 chars); routing :1084 regex [0-9]{22}.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> qualify_child_property_sources -> project_child_property_donors -> _raw_axis (:274-275)", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "current_asec_income_routing_source.py:50 TOKEN_MAX_CHARS", + "value": "64", + "enforced_at": [ + "current_asec_income_routing_source.py:476 (literal_code, \"TOKEN_BOUND\"; every _codes_frame call in this module, interest and dividend)", + "current_asec_income_routing_source.py:1106 (_read_capture, \"TOKEN_BOUND:\" + name; every non-coordinate, non-pattern field of every member record; interest :331 and dividend :355 call it with patterns={} so all their amount fields hit this site too)", + "current_asec_income_routing_source.py:1228 (project_income_routing, \"ROUTING_TOKEN_CONTRACT:\" + name)", + "current_asec_retirement_detail_source.py:226 (\"TOKEN_BOUND\"; not on the runners' path)" + ], + "refusal_codes": [ + "TOKEN_BOUND", + "TOKEN_BOUND:", + "ROUTING_TOKEN_CONTRACT: (all ValueError CURRENT_ASEC_INCOME_ROUTING_ + reason)" + ], + "encodes": "One ASEC CSV field literal at a time.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "1-8 characters per token; fraction-independent (the 2025 member is read whole: 142,125 rows x 46 READ_COLUMNS at every fraction)", + "bytes_at_full_source": "same", + "measurement_basis": "Printed widths in RECEIPT_ENTRIES/ACCOUNT_ENTRIES/ALLOCATION_ENTRIES and AMOUNT_FIELDS (max 8).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial -> register_property_kernels -> _Kernel._qualified -> sources.qualify_current_property_income_sources -> routing.qualify_current_asec_income_routing (:339) -> _read_capture (:1321); interest/dividend _capture_member -> routing._read_capture (:331/:355); child.qualify_child_property_sources -> interest/dividend qualifiers (:600, :602)", + "classification": "must-not-move", + "notes": "Also recorded in current_child_property_income_source._live (:138) as part of the implementation seal. The member row cap at :1081/:1110 (`len(records) < rows`, `== rows`) is pinned to coverage._MEMBER_PINS rows (142,125), an upstream row count, not a byte bound." + }, + { + "constant": "current_asec_income_routing_source.py:49 COORDINATE_WIDTHS", + "value": "{PH_SEQ: 5, A_LINENO: 2, A_AGE: 2} characters", + "enforced_at": [ + "current_asec_income_routing_source.py:1085-1092 (_read_capture regex `[0-9]{1,width}`, \"COORDINATE:\" + name) on every member record, entered from :1321 and from interest :331 / dividend :355" + ], + "refusal_codes": [ + "COORDINATE:PH_SEQ", + "COORDINATE:A_LINENO", + "COORDINATE:A_AGE (ValueError CURRENT_ASEC_INCOME_ROUTING_COORDINATE:)" + ], + "encodes": "Fixed-width printed coordinate fields of one ASEC record.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "at most 5/2/2 characters; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Printed dictionary widths.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "same as TOKEN_MAX_CHARS (routing/interest/dividend qualifiers -> _read_capture)", + "classification": "must-not-move", + "notes": "Recorded in child._live (:137)." + }, + { + "constant": "current_asec_income_routing_source.py:1084 [0-9]{22} (PERIDNUM)", + "value": "exactly 22 digits", + "enforced_at": [ + "current_asec_income_routing_source.py:1084 (_read_capture, \"PERSON_KEY\") on every member record" + ], + "refusal_codes": [ + "PERSON_KEY (ValueError CURRENT_ASEC_INCOME_ROUTING_PERSON_KEY)" + ], + "encodes": "The ASEC person key literal.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "22 characters; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Regex width.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "same as TOKEN_MAX_CHARS", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "current_asec_income_routing_source.py:1488-1491 _sha(out.to_json(orient=\"table\").encode()) == evidence[\"projection_sha256\"] and _sha(literals.to_json(orient=\"table\").encode()) == evidence[\"literals_sha256\"]", + "value": "consumer-side re-encoding compared by digest; encoder cap none (pandas to_json, produced at :1472-1473 in the same function)", + "enforced_at": [ + "current_asec_income_routing_source.py:1487-1492 (qualify_current_asec_income_routing, \"FINAL_PROJECTION_CHANGED\")" + ], + "refusal_codes": [ + "FINAL_PROJECTION_CHANGED (ValueError CURRENT_ASEC_INCOME_ROUTING_FINAL_PROJECTION_CHANGED)" + ], + "encodes": "Two roster-shaped tables: `out` = the selected ASEC persons x 158 columns (five family projections + 27 amount status/validity/zero_origin columns + native_person_id), and `literals` = the whole 2025 member x 46 READ_COLUMNS (upstream, 142,125 rows at every fraction).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "out 104.2 MB (14,212 selected ASEC persons x 7,329 B + 12,039); literals 96.4 MB (142,125 x 678 B, fraction-independent)", + "bytes_at_full_source": "out 1.04 GB (142,125 x 7,329 B); literals 96.4 MB", + "measurement_basis": "prop_measure_2.py routing_projection: routing.project_income_routing(raw, ages) on invented arrays plus the 27 status columns and native_person_id added at :1354-1369, then .to_json(orient='table').encode(), 3 vs 503 rows: 7,329 B/row, 158 columns. routing_literals: 46-column string frame, 3 vs 1,003 rows: 678 B/row.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "register_property_kernels -> _Kernel._qualified -> sources.qualify_current_property_income_sources -> routing.qualify_current_asec_income_routing (:339, and again per requalification iteration :370) -> :1472-1473, :1488-1491", + "classification": "stays", + "notes": "No cap exists, so it never binds; each qualify call materialises both strings twice (once for the evidence digest, once for the final check), ~2.3 GB of transient string at full source. The dividend and interest seals (dividend_values_seal :307-341, interest_values_seal :283-317) stream through hashlib via physical._series_digest and carry no bound; their `person`/`asec_literals` DataFrames (142,125 rows each) are retained whole in memory." + }, + { + "constant": "asec_coverage_authentication.py:39 _BODY_MAX (entered from this family as `budget=[coverage._BODY_MAX]`)", + "value": "268_435_456 (256 MiB)", + "enforced_at": [ + "asec_coverage_authentication.py:283-286 (_CsvBounds.feed: `self.budget[0] -= 4 + _NUMBERS.size + 12 + 22 + 32` per record, `-= 1` per PRPERTYP token byte, `_require(self.budget[0] >= 0, \"COVERAGE_BODY_BYTES\")`), inside _capture (:309-345)", + "entry sites in this family: current_asec_income_routing_source.py:1314-1320, current_asec_interest_source.py:324-330, current_asec_dividend_source.py:348-354" + ], + "refusal_codes": [ + "COVERAGE_BODY_BYTES (AsecCoverageAuthenticationError, a ValueError subclass)" + ], + "encodes": "The pre-charge for an encoded coverage body of the pinned 2025 ASEC person member (pppub25.csv, 142,125 records, 277,882,549 bytes): 119 bytes per record (4 + 48 for struct '<6q' + 12 + 22 + 32 + 1 PRPERTYP byte).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "16,912,875 charged (142,125 x 119); fraction-independent", + "bytes_at_full_source": "same 16,912,875", + "measurement_basis": "prop_measure.py asec_body_charge from coverage._NUMBERS.size (48) and the :283/:285 arithmetic; the budget admits 2,255,760 records, headroom 15.87x.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "register_property_kernels -> _Kernel._qualified -> sources.qualify_current_property_income_sources -> interest (:338) / routing (:339) / dividend (:340) qualifiers -> coverage._capture(budget=[coverage._BODY_MAX]); child.qualify_child_property_sources -> interest/dividend (:600/:602)", + "classification": "must-not-move", + "notes": "Defined and owned by the ASEC coverage family; this family only passes it as the encoder limit argument. The same _capture also enforces literal.MAX_MEMBER_BYTES = 1,000,000,000 (asec_person_coverage_source.py:33, at :314-317), _ROW_MAX = 1,048,576, _TOKEN_MAX = 65,536 and _CSV_HEADER_MAX = 65,536 (:272-276), all fraction-independent guards on the same upstream member; they are keyed by that family. Each of the three qualifiers copies the 278 MB member into a private temp dir once per call, and qualify_current_property_income_sources calls each qualifier twice (:351, :370)." + }, + { + "constant": "current_acs_income_anchor_source.py:44 len(value) <= 64 (_literal)", + "value": "64 characters", + "enforced_at": [ + "current_acs_income_anchor_source.py:43-45 (_literal, \"LITERAL_TYPE_OR_BOUND\"), applied to token/age/adjustment/allocation in parse_anchor (:57-59) for every selected ACS person and both anchors (_parsed_table :298-307)" + ], + "refusal_codes": [ + "LITERAL_TYPE_OR_BOUND (ValueError ACS_INCOME_ANCHOR_LITERAL_TYPE_OR_BOUND)" + ], + "encodes": "One ACS PUMS field literal at a time (INTP/RETP up to 7 chars, AGEP 2, ADJINC 7, allocation flag 1).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "1-7 characters per token; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Printed PUMS widths and the parse_anchor regexes (:60-63, :71-76).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "sources.qualify_current_property_income_sources -> acs.qualify_current_acs_income_anchors (:337) -> _parsed_table (:413) -> parse_anchor -> _literal; child.qualify_child_property_sources -> acs (:604)", + "classification": "must-not-move", + "notes": "Token width guard." + }, + { + "constant": "current_acs_income_anchor_source.py:134 all(len(v) <= 64 for v in row.values())", + "value": "64 characters per CSV cell", + "enforced_at": [ + "current_acs_income_anchor_source.py:134 (_scan, \"SOURCE_TOKEN_BOUND\") on every record of every person member of the captured ACS archive" + ], + "refusal_codes": [ + "SOURCE_TOKEN_BOUND (ValueError ACS_INCOME_ANCHOR_SOURCE_TOKEN_BOUND)" + ], + "encodes": "The eight COLUMNS cells of one ACS person record; the whole archive (3,422,888 records) is scanned at every fraction to pick the wanted keys.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "at most 13 characters per cell (SERIALNO); fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "byte-census-measurements.json acs_household_archive.serialno_widths {\"13\": 1,631,969}; other columns are 1-7 digits.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "acs.qualify_current_acs_income_anchors -> _capture_person (:405) -> _scan (:372) -> :134", + "classification": "must-not-move", + "notes": "The row cap at :132 (`count <= maximum`, \"SOURCE_ROW_SHAPE\") is pinned to the catalogue's people count (3,422,888) via :376 `maximum=total_rows - count`; an upstream row count, not a byte bound. _capture_person copies the 603 MB person archive to a temp dir on every qualify call (twice per qualify_current_property_income_sources, once per qualify_child_property_sources)." + }, + { + "constant": "current_acs_income_anchor_source.py:104/:108 _key regexes 2024(?:HU|GQ)[0-9]{7} and [0-9]{1,2} with 1 <= SPORDER <= 20", + "value": "SERIALNO exactly 13 characters; SPORDER 1-2 digits, at most 20", + "enforced_at": [ + "current_acs_income_anchor_source.py:102-112 (_key, \"SOURCE_HOUSEHOLD_KEY\" / \"SOURCE_PERSON_KEY\"), called at :135 (every scanned archive record), :189 (every ACS origin), :212 (_raw_table re-check)" + ], + "refusal_codes": [ + "SOURCE_HOUSEHOLD_KEY", + "SOURCE_PERSON_KEY (ValueError ACS_INCOME_ANCHOR_ + reason)" + ], + "encodes": "The fixed-width ACS household/person key.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "13 + 1-2 characters; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "byte-census-measurements.json: all 1,631,969 SERIALNOs are 13 wide; 4,139 person rows have a two-digit SPORDER.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "acs.qualify_current_acs_income_anchors -> _origins (:397) -> _key (:189); _capture_person -> _scan -> _key (:135)", + "classification": "must-not-move", + "notes": "The SPORDER <= 20 domain is the same household-size domain current_survey_household_roles.MAX_HOUSEHOLD_MEMBERS = 20 asserts." + }, + { + "constant": "current_acs_income_anchor_source.py:336 qualified.projection == qualified.anchors.reset_index().to_json(orient=\"table\", index=False).encode()", + "value": "consumer-side re-encoding in income_anchor_seal; encoder cap none (pandas to_json; produced at :414)", + "enforced_at": [ + "current_acs_income_anchor_source.py:334-340 (income_anchor_seal, \"PROJECTION_BINDING\"), called at :432 and :440 (qualify), current_property_income_sources.py:70 (_source_seals, inside property_income_sources_seal :91, itself called at :367, :372 x5 loop iterations, :393, graph_current_survey_property.py:185, :928, :931, current_property_completion_routing.py:342, :593), current_property_income_sources.py:344/:353/:370 (qualifier loop), current_child_property_income_source.py:609, :684" + ], + "refusal_codes": [ + "PROJECTION_BINDING (ValueError ACS_INCOME_ANCHOR_PROJECTION_BINDING)" + ], + "encodes": "The ACS anchors projection: one pandas table-orient row per selected ACS person (8 raw COLUMNS + native_person_id, source_year, dollar_year + two anchors x six parsed fields).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "233.8 MB (342,288 selected ACS persons x 683 B + 1,291)", + "bytes_at_full_source": "2.34 GB (3,422,888 x 683 B)", + "measurement_basis": "prop_measure.py acs_projection: acs._parsed_table(raw, origins).reset_index().to_json(orient='table', index=False).encode() on invented rows at full widths, 3 vs 1,003 rows: 683.0 B/row, fixed 1,291 B.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "register_property_kernels -> _Kernel._qualified (every property kernel run) -> sources.qualify_current_property_income_sources -> acs.qualify_current_acs_income_anchors (:337) -> :414 produce, :432/:440 seal; property_income_sources_seal -> _source_seals -> income_anchor_seal; child.qualify_child_property_sources -> :604, :609, :684", + "classification": "stays", + "notes": "No cap, so it never binds, but every seal call re-materialises the whole 2.34 GB string at full source and compares it to the retained 2.34 GB `projection` bytes held on QualifiedAcsIncomeAnchors; qualify_current_property_income_sources alone triggers it at least 2 (qualify) + 1 (:353) + 5x2 (:370-372 loop) + 2 (:367, :393) times, and each property kernel run (six kernel classes across 16 nodes) requalifies. The routing module's evidence digest of the same shape is separate (see the :1488 row)." + }, + { + "constant": "model_input.py:18 MATRIX_HEADER_MAX_BYTES (microcosm.fit; this family produces two more matrices through it)", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "model_input.py:86-87 (encode_recipient_matrix: `len(header) > MATRIX_HEADER_MAX_BYTES` -> ValueError \"Recipient matrix header exceeds its size bound.\"), entered from current_property_income_sources.py:239 (recipient_matrix) and graph_current_survey_property.py:173 (donor_matrix)", + "model_input.py:105-106 (decode_recipient_matrix: `length > MATRIX_HEADER_MAX_BYTES` -> ValueError \"Recipient matrix header is truncated or exceeds its bound.\"), entered from current_property_income_sources.py:213 (the 19-node family's predictor matrix)" + ], + "refusal_codes": [ + "ValueError(\"Recipient matrix header exceeds its size bound.\")", + "ValueError(\"Recipient matrix header is truncated or exceeds its bound.\")" + ], + "encodes": "The JSON header of a recipient matrix (schema_version, entity, rows, columns, index_name, index_identity = qrf._IndexIdentity digest, fixed size per qrf.py:577-600), length-prefixed by a 4-byte big-endian field (:91). The float64 body (8 x (1 + columns) bytes per row) is uncapped.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "header 488 B incl. magic and length at every fraction; body (not bounded): recipient_matrix 15.6 MB (278,964 ACS adults ESTIMATE x 56 B), donor_matrix at most 0.65 MB", + "bytes_at_full_source": "header 488 B; body: recipient_matrix 156.2 MB (2,789,653 ACS adults ESTIMATE x 56 B), donor_matrix at most 6.5 MB", + "measurement_basis": "prop_measure.py matrices: model_input.encode_recipient_matrix with the 45-node pilot's demographic_conditioning=True roster (5 predictors, byte-census-measurements.json predictor_projection.matrix) + property_reported_total = 6 float64 columns, 1 vs 1,001 rows: 56.01 B/row, 488 B fixed. Recipient rows = ACS selected persons aged 15+ with a known INTP (:230-234), ESTIMATED as 81.5% of ACS persons; donor rows = ASEC joint-eligible persons (<= the ASEC selected roster).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "register_property_kernels -> _Kernel._qualified -> sources.qualify_current_property_income_sources -> _compose -> encode_recipient_matrix (:239) and decode_recipient_matrix (:213); graph_current_survey_property._projection_values (:173) from _params (:338), _qualified (:853), _projection_result (:225)", + "classification": "must-not-move", + "notes": "The header is fixed-shape (the index identity is a sha256 of pandas hashes, not the index values), so the 64 KiB cap cannot bind; it protects the length-prefixed header parse. Definition and the predictor matrix belong to the 19-node family's model_input row; recorded here for the two property matrices." + }, + { + "constant": "graph_current_survey_property.py:245 artifacts[PROJECTION_NODE, name] == payload (donor_matrix, recipient_matrix, projection)", + "value": "consumer-side byte equality of re-encoded projection-node artifacts; encoder caps: none on the matrix bodies (MATRIX_HEADER_MAX_BYTES on the header), none on the evidence projection (codec.encode_json)", + "enforced_at": [ + "graph_current_survey_property.py:241-249 (_checked_projection_result, \"PROJECTION_ARTIFACTS\")", + "graph_current_survey_property.py:854-858 (_Kernel._qualified for every non-projection property node: `context.artifacts[edge.name].payload == projected[edge.artifact]`, \"SOURCE_ARTIFACT_BYTES\")" + ], + "refusal_codes": [ + "PROJECTION_ARTIFACTS (ValueError CURRENT_SURVEY_PROPERTY_PROJECTION_ARTIFACTS)", + "SOURCE_ARTIFACT_BYTES (ValueError CURRENT_SURVEY_PROPERTY_SOURCE_ARTIFACT_BYTES)" + ], + "encodes": "donor_matrix (encode_recipient_matrix over ASEC joint-eligible persons, re-encoded at :173 on every _projection_values call), recipient_matrix (over ACS eligible adults, encoded once in sources._compose :239 and retained), projection (fixed-shape evidence dict :180-215).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "recipient_matrix 15.6 MB (ESTIMATE 278,964 ACS adults x 56 B); donor_matrix at most 0.65 MB; projection ~2 KB", + "bytes_at_full_source": "recipient_matrix 156.2 MB (ESTIMATE 2,789,653 x 56 B); donor_matrix at most 6.5 MB", + "measurement_basis": "prop_measure.py matrices (56.01 B/row through model_input.encode_recipient_matrix); adult share ESTIMATED at 81.5% of ACS persons.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial (base call of the child path) -> reconstruct_property_results (:1695) -> _checked_projection_result (:654); verify_materialized_property_income (:1857; _pure_run :881) -> reconstruct_property_results (:1112); run_graph -> CurrentSurveyPropertyAttachKernel.run -> _reconstruct_attachment_results (:675); run_graph -> Filter/Columns/Attach kernels -> _qualified (:852-859)", + "classification": "stays", + "notes": "No cap on the bodies, so it never binds. The donor matrix is re-encoded on every _projection_values call (:173): in _params (once per current_survey_property_nodes call), in _qualified for each of the five non-projection property kernel runs, and in each reconstruct. The retained recipient_matrix bytes (156 MB at full source) live on QualifiedPropertyIncomeSources for the run's lifetime." + }, + { + "constant": "graph_current_survey_property.py:245 artifacts[PROJECTION_NODE, \"completion_routing\"] == routing.payload", + "value": "consumer-side byte equality of the completion routing payload rebuilt by current_property_completion_routing.build_property_completion_routing (:228); encoder cap none (json.dumps at current_property_completion_routing.py:577-591)", + "enforced_at": [ + "graph_current_survey_property.py:241-249 (_checked_projection_result, \"PROJECTION_ARTIFACTS\") when options.completion_routing is true (the 45-node pilot's execution-config.proposed.json sets property_income.completion_routing: true); _reconstruct_attachment_results (:676) passes completion_routing=False so the attach kernel does not rebuild it" + ], + "refusal_codes": [ + "PROJECTION_ARTIFACTS (ValueError CURRENT_SURVEY_PROPERTY_PROJECTION_ARTIFACTS)" + ], + "encodes": "The private completion routing payload: five _table_document tables \u2014 person (one row per stacked person, 12 columns), components (two rows per stacked person, 8 columns), reasons (one row per stacked person, 22 booleans), clones (one row per clone person, 4 columns + index), summary (fixed).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "212.9 MB (356,501 stacked persons x 597.33 B + 2,852)", + "bytes_at_full_source": "2.13 GB (3,565,013 x 597.33 B)", + "measurement_basis": "prop_measure.py completion_routing: tables shaped from build_property_completion_routing :442-576 (ACS-arm rows) encoded with the module's own _table_document and the identical json.dumps call, 2 vs 1,002 stacked persons: 597.33 B per stacked person (person + 2 component rows + reasons row + 2 clone rows), fixed 2,852 B. The 'stage' and flag keys match :578-585.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial (base call with property_income.completion_routing=True) -> run_graph -> CurrentSurveyPropertyProjectionKernel.run -> _projection_result (:914) -> build_property_completion_routing (:228) [producer]; reconstruct_property_results (:1695) and verify_materialized_property_income (:1857, :1112; _pure_run :881) -> _checked_projection_result -> _projection_result -> rebuild -> :245", + "classification": "stays", + "notes": "No cap anywhere: the payload is materialised whole by json.dumps (:577-591) at least three times per cold run (projection kernel, reconstruct at :1695, verify at :1857) and once more on every _pure_run/checked_view, each 2.13 GB at full source, and stored whole as the projection node's completion_routing artifact. The public receipt carries only the allowlisted summary (:230-234, property_completion_public_summary). The row-count lane did not census this document." + }, + { + "constant": "graph_property_tax_leaves.py:542 context.artifacts[\"rebase\"].payload == codec.encode_json(document)", + "value": "consumer-side re-encoding of the leaves kernel's diagnostics by the gate kernel; encoder cap none (codec.encode_json is a bare json.dumps, _graph_legacy_qrf.py:26-29)", + "enforced_at": [ + "graph_property_tax_leaves.py:541-544 (PropertyTaxLeafGateKernel.run, \"DIAGNOSTIC_PAYLOAD\")", + "graph_atomic_survey_financial.py:445-451 (_reconstruct_tax: `loaded[node.id, name] == payload` for the diagnostics and verification artifacts, \"PROPERTY_TAX_ARTIFACT\")" + ], + "refusal_codes": [ + "DIAGNOSTIC_PAYLOAD (ValueError PROPERTY_TAX_LEAVES_DIAGNOSTIC_PAYLOAD)", + "PROPERTY_TAX_ARTIFACT (ValueError from graph_atomic_survey_financial.require)" + ], + "encodes": "The tax-leaf diagnostics document (_diagnostics :338-371): fixed keys plus three roster lists over every row of the receiving population \u2014 person_ids (str per clone person), interest_known and dividend_known (bool per clone person). The population is the property receiving frame (FILTER-all over the completed clone population in the 45-node union, host._tax_nodes at graph_survey_completion_host.py:754), i.e. clone persons.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "14.3 MB (713,002 clone persons x 20 B + 2,724)", + "bytes_at_full_source": "142.6 MB (7,130,026 x 20 B + 2,724)", + "measurement_basis": "prop_measure.py tax_diagnostics: codec.encode_json(tax._diagnostics(person, split, params, bindings)) on an invented person frame at 7-digit clone ids with known interest and dividends, 2 vs 2,002 rows: 20.0 B/clone person, 2,724 B fixed (three binding entries).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> extend -> host._tax_nodes (:754) -> property_tax_leaf_nodes; host.run_graph -> PropertyTaxLeavesKernel.run (:486-489 produce) and PropertyTaxLeafGateKernel.run (:540-547) -> :542; host._reconstruct_tax (:805 in extend, :834 in _pure_run) -> both kernels again -> :447", + "classification": "stays", + "notes": "No cap in the runtime, so it never binds, but the document is materialised whole at least four times per tax pass (:489 artifact, :542 re-encoding, :547 verification artifact, and again in _reconstruct_tax) and \u2014 because :491 `receipt=document` and :551 `\"evidence\": document` place the roster in both node receipts \u2014 it is frozen into the manifest (graph_atomic_survey_population._states :107 `_freeze_json(receipt)`), compared by _check_node_states (graph_survey_population.py:921 `node.receipt == state[\"receipt\"]`) and canonical_json'd whole by manifest.to_json_bytes (graph_survey_completion_host.py:851, :867): about 285 MB of manifest JSON at full source for the two copies. The 45-node pilot harness's own export limit for manifest.json is 16 MiB (execution-config.proposed.json external_export_limits_bytes), which two copies at 20 B per clone person exceed above about 420,000 clone persons, roughly 1/17 of source \u2014 a harness limit, not a runtime bound. The signed-reconciliation summary compared at :741-745 and the draw summary at :506 are fixed-shape aggregates (graph_signed_reconciliation.py:220-228, graph_property_income.py:327-336)." + }, + { + "constant": "graph_property_income_receipts.py:224 codec.encode_raw_target(result.raw_draw, target=target, index=recipient.index) == raw_payload", + "value": "consumer-side re-encoding of each of the four raw property draws (LegacyQRFApplyKernel outputs); encoder cap none (_graph_legacy_qrf.encode_raw_target :170-175: magic + fixed metadata + float64 body)", + "enforced_at": [ + "graph_property_income_receipts.py:222-229 (verify_property_model_receipts, \"APPLICATION_VALUES\"), once per PROPERTY_COMPONENTS target" + ], + "refusal_codes": [ + "APPLICATION_VALUES (ValueError PROPERTY_MODEL_RECEIPTS_APPLICATION_VALUES)" + ], + "encodes": "A raw draw: 8 bytes per recipient row (ACS eligible adults) plus a fixed metadata line whose index identity is a digest (qrf._IndexIdentity), for each of the four components.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "2.23 MB per target (ESTIMATE 278,964 ACS adults x 8.003 B + 242), four targets", + "bytes_at_full_source": "22.3 MB per target (ESTIMATE 2,789,653 x 8 B), four targets", + "measurement_basis": "prop_measure.py raw_target: codec.encode_raw_target on invented rows at full id widths, 1 vs 1,001 rows: 8.003 B/row, 242 B fixed. Recipient count ESTIMATED as 81.5% of ACS persons.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial (base call of the child path, property_graph is not None) -> graph_atomic_survey_financial.py:1798-1808 verify_property_model_receipts(property_nodes, expected[DONOR_COLUMNS_NODE], expected[RECIPIENT_COLUMNS_NODE], loaded) -> :219-229", + "classification": "stays", + "notes": "A numpy body compared whole against the stored artifact; no bound to meet. The same raw draws are decoded by codec.read_raw_target in graph_current_survey_property._read_draws (:493) and graph_property_income.PropertyIncomeDrawColumnsKernel.run (:320), whose `len(body) != len(index) * 8` check (_graph_legacy_qrf.py:190-193) is an equality with the actual row count, not a ceiling. The declaration check at :90-98 (canonical_json of normative nodes) is fixed-shape." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prop/prop_measure.py", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prop/prop-measurements.json", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prop/prop_measure_2.py", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prop/prop-measurements-2.json", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prop/prop-derived-totals.json" + ], + "open_questions": [ + "Eligible-child count is not established by any recovered artifact: the 45-node pilot (native45-v5) exported no manifest (cold.log is empty, no manifest.json/graph.json outside the graph store) and the 1/1000 preparation carries no age column. Every draw-document byte figure uses the row-count lane's 18.5% under-15 share (ESTIMATE); the base-64 MiB binding verdict at full source is robust for any share above 10.4% of stacked persons, and the pattern mix (180.5 B zero rows vs 288.5 B positive rows) only widens the range. The recipient-matrix and raw-draw figures likewise use an ESTIMATED 81.5% ACS adult share.", + "graph_joint_empirical.MAX_RECIPIENTS = 1,048,576 (:49, enforced :530 before the eligibility filter on the whole RECIPIENT support table, which graph_child_property_income._projections :378 builds from every stacked person) binds at full source (3,565,013 rows; first at about fraction 0.294) and was not lifted on this branch, while the two upstream ceilings on the same roster (MAX_ORIGINALS, MAX_RECIPIENT_ROWS) were raised to 15,000,000. It is now the first refusal on the child lane above roughly 1/3 of source. Its sibling MAX_DONORS cannot bind.", + "The completion routing payload (current_property_completion_routing.py:577-591, produced on the 45 path because the pilot config sets completion_routing=true) has no cap at all and measures 597 B per stacked person: 2.13 GB at full source, materialised whole by json.dumps at least three times per cold run and once per _pure_run, and compared byte-for-byte at graph_current_survey_property.py:245. It is not in the row-count census. Whether it should take the segmented shape, a cap, or be dropped from the stored artifact roster is a design question for the lane.", + "The tax-leaf diagnostics roster (20 B per clone person, 142.6 MB at full source) is carried inside two node receipts (graph_property_tax_leaves.py:491 and :551) and therefore inside the manifest JSON (about 285 MB at full source) with no runtime cap; the pilot harness's 16 MiB manifest export limit would refuse it above about 1/17 of source. Whether receipts should carry per-row lists at all is outside this census's remit but affects the transport lane.", + "current_acs_income_anchor_source retains a 683 B-per-ACS-person to_json projection (2.34 GB at full source) as bytes on QualifiedAcsIncomeAnchors and re-encodes it on every income_anchor_seal call (dozens of times per property kernel run through property_income_sources_seal); current_asec_income_routing_source materialises a 7,329 B-per-selected-ASEC-person to_json string (1.04 GB at full source) twice per qualify. Neither has a cap, so neither binds, but both are whole-copy materialisations the transport lane may want to convert to streamed digests like the interest/dividend seals already are.", + "current_property_income_sources.py:320 re-encodes the whole origins block of the preparation (codec.encode_json(json.loads(entry[1])[\"origins\"])) with no cap: 718,800 bytes at 1/1000, SCALED to about 720 MB at full source and 72 MB at 1/10 (prop-measurements-2.json origins_document); child (:259, :618), acs (:396) and completion routing (:130) each json.loads the same origins/preparation payload again. These are consumer-side parses of the 19-node family's segmented preparation document, not bounds, and are recorded here only as sizes.", + "graph_child_property_income.py:85 cites docs/us-native-byte-transports.md, which does not exist in this tree; the neighbouring docs are us-native-scale-transport.md and us-native-row-ceilings.md.", + "No byte bounds of their own were found in graph_survey_completion.py, graph_survey_completion_host.py, current_asec_property_basis.py, property_income_constants.py, graph_property_income.py, current_property_income_sources.py, current_property_completion_routing.py, current_asec_dividend_source.py or current_asec_interest_source.py; the last two enter coverage._BODY_MAX and routing's _read_capture widths (recorded), copy the 278 MB member per call, and seal through streaming digests. asec_coverage_authentication's sibling limits (_ROW_MAX, _TOKEN_MAX, _CSV_HEADER_MAX, literal.MAX_MEMBER_BYTES) are enforced in the same _capture this family enters but are keyed by the ASEC family.", + "The 45-node graph's household-roles and person-status modules were not in this family's module list and were not read; per the brief they are reachable only from the 49/51-node variants." + ] + }, + { + "family": "Roles, person status and the PUF modules the runners import", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_household_roles.py (1-1506, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_household_roles.py (1-566, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_person_status.py (1-822, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_person_status.py (1-365, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_person_status_source.py (1-541, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py (1-1714, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py (1-2037, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_raw_source.py (1-1727, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py (1-1215, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_qrf_chain.py (1-1112, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py (1-4127, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source.py (1-520, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_aggregate_records.py (1-1541, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py (1-690, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py (1-801, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py (1-868, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_puf_transfer.py (1-408, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py (1-460, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py (1-4684, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/post_transfer_calibration.py (1-1856, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_raw_source_definition.json (single line; csv_profile, fields, sources)", + "packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py:64 (PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID definition)", + "packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py:22,630-638,763-766 (grep only: enforcement of the clone-safe id bound)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py:45-60,76-82,116-130 (_encode, MAX_PAYLOAD_BYTES, error class)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py:265-300 (_bounded_json), :790-812 and :1035-1050 (lazy puf_support imports)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py:120-160 (_person_status_module lazy import)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_completion_host.py:25-70 (_roles lazy import, household_roles flag)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py:88-92,149-150,308-322,520-545 (puf_support call sites, COMBINED_CLONE_ATTACHMENT_FRACTION = 1.0)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py:1055-1090 (acs_household_reference_states diagnostics shape)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py:22-27,41-60,108-152 (_origins columns, _key widths)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py:36-50,315,378,389 (constants and helper names used by the roles/status scans)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py:42-43,131-140,160 (_copy cap/exact_size)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py:39,206,244-286,309-324,414,424,519,632-644 (grep: _BODY_MAX budget)", + "packages/microcosm-fit/src/microcosm/fit/_graph_legacy_qrf.py:20-40 (encode_json: no cap)", + "packages/microcosm-graph/src/microcosm/graph/canonical.py:43-58 (canonical_json: no cap)", + "experiments/native-byte-transports/census-brief.md (full)", + "experiments/native-byte-transports/byte_census_measure.py (full)", + "experiments/native-byte-transports/byte-census-measurements.json (full)", + "experiments/native-row-ceilings/census.json (the 21 bounds and 6 verdicts in this family)", + "_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json (origins.persons.columns and one row per arm, for row shapes)" + ], + "rows": [ + { + "constant": "current_survey_household_roles.py:86 MAX_PERSONS", + "value": "64 * 1024**2 // 32 = 2,097,152", + "base_value": "2,097,152", + "head_value": "2,097,152", + "enforced_at": [ + "current_survey_household_roles.py:663-669 (_origins: `and 0 < len(original) <= MAX_PERSONS,` -> \"ORIGIN_ROSTER\"); ValueError(\"CURRENT_SURVEY_HOUSEHOLD_ROLES_ORIGIN_ROSTER\") via require at :239-241" + ], + "refusal_codes": [ + "ORIGIN_ROSTER" + ], + "encodes": "Row count of the origins roster rebuilt from document[\"origins\"][\"persons\"][\"rows\"] (:651-652): one row per stacked original person (ACS + ASEC channels, pre-clone), compared row-for-row to preparation_frame.person (:672-683).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "356,501 rows (a row count, not bytes)", + "bytes_at_full_source": "3,565,013 rows (a row count, not bytes)", + "measurement_basis": "Counts from census-brief \u00a73 (stacked persons); no encoding is involved at this site.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "reachable_from": "none", + "reach_chain": "graph_survey_completion_host(household_roles=True) -> _roles() (graph_survey_completion_host.py:37-40) -> graph_current_survey_household_roles.register_household_role_kernels / qualify_current_survey_household_roles (:1104) -> _origins (:1127). The 45-node pilot passed household_roles=False; only the 49/51-node variants reach it.", + "classification": "not-a-transport", + "notes": "A row cap wearing a byte-derived expression (64 MiB / 32 B per row); nothing serialises it. It is the first refusal in qualify: `origins = _origins(state.frame, document)` at :1127 runs before `_acs_roster` (:1136) and before `projection = _projection_bytes(table)` (:1151), so at full source it pre-empts every byte bound in this family. First binds at fraction 2,097,152 / 3,565,013 = 0.588. Unchanged on this branch (git diff origin/native-row-ceilings is empty for the module)." + }, + { + "constant": "current_survey_household_roles.py:88 MAX_HOUSEHOLD_MEMBERS", + "value": "20 (literal)", + "base_value": "20", + "head_value": "20", + "enforced_at": [ + "current_survey_household_roles.py:715-720 (`and 1 <= int(line) <= MAX_HOUSEHOLD_MEMBERS` -> \"ACS_LINE_KEY\")", + "current_survey_household_roles.py:749-753 (\"ASEC_LINE_KEY\")", + "current_survey_household_roles.py:792-796 (`and 1 <= int(row[\"SPORDER\"]) <= MAX_HOUSEHOLD_MEMBERS` -> \"ACS_SOURCE_LINE\")" + ], + "refusal_codes": [ + "ACS_LINE_KEY", + "ASEC_LINE_KEY", + "ACS_SOURCE_LINE" + ], + "encodes": "A single printed within-household line number (ACS SPORDER, ASEC A_LINENO), one scalar per person; not a byte count.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "20 (scalar domain; independent of the fraction)", + "bytes_at_full_source": "20 (same)", + "measurement_basis": "Read from the code; each site pairs it with a `[0-9]{1,2}` regex on one token.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Same chain as MAX_PERSONS (49/51-node variants only): qualify -> _acs_keys (:698) / _asec_keys (:728) / _acs_roster -> _scan_acs_relationships (:760).", + "classification": "must-not-move", + "notes": "Protects a token/scalar domain (the published line-number range); current_survey_health_source._key hardcodes the same 20 at :53. Not a transport of any kind." + }, + { + "constant": "current_survey_household_roles.py:89 MAX_ROSTER_ROWS", + "value": "MAX_ROSTER_ROWS = MAX_PERSONS = 2,097,152", + "base_value": "2,097,152", + "head_value": "2,097,152", + "enforced_at": [ + "current_survey_household_roles.py:799 (_scan_acs_relationships: `require(len(selected) < MAX_ROSTER_ROWS, \"ACS_ROSTER_BOUND\")`, checked before each insertion into the `roster` dict)" + ], + "refusal_codes": [ + "ACS_ROSTER_BOUND" + ], + "encodes": "Row count of the retained ACS person rows: every member of every selected ACS household read from the pinned person archive (selection is by household, :760-766). At full source every ACS household is selected, so this is the ACS person roster.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "about 342,288 rows (ACS persons at 1/10; a row count)", + "bytes_at_full_source": "3,422,888 rows (all ACS persons; a row count)", + "measurement_basis": "Counts from census-brief \u00a73 (ACS persons); the dict holds one (SERIALNO, SPORDER) -> RELSHIPP entry per retained row.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: qualify (:1104) -> _acs_roster (:1136, :1054) -> _scan_acs_relationships (:1084).", + "classification": "not-a-transport", + "notes": "A row cap aliasing MAX_PERSONS. Exceeded at full source (3,422,888 > 2,097,152) but never reached there, because ORIGIN_ROSTER (:667) refuses first at :1127. The scan also carries a per-file row cap from the catalogue, `maximum=document[\"counts\"][\"people\"] - count` (:1088) -> \"ACS_SOURCE_ROW_SHAPE\" at :783, which is an assertion about the real PUMS file's row count." + }, + { + "constant": "current_survey_household_roles.py:90 MAX_RECEIPT_BYTES", + "value": "64 * 1024 = 65,536", + "base_value": "65,536", + "head_value": "65,536", + "enforced_at": [ + "current_survey_household_roles.py:1152-1179 (`receipt = source._encode({...}, maximum=MAX_RECEIPT_BYTES)`), which raises inside survey_population_preparation._encode at survey_population_preparation.py:116-121 (`_require(len(result) + len(encoded) <= maximum, \"PAYLOAD_LIMIT\")`, SurveyPopulationPreparationError, a ValueError subclass at :78)" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT" + ], + "encodes": "The household-roles receipt: a fixed-shape JSON object of aggregates, digests, the canonical-requirement block and `acs_relationship_diagnostics` (14 scalar counters plus a 4-entry state_counts map from asec_demographic_source.py:1064-1084). No per-row data.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "3,809 (measured; the shape does not move with the fraction, only integer widths)", + "bytes_at_full_source": "3,809 (measured with every counter set to full-source magnitudes)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> roles_receipt: the document built with the code's keys at :1153-1177 and encoded through survey_population_preparation._encode(maximum=MAX_RECEIPT_BYTES); headroom 17.2x.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: qualify_current_survey_household_roles (:1104) -> :1152.", + "classification": "not-a-transport", + "notes": "_encode appends chunk by chunk with the bound checked before each append (streams), but the document is fixed-shape by contract (PUBLIC_RECEIPT_KEYS at :205-236 admits only counts, digests and flags), so no fraction can move it." + }, + { + "constant": "current_survey_household_roles.py:609 ", + "value": "len(token) <= 64 -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "current_survey_household_roles.py:609 (acs_relationship_token: `require(type(token) is str and len(token) <= 64, \"ACS_TOKEN_BOUND\")`)" + ], + "refusal_codes": [ + "ACS_TOKEN_BOUND" + ], + "encodes": "One RELSHIPP token from the roster dict, checked per person before the `[0-9]{1,2}` parse.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "2 characters per real token (RELSHIPP is a two-digit code); cap 64", + "bytes_at_full_source": "2 characters per token; cap 64", + "measurement_basis": "Read from the code (:610 `re.fullmatch(r\"[0-9]{1,2}\", token, re.ASCII)`).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: qualify -> _acs_states (:1140, :805) -> acs_relationship_token (:826).", + "classification": "must-not-move", + "notes": "A token width; the same 64 is the coverage family's MAX_TOKEN_BYTES scale for ACS cells." + }, + { + "constant": "current_survey_household_roles.py:786 ", + "value": "len(row[column]) <= 64 for column in ACS_ROSTER_COLUMNS -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "current_survey_household_roles.py:785-788 (_scan_acs_relationships: `require(all(len(row[column]) <= 64 for column in ACS_ROSTER_COLUMNS), \"ACS_SOURCE_TOKEN_BOUND\")`)" + ], + "refusal_codes": [ + "ACS_SOURCE_TOKEN_BOUND" + ], + "encodes": "The three cells (SERIALNO, SPORDER, RELSHIPP) of one ACS person record as the pinned member is streamed.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13 characters (SERIALNO `2024HU0001234`) is the widest real cell; cap 64", + "bytes_at_full_source": "13 characters; cap 64", + "measurement_basis": "Read from the code (:712 `2024(?:HU|GQ)[0-9]{7}`).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: qualify -> _acs_roster (:1136) -> _scan_acs_relationships (:1084).", + "classification": "must-not-move", + "notes": "The record itself is produced by acs_person_coverage_authentication._records / _decode_record (:769-770), whose own line/token caps (that module's MAX_RECORD_BYTES 400,000, MAX_TOKEN_BYTES 65,536, MAX_CSV_HEADER_BYTES 65,536 at :48-50) apply first; those are the coverage family's rows." + }, + { + "constant": "current_survey_household_roles.py:937 ", + "value": "none (consumer-side re-encoding: `value.projection == _projection_bytes(value.rows)`; the encoder `_projection_bytes` at :634-635 has no cap)", + "base_value": "none", + "head_value": "none", + "enforced_at": [ + "current_survey_household_roles.py:933-940 (household_role_projection_seal: `value.projection == _projection_bytes(value.rows) and json.loads(value.receipt)[\"projection_sha256\"] == _sha(value.projection)` -> \"PROJECTION_BINDING\"); called at :1183, :1220 (FINAL_PROJECTION) and from graph_current_survey_household_roles.py:202, :418, :478, :497", + "graph_current_survey_household_roles.py:266-281 (_check_artifacts: `value.payload == expected[edge.producer, edge.artifact]` -> \"ARTIFACT_PAYLOAD\", against the retained qualified.projection bytes from _expected_artifacts at :261-262; not a re-encoding but a whole-bytes equality)" + ], + "refusal_codes": [ + "PROJECTION_BINDING", + "ARTIFACT_PAYLOAD" + ], + "encodes": "The roles projection: `table.reset_index().to_json(orient=\"table\", index=False).encode()` (:635) over the 13 declared COLUMN_TOKENS plus person_id, one row per stacked original person.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "231,725,650 (650.0 B/person x 356,501)", + "bytes_at_full_source": "2,317,258,450 (650.0 B/person x 3,565,013)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> roles_projection: rows built through `_observation_table` with the real ORIGIN/UNIVERSE literals and person_id/native ids at full-source widths, encoded by `_projection_bytes`, 3 vs 1,203 rows (2,969 vs 782,969 bytes). Not the lane JSON's 324 B/person: that table carried only 7 of the 13 columns and placeholder strings (see open_questions).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: qualify (:1151, :1183, :1220) and the roles graph kernel (_HouseholdRoleKernel.__init__ :418, run :478/:497).", + "classification": "segment", + "notes": "The producer has no cap, so binds is false by definition; the whole 2.3 GB string is re-encoded on every seal call (at least five times per run at full source). The bytes meet the consumer cap MAX_ARTIFACT_BYTES (next row) from 103,244 persons upward. `_table_stamp` at :638-642 also materialises `table[name].to_json()` per column for a digest with no cap." + }, + { + "constant": "current_survey_household_roles.py:1384 ", + "value": "0 < len(receiving_version) <= 256 -> 256", + "base_value": "256", + "head_value": "256", + "enforced_at": [ + "current_survey_household_roles.py:1382-1387 (household_role_binding_receipt: `and 0 < len(receiving_version) <= 256` -> \"RECEIPT_INPUT\")" + ], + "refusal_codes": [ + "RECEIPT_INPUT" + ], + "encodes": "The receiving population version string (a node id), one scalar.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "a node id of a few dozen characters; cap 256", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: graph _result (graph_current_survey_household_roles.py:343-348) -> household_role_binding_receipt.", + "classification": "must-not-move", + "notes": "A string-length width on an identifier." + }, + { + "constant": "graph_current_survey_household_roles.py:70 MAX_ARTIFACT_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "base_value": "67,108,864", + "head_value": "67,108,864", + "enforced_at": [ + "graph_current_survey_household_roles.py:266-276 (_check_artifacts: `and len(value.payload) <= MAX_ARTIFACT_BYTES,` -> \"ARTIFACT_TYPE\", over every artifact input of the node: the ordering edge and host edges of the source node (:228) and the ordering edge plus the projection edge of the bind node (:250)); require is roles.require (:57), ValueError(\"CURRENT_SURVEY_HOUSEHOLD_ROLES_ARTIFACT_TYPE\")" + ], + "refusal_codes": [ + "ARTIFACT_TYPE" + ], + "encodes": "Whole artifact payloads as bytes objects; on the bind node this is the roles projection (previous row: roster of stacked persons, 13 columns + person_id, pandas orient=\"table\" JSON), plus the host-edge payloads.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "231,725,650 (3.45x the cap)", + "bytes_at_full_source": "2,317,258,450 (34.5x the cap)", + "measurement_basis": "Same measurement as the projection row (650.0 B/person through roles._projection_bytes); the cap admits 103,244 persons (67,108,864 // 650).", + "binds_at_tenth": true, + "binds_at_full_source": true, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: graph_survey_completion_host(household_roles=True) -> _roles() -> register_household_role_kernels (:538) -> _HouseholdRoleKernel.run (:471) -> _check_artifacts (:485); also expected_survey_household_roles_population (:361).", + "classification": "consumer", + "notes": "A consumer-side cap on bytes the producer issues whole; the producer `_projection_bytes` carries no cap, so this row takes the producer's total. At full source the row cap MAX_PERSONS (:667) refuses before any projection exists, so this is the binding refusal only between 103,244 persons and 2,097,152 persons (fractions 0.029 to 0.588). The bind node's own output `canonical_json(receipt)` at :351 is fixed-shape. `_seal_result` at :452-469 materialises `value.to_json()` of the bound is_household_head column (clone persons, 7,130,026 at full source) for a digest with no cap." + }, + { + "constant": "graph_current_survey_household_roles.py:131 ", + "value": "len(value) == 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "graph_current_survey_household_roles.py:130-131 (_digest) used at :146-152 (_check_host_binding: `all(_digest(value) for value in pin.values())` -> \"HOST_PIN_DIGESTS\")" + ], + "refusal_codes": [ + "HOST_PIN_DIGESTS" + ], + "encodes": "Three sha256 hex strings per host pin (producer_key, artifact_key, payload_sha256).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: current_survey_household_roles_nodes (:192) -> _check_host_binding (:210).", + "classification": "must-not-move", + "notes": "A fixed-width field (sha256 hex)." + }, + { + "constant": "graph_current_survey_household_roles.py:205 ", + "value": "0 < len(receiving_version) <= 256 -> 256", + "base_value": "256", + "head_value": "256", + "enforced_at": [ + "graph_current_survey_household_roles.py:203-209 (current_survey_household_roles_nodes: `and 0 < len(receiving_version) <= 256` -> \"FRAGMENT_INPUT\")" + ], + "refusal_codes": [ + "FRAGMENT_INPUT" + ], + "encodes": "The receiving population version string, one scalar.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "a node id; cap 256", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: register_household_role_kernels -> current_survey_household_roles_kernels -> current_survey_household_roles_nodes.", + "classification": "must-not-move", + "notes": "Identifier width; independent literal from current_survey_household_roles.py:1384." + }, + { + "constant": "graph_current_survey_person_status.py:59 MAX_ARTIFACT_BYTES", + "value": "2**31 = 2,147,483,648", + "base_value": "2,147,483,648", + "head_value": "2,147,483,648", + "enforced_at": [ + "graph_current_survey_person_status.py:387 (projection_payload: `require(len(payload) <= MAX_ARTIFACT_BYTES, \"PROJECTION_BYTES\")`, producer side)", + "graph_current_survey_person_status.py:562-568 (_check_artifacts: `and len(value.payload) <= MAX_ARTIFACT_BYTES,` -> \"ARTIFACT_TYPE\", consumer side, every artifact input of all four nodes)", + "graph_current_survey_person_status.py:569-573 (consumer-side re-encoding: `value.payload == expected[edge.producer, edge.artifact]` -> \"ARTIFACT_PAYLOAD\", where expected = projection_payload(qualified) at :548, re-run per kernel run and verify)" + ], + "refusal_codes": [ + "PROJECTION_BYTES", + "ARTIFACT_TYPE", + "ARTIFACT_PAYLOAD" + ], + "encodes": "The person-status source projection: canonical_json of {protocol, source_receipt_sha256, origins: _table_document(origins), raw: _table_document(raw)} (:377-388), where origins carries the 10 preparation columns (person_id, source, source_year, survey_year, raw_native_household_id, raw_native_person_id, native_line_numeric_original, selected_receiving_person_id, selected_receiving_household_id, household_id) and raw the 45 RAW_COLUMNS (24 ASEC + 21 ACS literals, the opposite arm null), integers as {\"integer_literal\": \"...\"} (:355-363); one row per stacked original person.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "155,036,654 (433.65 B per ACS person x 342,288 + 464.64 B per ASEC person x 14,212)", + "bytes_at_full_source": "1,550,372,341 (433.65 x 3,422,888 + 464.64 x 142,125); 0.722 of the cap, headroom 1.39x; the cap admits 4,938,044 persons at the weighted 434.89 B/person", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> person_status_projection: origins rows copied from the recovered preparation.json's real ACS and ASEC rows with ids reset to full-source widths (7-digit person ids, 13-char SERIALNO, 22-digit PERIDNUM), raw rows at real literal widths, typed through current_survey_person_status_source._table and encoded through status_graph._table_document + canonical_json, 3 vs 1,203 rows per arm (ACS 2,750 vs 523,130 bytes; ASEC 2,849 vs 560,423).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "graph_atomic_survey_financial._person_status_module (:132-136) is imported only inside _PersonStatusBoundary (:139-160), constructed only for the explicitly enabled person-status graph (person_status=True; the 19-node run and the 45-node pilot pass person_status=False). Would reach: _PersonStatusBoundary -> graph._result (:613) -> status_tables/projection_payload (:387) and _StatusKernel.run (:769) -> _check_artifacts (:782).", + "classification": "stays", + "notes": "Headroom 1.39x at full source, so it does not meet its bound; but the 1.55 GB payload is materialised whole and re-encoded by `projection_payload(qualified)` on every _check_artifacts (:556-558) and every _result (:614). Two larger unbounded materialisations sit beside it: `_columns_payload` (:391-401) digests `canonical_json(_table_document(table))` for the literal table (816,423,843 B at full source) and for the 121-column recoded table (4,853,789,741 B at full source; 485,377,090 at 1/10) with no cap at all (see open_questions). The row-ceilings lane's 447 B/person estimate is replaced by the measured 434.89 weighted." + }, + { + "constant": "graph_current_survey_person_status.py:432 ", + "value": "len(v) == 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "graph_current_survey_person_status.py:427-436 (_check_pins: `type(v) is str and len(v) == 64 and not set(v) - set(\"0123456789abcdef\")` -> \"HOST_PIN_DIGESTS\")" + ], + "refusal_codes": [ + "HOST_PIN_DIGESTS" + ], + "encodes": "Three sha256 hex strings per host pin.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "person_status=True graphs only: current_survey_person_status_nodes (:451) -> _check_pins (:462).", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "current_survey_person_status.py:137 ", + "value": "len(token) <= 64 -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "current_survey_person_status.py:137 (literal_code: `require(type(token) is str and len(token) <= 64, \"LITERAL_TYPE_OR_BOUND\")`)" + ], + "refusal_codes": [ + "LITERAL_TYPE_OR_BOUND" + ], + "encodes": "One published status/flag/age literal, checked before the width-1/2 numeric regex (:141).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "1-2 characters per real token; cap 64", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code; `width=` defaults 1 or 2 at :131, :152, :175-177, :206-207, :240, :243, :290, :295.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "person_status=True graphs only: current_survey_person_status_source._combine (:167, :177) and status_graph.status_tables (:327) -> recode_person_status -> literal_code.", + "classification": "must-not-move", + "notes": "Token width on a code literal; the 1/2-digit `width` arguments are the code grammar, not byte bounds." + }, + { + "constant": "current_survey_person_status.py:236 ", + "value": "len(row[c]) <= 64 for c in columns -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "current_survey_person_status.py:235-238 (recode_person_status: `require(all(type(row[c]) is str and len(row[c]) <= 64 for c in columns), \"LITERAL_TYPE_OR_BOUND\")`)" + ], + "refusal_codes": [ + "LITERAL_TYPE_OR_BOUND" + ], + "encodes": "Every cell of one person's arm columns (24 ASEC or 21 ACS literals) before recoding.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "22 characters (PERIDNUM) is the widest real cell; cap 64", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code and current_survey_health_source._key (:44 `[0-9]{22}`).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "person_status=True graphs only: _combine (:177) / status_tables (:327) -> recode_person_status.", + "classification": "must-not-move", + "notes": "Token width." + }, + { + "constant": "current_survey_person_status_source.py:96 ", + "value": "len(row[c]) <= 64 for c in columns -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "current_survey_person_status_source.py:96 (_scan: `require(all(len(row[c]) <= 64 for c in columns), \"SOURCE_TOKEN_BOUND\")`)" + ], + "refusal_codes": [ + "SOURCE_TOKEN_BOUND" + ], + "encodes": "The arm's cells of one CSV record while the pinned ASEC member (:389-396) or the ACS person archive members (:413-427) are streamed.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "22 characters (PERIDNUM) is the widest real cell; cap 64", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "person_status=True graphs only: qualify_current_survey_person_status (:302) -> _scan (:390, :418).", + "classification": "must-not-move", + "notes": "The same helper carries the per-file row cap `count <= maximum` -> \"SOURCE_ROW_SHAPE\" at :94, where maximum is the pinned member's row count (:395) or the ACS catalogue's people count (:423): assertions about the real files. Records come from acs_person_coverage_authentication._records/_decode_record (:80-81), whose caps are the coverage family's rows." + }, + { + "constant": "survey_population_preparation.py:50 MAX_PAYLOAD_BYTES (definition owned by the survey-population family; this row records the enforcement sites inside this family)", + "value": "64 * 1024**2 = 67,108,864", + "base_value": "67,108,864", + "head_value": "67,108,864", + "enforced_at": [ + "current_survey_person_status_source.py:436-482 (`receipt = source._encode({...})` with the default maximum=MAX_PAYLOAD_BYTES -> \"PAYLOAD_LIMIT\", SurveyPopulationPreparationError at survey_population_preparation.py:116-121)", + "current_survey_amounts.py:362-370 (`projection = predictors.host.survey_graph._bounded_json({...}, predictors.source.MAX_PAYLOAD_BYTES)` -> \"TRANSPORT_LIMIT\", SurveyPopulationGraphError, graph_survey_population.py:270-290)" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT", + "TRANSPORT_LIMIT" + ], + "encodes": "Two fixed-shape documents: the person-status receipt (digests, periods, column rosters, 187 observation column names; :437-481) and the amounts projection (evidence dict plus three digests; :326-370). Neither carries per-row data.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "person-status receipt 12,379 B (measured); amounts projection not measured (fixed shape: evidence keys plus sha256 strings)", + "bytes_at_full_source": "12,379 B (same document; counters at full-source magnitudes)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> person_status_receipt (document built with the keys at :437-481 and encoded through survey_population_preparation._encode).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Neither site is on a runner path: qualify_current_survey_person_status runs only for person_status=True graphs; current_survey_amounts is listed off every path in census-brief \u00a72.", + "classification": "not-a-transport", + "notes": "Both documents are fixed-shape at these sites (5,400x headroom for the receipt). The definition row belongs to the survey-population family; merge these two sites into it. The amounts projection's inputs (origins, native, reports, donor_columns via to_json(orient=\"table\") at :139, :345, :366-367) are roster-sized digest inputs with no cap (open_questions)." + }, + { + "constant": "puf_growth.py:193 MAX_EXACT_AMOUNT", + "value": "2**53 = 9,007,199,254,740,992", + "base_value": "2**53", + "head_value": "2**53", + "enforced_at": [ + "puf_growth.py:1243-1250 (_money_values: `bool((np.abs(series.to_numpy(dtype=object)) <= MAX_EXACT_AMOUNT).all())` -> \"MONEY_MAGNITUDE\"); PufGrowthRefusalError (ValueError subclass, :207-213)" + ], + "refusal_codes": [ + "MONEY_MAGNITUDE" + ], + "encodes": "The magnitude of one integer money value before the float64 working conversion (float64's exact-integer width).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "PUF amounts are at most 12 digits (< 10**12); width 2**53", + "bytes_at_full_source": "same; invariant to the survey fraction", + "measurement_basis": "Read from the code; puf_monetary_source._MAX_AMOUNT_DIGITS = 12 bounds the delivered tokens.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports puf_growth (only puf_price_baseline, puf_monetary_agi_projection, graph_puf_detail_transfer, puf_detail_transfer, puf_growth_graph, graph_full_puf_enrichment do). Would reach: apply_puf_growth (:1305) -> _money_values (:1348, :1373).", + "classification": "must-not-move", + "notes": "An encoder width (float64 mantissa), not a resource ceiling." + }, + { + "constant": "puf_growth.py:197 MAX_ROWS", + "value": "2_000_000 (literal)", + "base_value": "2,000,000", + "head_value": "2,000,000", + "enforced_at": [ + "puf_growth.py:1207 (_checked_source_table: `_require(0 < rows <= MAX_ROWS, \"SOURCE_TABLE_ROWS\")`)", + "puf_growth.py:1441 (apply_puf_design_weight_growth: `_require(values.ndim == 1 and 0 < values.size <= MAX_ROWS, \"WEIGHT_SHAPE\")`)" + ], + "refusal_codes": [ + "SOURCE_TABLE_ROWS", + "WEIGHT_SHAPE" + ], + "encodes": "Row count of the PUF money view / design-weight vector at IRS return grain (207,696 delivered returns; 207,692 after the four disclosure aggregates are removed by puf_price_baseline.adapt_projection_to_money_view).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "207,696 rows (a row count; the PUF file does not scale with the survey fraction)", + "bytes_at_full_source": "207,696 rows (same)", + "measurement_basis": "puf_2015_raw_source_definition.json sources.main.data_records = 207696; comment at :195-196 calls it an allocation guard.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path (see MAX_EXACT_AMOUNT); would reach via apply_puf_growth / apply_puf_design_weight_growth.", + "classification": "not-a-transport", + "notes": "A row count with 9.6x headroom over the fixed PUF delivery; the row-count lane's PUF-grain rule applies." + }, + { + "constant": "puf_growth.py:200 MAX_DOCUMENT_BYTES", + "value": "512 * 1024 = 524,288", + "base_value": "524,288", + "head_value": "524,288", + "enforced_at": [ + "puf_growth.py:233-236 (_parse: `type(document) is bytes and 0 < len(document) <= MAX_DOCUMENT_BYTES` -> \"DOCUMENT_SIZE_OR_TYPE\"); reached from parse_growth_recipe (:555), GrowthFactorTable.from_bytes (:738) and GrownPufTable.receipt (:1195)", + "puf_growth.py:708-712 (GrowthFactorTable.__post_init__: `and 0 < len(self.document) <= MAX_DOCUMENT_BYTES` -> \"DOCUMENT_SIZE_OR_TYPE\")" + ], + "refusal_codes": [ + "DOCUMENT_SIZE_OR_TYPE" + ], + "encodes": "Fixed-shape JSON documents: a growth factor table, a recipe identity, and the apply_puf_growth receipt. No per-return data.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "factor document 1,008 B; recipe identity 3,204 B; growth receipt 898 B (measured with the CPI-U price baseline over the 13 monetary fields)", + "bytes_at_full_source": "same (row-independent; the receipt carries `rows` as one integer)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> growth_documents: puf_price_baseline.price_baseline_factor_document / compile_price_baseline on invented resource bytes carrying the two published CPI-U records, then apply_puf_growth on an 8-row table.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path; would reach via GrowthFactorTable.from_bytes / parse_growth_recipe / GrownPufTable.receipt.", + "classification": "not-a-transport", + "notes": "520x headroom; a parser allocation guard on fixed documents." + }, + { + "constant": "puf_growth.py:261 ", + "value": "len(value) == 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_growth.py:258-263 (_digest) used at :847-849 (`_require(_digest(factors.provenance[\"resource_sha256\"]), \"DEVELOPMENTAL_RESOURCE_DIGEST\")`)" + ], + "refusal_codes": [ + "DEVELOPMENTAL_RESOURCE_DIGEST" + ], + "encodes": "One sha256 hex string in a factor table's provenance.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path; CompiledPufGrowth.__post_init__ (:941-942) -> _check_developmental_provenance.", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "puf_growth.py:268 ", + "value": "0 < len(value) <= 128 -> 128", + "base_value": "128", + "head_value": "128", + "enforced_at": [ + "puf_growth.py:266-272 (_identifier: `type(value) is str and 0 < len(value) <= 128 and value.strip() == value`), raising the caller's reason: RULE_FIELD_NAME (:376), RULE_FACTOR_SERIES (:401), RECIPE_ID (:437), SERIES_NAME (:623), TABLE_ID (:713), RESOLVED_FIELD (:882), RESOLVED_SERIES (:889), ROSTER_FIELD (:1500)" + ], + "refusal_codes": [ + "RULE_FIELD_NAME", + "RULE_FACTOR_SERIES", + "RECIPE_ID", + "SERIES_NAME", + "TABLE_ID", + "RESOLVED_FIELD", + "RESOLVED_SERIES", + "ROSTER_FIELD" + ], + "encodes": "Identifier strings (field names, series names, recipe/table ids), one scalar each.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "the longest packaged identifier is PRICE_BASELINE_SERIES_NAME (52 chars); cap 128", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code and puf_price_baseline.py:140.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Identifier width." + }, + { + "constant": "puf_growth.py:666 ", + "value": "depth <= 64 (JSON nesting depth) -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_growth.py:664-666 (_freeze_provenance: `_require(depth <= 64, \"TABLE_PROVENANCE_DEPTH\")`)" + ], + "refusal_codes": [ + "TABLE_PROVENANCE_DEPTH" + ], + "encodes": "Nesting depth of a factor table's provenance tree (a JSON token cap, not bytes).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "depth 1 in the packaged price-baseline provenance; cap 64", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code and puf_price_baseline.py:327-344.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path; GrowthFactorTable.__post_init__ (:730).", + "classification": "must-not-move", + "notes": "A token/structure cap on a fixed document." + }, + { + "constant": "puf_monetary_source.py:180 BODY_MAX_BYTES", + "value": "64 * 1024 * 1024 = 67,108,864", + "base_value": "67,108,864", + "head_value": "67,108,864", + "enforced_at": [ + "puf_monetary_source.py:1386-1389 (encode_monetary_projection: `total, sizes = payload_bound(rows, projection)` / `if total > BODY_MAX_BYTES: raise _refuse(\"BODY_OVER_BOUND\", total)`)", + "puf_monetary_source.py:1566-1568 (_envelope_columns, decode: `total += size` / `if total > BODY_MAX_BYTES: raise _refuse(\"ENVELOPE_BODY_OVER_BOUND\", total)`)", + "puf_monetary_agi_projection.py:292-294 (encode_agi_projection: same payload_bound check -> \"BODY_OVER_BOUND\"; decode goes through the imported _envelope_columns)" + ], + "refusal_codes": [ + "BODY_OVER_BOUND", + "ENVELOPE_BODY_OVER_BOUND" + ], + "encodes": "The column bodies of the monetary projection envelope over every delivered PUF return: 3 structural typed columns (RECID i8, disclosure_aggregate u1, demographic_status i1) plus, per projected column, an i8 typed body and a 13-byte fixed-width lexical body: 262 B/row for 12 columns, 283 B/row for the 13-column AGI edition.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "54,416,352 (12 columns x 207,696 rows) and 58,777,968 (13 columns); invariant to the survey fraction", + "bytes_at_full_source": "54,416,352 / 58,777,968 (same)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers: monetary.payload_bound(207696, packaged_projection()) and payload_bound(207696, packaged_agi_projection()); headroom 1.233x (cap admits 256,140 rows) and 1.142x (237,133 rows).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports puf_monetary_source; would reach via USPufMonetarySourceKernel.run (:1811) -> encode_monetary_projection, or decode_monetary_projection (:1616) -> _envelope_columns.", + "classification": "must-not-move", + "notes": "Sized by the upstream PUF delivery's row count (207,696) and the packaged column roster, never by the survey selection; the bodies exist whole (`bodies` list joined at :1457-1459). Only another projected column or a larger PUF vintage can move it." + }, + { + "constant": "puf_monetary_source.py:184 HEADER_MAX_BYTES", + "value": "64 * 1024 = 65,536", + "base_value": "65,536", + "head_value": "65,536", + "enforced_at": [ + "puf_monetary_source.py:1454-1456 (`encoded = canonical_json(header)` / `if len(encoded) > HEADER_MAX_BYTES: raise _refuse(\"HEADER_OVER_BOUND\", len(encoded))`)", + "puf_monetary_source.py:1642-1644 (decode: `length = int.from_bytes(payload[start : start + 4], \"big\")` / `if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: raise _refuse(\"ENVELOPE_HEADER_LENGTH\")`)", + "puf_monetary_agi_projection.py:356-358 (\"HEADER_OVER_BOUND\") and :413-415 (\"ENVELOPE_HEADER_LENGTH\")" + ], + "refusal_codes": [ + "HEADER_OVER_BOUND", + "ENVELOPE_HEADER_LENGTH" + ], + "encodes": "The canonical JSON envelope header: 27 column entries (12 columns) or 29 (13), source pins, and the facts block with column_facts for three row classes and column_metadata per column. No per-row data.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "25,916 B (12 columns) / 27,919 B (13 columns), measured with 207,696-row facts; headroom 2.53x / 2.35x", + "bytes_at_full_source": "same (row-independent)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers: header dict built exactly as :1418-1453 with facts from monetary._facts_from_arrays over invented 207,696-row arrays (sums of 10-digit amounts) and canonical_json.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path (see BODY_MAX_BYTES).", + "classification": "not-a-transport", + "notes": "Fixed-shape; grows only with more projected columns (about 2 KB per column from the 12->13 measurement). The 4-byte big-endian length prefix at :1458 and puf_monetary_agi_projection.py:360 is a wire width (2**32-1) that this cap keeps far from binding. The decoder's `canonical_json(recomputed) != canonical_json(dict(facts))` at :1722-1724 (and agi :484-486) is a consumer-side re-encoding of the fixed facts block." + }, + { + "constant": "puf_monetary_source.py:187 LEXICAL_WIDTH_MAX", + "value": "64 (literal)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_monetary_source.py:510-512 (_projected_column: `if width > LEXICAL_WIDTH_MAX: raise _refuse(\"PROJECTION_LEXICAL_WIDTH\", field)`)", + "puf_monetary_source.py:1549-1551 (_envelope_columns: `if width > LEXICAL_WIDTH_MAX or width != widths[field]: raise _refuse(\"ENVELOPE_COLUMN_WIDTH\", name)`; also used by puf_monetary_agi_projection.py:448)", + "per-token enforcement of the declared width (13) that this cap bounds: :872-873 \"AMOUNT_TOKEN_OVER_WIDTH\", :913-914 \"AGGREGATE_TOKEN_OVER_WIDTH\", :1329-1330 \"LEXICAL_OVER_WIDTH\", :840-841 \"KEY_TOKEN_OVER_WIDTH\" (RECID width 64 from the definition)" + ], + "refusal_codes": [ + "PROJECTION_LEXICAL_WIDTH", + "ENVELOPE_COLUMN_WIDTH", + "AMOUNT_TOKEN_OVER_WIDTH", + "AGGREGATE_TOKEN_OVER_WIDTH", + "LEXICAL_OVER_WIDTH", + "KEY_TOKEN_OVER_WIDTH" + ], + "encodes": "The per-cell allocation of a fixed-width NUL-padded ASCII lexical column (_LEXICAL_KIND :204), read back by `body[position * width : (position + 1) * width]` (:1343) and sized `rows * width` (:1365, :1552).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "13 declared for all 12 (and 13) packaged columns; cap 64", + "bytes_at_full_source": "13; invariant", + "measurement_basis": "puf_2015_monetary_source_projection.json and puf_2015_monetary_agi_source_projection.json (lexical_width 13 on every column, per the row-ceilings verdict read of both files).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "A wire-format width: changing it changes the stored artifact layout. The lower bound `width < digits + 1` at :515 makes 13 the exact declared width (12 digits + sign)." + }, + { + "constant": "puf_monetary_source.py:338 _MAX_AMOUNT_DIGITS", + "value": "12 (literal)", + "base_value": "12", + "head_value": "12", + "enforced_at": [ + "puf_monetary_source.py:514-516 (`if digits > _MAX_AMOUNT_DIGITS or width < digits + 1: raise _refuse(\"PROJECTION_WIDTH_BELOW_GRAMMAR\", field)`)", + "puf_monetary_source.py:879-880 (_amount: `if len(digits) > column.max_digits: raise _refuse(\"AMOUNT_TOKEN_OVER_DIGITS\", label)`)", + "puf_monetary_source.py:927-928 (_aggregate_token: \"AGGREGATE_TOKEN_OVER_DIGITS\")" + ], + "refusal_codes": [ + "PROJECTION_WIDTH_BELOW_GRAMMAR", + "AMOUNT_TOKEN_OVER_DIGITS", + "AGGREGATE_TOKEN_OVER_DIGITS" + ], + "encodes": "Decimal digits per delivered amount token (the publisher's 12-digit amount fields).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "12 declared; invariant", + "bytes_at_full_source": "12; invariant", + "measurement_basis": "Read from the code; the packaged projections declare field_width_digits 12.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "The AGGREGATE_SENTINEL = -(2**63) argument at :239-244 depends on tokens being at most twelve digits; not a resource ceiling." + }, + { + "constant": "puf_monetary_source.py:334 _INT64_MAX / :335 _INT64_MIN", + "value": "2**63 - 1 = 9,223,372,036,854,775,807 / -(2**63)", + "base_value": "2**63-1 / -(2**63)", + "head_value": "2**63-1 / -(2**63)", + "enforced_at": [ + "puf_monetary_source.py:845-846 (_key_token: `if number > _INT64_MAX: raise _refuse(\"KEY_TOKEN_INT64_OVERFLOW\")`)", + "puf_monetary_source.py:884-885 (_amount: `if not _INT64_MIN <= number <= _INT64_MAX: raise _refuse(\"AMOUNT_TOKEN_INT64_OVERFLOW\", label)`)", + "puf_monetary_source.py:1178-1180 (_column_facts: `total = int(sum(...))` / `if not _INT64_MIN <= total <= _INT64_MAX: raise _refuse(\"COLUMN_SUM_OVERFLOW\")`)" + ], + "refusal_codes": [ + "KEY_TOKEN_INT64_OVERFLOW", + "AMOUNT_TOKEN_INT64_OVERFLOW", + "COLUMN_SUM_OVERFLOW" + ], + "encodes": "The int64 storage width of typed bodies (`", + "value": "len(value) != 64 -> 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_monetary_source.py:429-436 (_hex64 -> \"PROJECTION_DIGEST\"), called at :585, :1428-1430, :1485, :1508, :1562, :1669-1672 and from puf_monetary_agi_projection.py:330, :372, :437-440" + ], + "refusal_codes": [ + "PROJECTION_DIGEST" + ], + "encodes": "sha256 hex strings in projection documents and envelope headers.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "puf_2015_raw_source_definition.json:1 csv_profile.record_cap_per_file", + "value": "1000000 (JSON literal) = 1,000,000 records per delivered file", + "base_value": "1,000,000", + "head_value": "1,000,000", + "enforced_at": [ + "puf_raw_source.py:724-738 (_records: `limit = profile[\"record_cap_per_file\"]` / `if count > limit: raise _refuse(\"RECORD_CAP\", pin.name)`)", + "puf_monetary_source.py:989-1004 (_records: same profile value -> \"RECORD_CAP\")" + ], + "refusal_codes": [ + "RECORD_CAP" + ], + "encodes": "Record count of one delivered PUF CSV as it is streamed (main 207,696 data records; demographic 119,675).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "207,696 records (main), 119,675 (demographic); invariant", + "bytes_at_full_source": "same", + "measurement_basis": "puf_2015_raw_source_definition.json sources.*.data_records.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path (USPufRawSourceKernel.run :1544 / decode_puf_monetary_source :1012).", + "classification": "must-not-move", + "notes": "A record cap on the real upstream files, defined in the packaged data document and pinned into the kernel identity through csv_acceptance_profile; the exact counts are separately asserted by _check_declared_expectations (:846-886, \"DECLARED_EXPECTATION\")." + }, + { + "constant": "puf_2015_raw_source_definition.json:1 csv_profile.logical_record_character_cap", + "value": "1048576 (JSON literal) = 1,048,576 characters per record", + "base_value": "1,048,576", + "head_value": "1,048,576", + "enforced_at": [ + "puf_raw_source.py:726-742 (_records: `cap = profile[\"logical_record_character_cap\"]` / `if sum(len(cell) for cell in record) > cap: raise _refuse(\"RECORD_CHARACTER_CAP\", pin.name)`)", + "puf_monetary_source.py:992-1008 (_records -> \"RECORD_CHARACTER_CAP\")" + ], + "refusal_codes": [ + "RECORD_CHARACTER_CAP" + ], + "encodes": "Character count of one delivered CSV record (222 fields for main, 7 for demographic).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "a main record averages about 607 characters (126,034,649 bytes / 207,697 lines); cap 1,048,576", + "bytes_at_full_source": "same; invariant", + "measurement_basis": "Derived from the pinned main bytes and data_records in puf_2015_raw_source_definition.json.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "A per-record width cap on the real upstream file; the paired `len(record) != width` (\"RECORD_WIDTH\", :739-740 / :1005-1006) is the exact 222/7 field count." + }, + { + "constant": "puf_2015_raw_source_definition.json:1 csv_profile.header_field_cap", + "value": "1024 (JSON literal) = 1,024 header fields", + "base_value": "1,024", + "head_value": "1,024", + "enforced_at": [ + "puf_raw_source.py:715-716 (_check_header: `if len(header) > profile[\"header_field_cap\"]: raise _refuse(\"HEADER_WIDTH_LIMIT\", pin.name)`)", + "puf_monetary_source.py:975-976 (_check_header -> \"HEADER_WIDTH_LIMIT\")" + ], + "refusal_codes": [ + "HEADER_WIDTH_LIMIT" + ], + "encodes": "The field count of the delivered CSV header row (222 main, 7 demographic).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "222 fields; cap 1,024", + "bytes_at_full_source": "same", + "measurement_basis": "puf_2015_raw_source_definition.json sources.main.delivered_header_width = 222.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Followed by the exact-header equality (\"HEADER_MISMATCH\", \"HEADER_DIGEST\")." + }, + { + "constant": "puf_2015_raw_source_definition.json:1 csv_profile.field_size_limit", + "value": "131072 (JSON literal) = 131,072 characters per CSV field (the stdlib csv module's process-global limit)", + "base_value": "131,072", + "head_value": "131,072", + "enforced_at": [ + "puf_raw_source.py:625-629 (_check_csv_profile: `if csv.field_size_limit() != declared: raise _refuse(\"CSV_FIELD_SIZE_LIMIT\")`, preceded by \"DEFINITION_CSV_FIELD_SIZE_LIMIT\")", + "puf_monetary_source.py:791-795 (_check_csv_profile: same two refusals)" + ], + "refusal_codes": [ + "DEFINITION_CSV_FIELD_SIZE_LIMIT", + "CSV_FIELD_SIZE_LIMIT" + ], + "encodes": "The width of one CSV cell the stdlib reader will accept; the code pins the process value to the declared one rather than setting it.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "the widest delivered cell is the 64-character RECID allocation; limit 131,072", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code and the definition document.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Part of the producer identity (csv_acceptance_profile, :589-612)." + }, + { + "constant": "puf_raw_source.py:117 PUF_RAW_SOURCE_MAX_BYTES", + "value": "128 * 1024 * 1024 = 134,217,728", + "base_value": "134,217,728", + "head_value": "134,217,728", + "enforced_at": [ + "puf_raw_source.py:318-319 (_pin: `if size > PUF_RAW_SOURCE_MAX_BYTES: raise _refuse(\"DEFINITION_SOURCE_OVER_CAP\", name)`)", + "puf_raw_source.py:502-503 (load_pinned_puf_bytes: `if pin.bytes > PUF_RAW_SOURCE_MAX_BYTES: raise _refuse(\"CODEC_PIN_OVER_CAP\", pin.name)`)", + "puf_raw_source.py:1218-1220 (_check_envelope_sources: `if size > PUF_RAW_SOURCE_MAX_BYTES: raise _refuse(\"ENVELOPE_SOURCE_OVER_CAP\", name)`)" + ], + "refusal_codes": [ + "DEFINITION_SOURCE_OVER_CAP", + "CODEC_PIN_OVER_CAP", + "ENVELOPE_SOURCE_OVER_CAP" + ], + "encodes": "The declared byte size of one delivered PUF file (main 126,034,649; demographic 2,225,050), read whole into memory by the codec (:519-542).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "126,034,649 (main); 2,225,050 (demographic); invariant", + "bytes_at_full_source": "same", + "measurement_basis": "puf_2015_raw_source_definition.json sources.main.bytes / sources.demographic.bytes; headroom 1.065x over the real main delivery.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path; would reach via puf_raw_source_codecs -> load_pinned_puf_bytes.", + "classification": "must-not-move", + "notes": "Chosen from the upstream file's real size (:29-31); exact identity is enforced separately by `before.st_size != pin.bytes` (\"CODEC_SIZE\", :513-514), `total != pin.bytes` (\"CODEC_LENGTH\", :532-533), sha256 and git-blob sha1." + }, + { + "constant": "puf_raw_source.py:521 ", + "value": "limit = pin.bytes + 1 -> 126,034,650 (main) / 2,225,051 (demographic): a read(N+1)-style cap keyed to the pin", + "base_value": "pin.bytes + 1", + "head_value": "pin.bytes + 1", + "enforced_at": [ + "puf_raw_source.py:521-528 (load_pinned_puf_bytes: `limit = pin.bytes + 1` / `chunk = os.read(descriptor, 1024 * 1024)` / `total += len(chunk)` / `if total > limit: raise _refuse(\"CODEC_GREW\", pin.name)`)" + ], + "refusal_codes": [ + "CODEC_GREW" + ], + "encodes": "Bytes read from the delivered PUF file, accumulated in 1 MiB chunks that are retained (`chunks.append(chunk)`, :531) and joined (:542).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "126,034,649 (main); invariant", + "bytes_at_full_source": "same", + "measurement_basis": "Definition document pins.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Protects the upstream file's real size: one byte over the pin refuses. The 1 MiB read size at :523 is a chunk, not a cap." + }, + { + "constant": "puf_raw_source.py:121 BODY_MAX_BYTES", + "value": "64 * 1024 * 1024 = 67,108,864", + "base_value": "67,108,864", + "head_value": "67,108,864", + "enforced_at": [ + "puf_raw_source.py:1086-1089 (encode_return_status: `total, sizes = _payload_bound(rows, definition)` / `if total > BODY_MAX_BYTES: raise _refuse(\"BODY_OVER_BOUND\", total)`)", + "puf_raw_source.py:1282-1284 (_envelope_columns: `total += size` / `if total > BODY_MAX_BYTES: raise _refuse(\"ENVELOPE_BODY_OVER_BOUND\", total)`)" + ], + "refusal_codes": [ + "BODY_OVER_BOUND", + "ENVELOPE_BODY_OVER_BOUND" + ], + "encodes": "The return-status artifact's column bodies over 207,696 returns: 14 typed columns (29 B/row) plus 12 fixed-width lexical columns (RECID 64, S006 32, FLPDYR 8, nine x 4 = 140 B/row) = 169 B/row.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "35,100,624 (207,696 x 169); invariant", + "bytes_at_full_source": "same; headroom 1.912x; the cap admits 397,093 rows", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers.puf_raw_source: raw._payload_bound(207696, packaged_definition()).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Sized by the upstream row count and the definition's widths; the survey fraction never enters." + }, + { + "constant": "puf_raw_source.py:124 HEADER_MAX_BYTES", + "value": "64 * 1024 = 65,536", + "base_value": "65,536", + "head_value": "65,536", + "enforced_at": [ + "puf_raw_source.py:1153-1155 (`encoded = canonical_json(header)` / `if len(encoded) > HEADER_MAX_BYTES: raise _refuse(\"HEADER_OVER_BOUND\", len(encoded))`)", + "puf_raw_source.py:1394-1396 (decode_return_status: `if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: raise _refuse(\"ENVELOPE_HEADER_LENGTH\")`)" + ], + "refusal_codes": [ + "HEADER_OVER_BOUND", + "ENVELOPE_HEADER_LENGTH" + ], + "encodes": "The return-status envelope header: 26 column entries, two source pins, and the fixed facts block (code_frequencies over a handful of code values).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "6,027 B measured with 207,696-row facts; headroom 10.9x", + "bytes_at_full_source": "same (row-independent)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers.puf_raw_source: header built exactly as :1124-1152 with raw._facts_from_arrays over invented 207,696-row typed arrays.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "not-a-transport", + "notes": "The 4-byte big-endian length prefix at :1157 is the wire width; the decoder's `canonical_json(recomputed) != canonical_json(dict(facts))` at :1464-1465 re-encodes the fixed facts block." + }, + { + "constant": "puf_raw_source.py:185 _LEXICAL_WIDTH_MAX", + "value": "64 (literal)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_raw_source.py:1271-1273 (_envelope_columns: `width = _envelope_int(entry[\"width\"], f\"{name}.width\", minimum=1)` / `if width > _LEXICAL_WIDTH_MAX: raise _refuse(\"ENVELOPE_COLUMN_WIDTH\", name)`), followed by the exact-width match at :1274-1275 whenever a definition resolves", + "the independent hardcoded 64 at puf_raw_source.py:381-383 (_definition_from_document: `not 0 < width <= 64` -> \"DEFINITION_LEXICAL_WIDTH\")", + "per-token enforcement of the declared widths: :664-665 (_token \"TOKEN_OVER_WIDTH\"), :1037-1038 (_fixed_ascii \"LEXICAL_OVER_WIDTH\")" + ], + "refusal_codes": [ + "ENVELOPE_COLUMN_WIDTH", + "DEFINITION_LEXICAL_WIDTH", + "TOKEN_OVER_WIDTH", + "LEXICAL_OVER_WIDTH" + ], + "encodes": "Per-cell allocation of the fixed-width NUL-padded lexical bodies (RECID declares exactly 64, S006 32, FLPDYR 8, nine fields 4).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 (RECID, exactly at the cap); invariant", + "bytes_at_full_source": "same", + "measurement_basis": "puf_2015_raw_source_definition.json fields[*].lexical_width.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "A wire-format width mirrored by three separate 64 literals (:185, :382, and puf_monetary_source.py:187)." + }, + { + "constant": "puf_raw_source.py:230 _INT64_MAX", + "value": "2**63 - 1 = 9,223,372,036,854,775,807", + "base_value": "2**63-1", + "head_value": "2**63-1", + "enforced_at": [ + "puf_raw_source.py:671-675 (_int64: `if value > _INT64_MAX: raise _refuse(\"TOKEN_INT64_OVERFLOW\", label)`)" + ], + "refusal_codes": [ + "TOKEN_INT64_OVERFLOW" + ], + "encodes": "The int64 storage width of the typed RECID/S006 columns (`", + "value": "len(value) != 64 / len(value) != 40 (sha256 / git-blob sha1 hex widths)", + "base_value": "64 / 40", + "head_value": "64 / 40", + "enforced_at": [ + "puf_raw_source.py:284-291 and :294-301 (-> \"DEFINITION_DIGEST\"), called at :325-329, :1192, :1215-1217, :1278" + ], + "refusal_codes": [ + "DEFINITION_DIGEST" + ], + "encodes": "Digest strings in the definition document and envelope header.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "64 / 40 characters exactly", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Fixed-width fields." + }, + { + "constant": "puf_full_source.py:27 FULL_SOURCE_MAX_BYTES", + "value": "128 * 1024 * 1024 = 134,217,728", + "base_value": "134,217,728", + "head_value": "134,217,728", + "enforced_at": [ + "puf_full_source.py:194-197 (decode_full_puf_source: `rows * len(PROJECTED_COLUMNS) * 8 <= FULL_SOURCE_MAX_BYTES` -> \"FULL_PROJECTION_BODY_LIMIT\")", + "puf_full_source.py:290 (encode_full_puf_source: `_require(sum(len(b) for b in blocks) <= FULL_SOURCE_MAX_BYTES, \"FULL_BODY_LIMIT\")`, blocks = status payload + 59 int64 columns)", + "puf_full_source.py:297-300 (decode_full_puf_artifact: `len(payload) <= len(_MAGIC) + 8 + _HEADER_LIMIT + FULL_SOURCE_MAX_BYTES` -> \"FULL_ARTIFACT_LIMIT\")", + "puf_full_source.py:346-350 (`0 < h[\"status_bytes\"] <= FULL_SOURCE_MAX_BYTES` -> \"FULL_STATUS_LENGTH\")" + ], + "refusal_codes": [ + "FULL_PROJECTION_BODY_LIMIT", + "FULL_BODY_LIMIT", + "FULL_ARTIFACT_LIMIT", + "FULL_STATUS_LENGTH" + ], + "encodes": "Two sums at PUF grain: the 59 x int64 projection body (472 B/row) at :195, and at :290 that body plus the embedded return-status payload (169 B/row of bodies plus its 6 KB header).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": ":195 arm 98,032,512; :290 arm about 133,139,209 (estimate: 98,032,512 + 35,100,624 body + 6,027 header + magic/prefix); invariant", + "bytes_at_full_source": "same; headroom at :290 about 1,078,519 B (0.8%); the encode cap admits about 209,378 rows against 207,696 delivered", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers.puf_full_source: exact column arithmetic plus the measured raw header; an actual status payload was not built (open_questions).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports puf_full_source (importers: graph_full_puf_enrichment, puf55_survey_recipients, graph_puf55_canonical_donor, puf59_canonical, puf_full_source_graph).", + "classification": "must-not-move", + "notes": "The tightest fixed ceiling in the family, but it cannot move with the survey fraction; the exact upstream sizes are asserted at :190 (`len(data) == pin.bytes` -> \"FULL_SOURCE_SIZE\") and :343-345 (\"FULL_ARTIFACT_ROWS\")." + }, + { + "constant": "puf_full_source.py:94 AGGREGATE_LEXEME_MAX_CHARACTERS", + "value": "64 (literal; the name is never compared - the value is duplicated into `_AGGREGATE = re.compile(r\"[\\x20-\\x7e]{0,64}\\Z\", re.ASCII)` at :95)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_full_source.py:219-222 (decode_full_puf_source: `all(_AGGREGATE.fullmatch(token) for token in tokens.values())` -> \"FULL_AGGREGATE_LEXICAL_GRAMMAR\")", + "puf_full_source.py:391-398 (decode_full_puf_artifact: `type(v) is str and _AGGREGATE.fullmatch(v)` -> \"FULL_AGGREGATE_LEXICAL_GRAMMAR\")" + ], + "refusal_codes": [ + "FULL_AGGREGATE_LEXICAL_GRAMMAR" + ], + "encodes": "Characters of one retained disclosure-aggregate token: 4 aggregate rows x 59 projected columns = 236 tokens, kept verbatim in the JSON header.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "at most 64 characters per token; the delivered widths live in the gated PUF file and were not read", + "bytes_at_full_source": "same; invariant", + "measurement_basis": "Read from the code; the header measurement below bounds the worst case (64-char tokens) at 24,845 B.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "A lexical grammar over upstream-delivered literals; editing the named constant alone changes nothing (maintenance trap noted by the row-ceilings lane)." + }, + { + "constant": "puf_full_source.py:254 _HEADER_LIMIT", + "value": "128 * 1024 = 131,072", + "base_value": "131,072", + "head_value": "131,072", + "enforced_at": [ + "puf_full_source.py:289 (encode: `_require(len(header) <= _HEADER_LIMIT, \"FULL_HEADER_LIMIT\")`)", + "puf_full_source.py:297-300 (a term of \"FULL_ARTIFACT_LIMIT\")", + "puf_full_source.py:303-304 (decode: `size = struct.unpack(\" 0, \"FULL_HEADER_LIMIT\")`)" + ], + "refusal_codes": [ + "FULL_HEADER_LIMIT", + "FULL_ARTIFACT_LIMIT" + ], + "encodes": "The full-source artifact's JSON header: 59 column entries plus the 236 retained aggregate tokens and source pins.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "12,573 B with 12-character aggregate tokens; 24,845 B worst case at 64-character tokens; headroom 10.4x / 5.3x", + "bytes_at_full_source": "same (row-independent)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers.puf_full_source: header built as :272-288 with json.dumps(sort_keys, ascii).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "not-a-transport", + "notes": "The 8-byte `struct.pack(\" _MONEY / :91 _COUNT", + "value": "`-?[0-9]{1,12}` (12 digits) and `[0-9]{1,2}` (2 digits); mirrored by `cap = 99 if name in COUNT_COLUMNS else 999_999_999_999` at :404", + "base_value": "12 / 2 digits", + "head_value": "12 / 2 digits", + "enforced_at": [ + "puf_full_source.py:225-231 (decode_full_puf_source: `grammar.fullmatch(token) is not None` -> \"FULL_ORDINARY_TOKEN_GRAMMAR:\" + name)", + "puf_full_source.py:404-409 (decode_full_puf_artifact: `((value[~aggregate] >= -cap) & (value[~aggregate] <= cap)).all()` -> \"FULL_ORDINARY_TYPED_DOMAIN\")" + ], + "refusal_codes": [ + "FULL_ORDINARY_TOKEN_GRAMMAR", + "FULL_ORDINARY_TYPED_DOMAIN" + ], + "encodes": "Digit width of one ordinary-return amount or count token (the publisher's 12-digit amount fields; 2-digit exemption counts).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "12 / 2 digits; invariant", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Token grammar widths on upstream literals; the same 12-digit rule as puf_monetary_source._MAX_AMOUNT_DIGITS." + }, + { + "constant": "puf_price_baseline.py:153 _CPI_RESOURCE_MAX_BYTES", + "value": "16 * 1024 * 1024 = 16,777,216", + "base_value": "16,777,216", + "head_value": "16,777,216", + "enforced_at": [ + "puf_price_baseline.py:267-270 (verify_cpi_resource_bytes: `_require(len(resource_bytes) <= _CPI_RESOURCE_MAX_BYTES, \"PRICE_BASELINE_RESOURCE_OVERSIZE\")`); PufGrowthRefusalError via _require at :179-181" + ], + "refusal_codes": [ + "PRICE_BASELINE_RESOURCE_OVERSIZE" + ], + "encodes": "The retained BLS flat file cu.data.1.AllItems (an upstream resource), split into lines after the check (:216).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "about 2.7 MB per the comment at :150-152; no in-tree copy of the file exists, so not measured", + "bytes_at_full_source": "same; invariant", + "measurement_basis": "Comment at :150-152 only; tests/test_us_puf_price_baseline.py:421-426 exercises cap + 1.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports puf_price_baseline (importers: graph_full_puf_enrichment, graph_puf_detail_transfer).", + "classification": "must-not-move", + "notes": "Protects an upstream file's real size (6x headroom by the comment's figure); exact identity is by the whole-file sha256 at :272-273." + }, + { + "constant": "full_puf_enrichment.py:156 ", + "value": "abs(int(v)) <= 2**53 -> 9,007,199,254,740,992", + "base_value": "2**53", + "head_value": "2**53", + "enforced_at": [ + "full_puf_enrichment.py:151-160 (_numeric: `all(not isinstance(v, (int, np.integer)) or abs(int(v)) <= 2**53 for v in observed)` -> f\"PUF_FLOAT64_INTEGER_RANGE:{label}\"); ValueError(code) via _require at :130-132" + ], + "refusal_codes": [ + "PUF_FLOAT64_INTEGER_RANGE:{label}" + ], + "encodes": "The magnitude of one integer donor/recipient cell before the float64 model boundary.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "PUF-derived integers are at most 12 digits; width 2**53", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports full_puf_enrichment (importers: graph_puf55_canonical_donor, graph_full_puf_enrichment, puf55_*).", + "classification": "must-not-move", + "notes": "An encoder width (float64), duplicated from puf_growth.MAX_EXACT_AMOUNT as a literal." + }, + { + "constant": "graph_full_puf_enrichment.py:580 ", + "value": "none (consumer-side re-encoding `values[\"placement\"].payload == _placement(binding)`; the encoder codec.encode_json has no cap and no len() check exists at the consumer)", + "base_value": "none", + "head_value": "none", + "enforced_at": [ + "graph_full_puf_enrichment.py:611-615 (_checked_artifacts: `require(values[\"placement\"].payload == _placement(binding), \"FULL_PUF_PLACEMENT_BINDING\")`), reached from FullPufAttachKernel.run (:725), verify_materialized_full_puf_attachment (:742-744) and load_full_puf_attachment_artifacts (:866)" + ], + "refusal_codes": [ + "FULL_PUF_PLACEMENT_BINDING" + ], + "encodes": "The placement artifact `codec.encode_json({**params, \"target_order\", \"selected_ids\": {entity: frame.table(entity).loc[mask, id_column].tolist()}, ...})` (:580-594): the id lists of every person and tax unit under the PUF clone masks, i.e. the clone-1 half of the clone roster (stacked persons plus stacked tax units).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "5,127,790 SCALED (9.0 B per 8-digit clone id x 356,501 persons + 9.0 x 213,253 tax units)", + "bytes_at_full_source": "51,277,932 SCALED (9.0 x 3,565,013 persons + 9.0 x 2,132,535 tax units; tax units are the 1/1000 roster ratio 2128/1584)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> id_lists: codec.encode_json({\"selected_ids\": {\"person\": ids}}) at 1 vs 100,001 ids with 8-digit clone-1 ids (10**7 + stacked ordinal).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports graph_full_puf_enrichment (its importers are the property/housing/health graphs and the puf55 route, none on the 19/45/age paths); would reach via full_puf_attachment_nodes -> FullPufMaskKernel.run (:699) producing it and FullPufAttachKernel.run (:718) re-encoding it.", + "classification": "segment", + "notes": "A whole-roster JSON produced whole and re-encoded whole by three consumers; no explicit cap anywhere, so binds is false by definition and the segmented shape is the lane's choice. The `to_bytes(8, \"little\")` at :113 is a digest length prefix, not a bound." + }, + { + "constant": "graph_current_survey_puf_transfer.py:131 ", + "value": "none (consumer-side re-encoding `placement.payload == _binding(...)`; codec.encode_json has no cap)", + "base_value": "none", + "head_value": "none", + "enforced_at": [ + "graph_current_survey_puf_transfer.py:288-298 (CurrentSurveyAttachKernel.run: `placement.payload == _binding(projection.payload, matrix.payload, matrix.producer_key, self.clone_population.frame, mask)` -> \"SURVEY_TRANSFER_PLACEMENT_BINDING\")", + "graph_current_survey_puf_transfer.py:388-394 (verify_materialized_current_survey_transfer: same equality)" + ], + "refusal_codes": [ + "SURVEY_TRANSFER_PLACEMENT_BINDING" + ], + "encodes": "The placement artifact `codec.encode_json({..., \"selected_ids\": frame.table(\"tax_unit\").tax_unit_id.to_numpy()[mask].tolist(), ...})` (:131-147): the tax-unit ids under detail.MASK (clone-1 tax units).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "1,919,281 SCALED (9.0 B x 213,253 tax units)", + "bytes_at_full_source": "19,192,815 SCALED (9.0 B x 2,132,535 tax units)", + "measurement_basis": "Same encoder measurement as the graph_full_puf_enrichment row (9.0 B per 8-digit clone id through codec.encode_json); census-brief \u00a72 says not to spend measurement effort here, so this is the one-line per-id figure only.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "census-brief \u00a72 lists graph_current_survey_puf_transfer as off every path; its only importer is graph_full_puf_enrichment.", + "classification": "segment", + "notes": "No cap at producer or consumer; recorded for completeness." + }, + { + "constant": "current_survey_amounts.py:219 ", + "value": "none (consumer-side re-encoding of another module's roster document: `codec.sha(uc.person.to_json(orient=\"table\").encode()) == uc.evidence[\"projection_sha256\"]`; pandas to_json has no cap)", + "base_value": "none", + "head_value": "none", + "enforced_at": [ + "current_survey_amounts.py:217-222 (qualify_current_survey_amounts: `require(codec.sha(uc.person.to_json(orient=\"table\").encode()) == uc.evidence[\"projection_sha256\"], \"UC_PROJECTION_CHANGED\")`)" + ], + "refusal_codes": [ + "UC_PROJECTION_CHANGED" + ], + "encodes": "The ASEC-arm unemployment person table produced by current_asec_unemployment_source.qualify_current_asec_unemployment (one row per ASEC original person), re-encoded whole as pandas orient=\"table\" JSON for a digest comparison.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "not measured (14,212 ASEC persons at 1/10; the producer's column roster lives in current_asec_unemployment_source, another family's module)", + "bytes_at_full_source": "not measured (142,125 ASEC persons)", + "measurement_basis": "Not measured: the document's schema is defined outside this family; see open_questions.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "census-brief \u00a72 lists current_survey_amounts as off every path (importers: graph_current_survey_housing, graph_full_puf_enrichment, graph_us_survey_enrichment).", + "classification": "segment", + "notes": "The same function also materialises four more roster-sized to_json(orient=\"table\") strings for digests with no cap: origins (:139, stacked persons), native and reports (:366-367, stacked persons x amount columns), and each route's donor_columns (:345, ASEC donors). Only the final evidence document is bounded (see the MAX_PAYLOAD_BYTES row)." + }, + { + "constant": "puf_support.py:2331 _INT64_MAX", + "value": "2**63 - 1 = 9,223,372,036,854,775,807", + "base_value": "2**63-1", + "head_value": "2**63-1", + "enforced_at": [ + "puf_support.py:2360-2368 (_remap_ids: `shift = int(clone_index) * int(id_multiplier)` / `if len(values) and shift + int(values.max()) > _INT64_MAX: raise ValueError(\"PUF support ID remap would overflow int64: ... Assembled source IDs must not exceed {PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID}.\")`, a prose ValueError with no short code)" + ], + "refusal_codes": [ + "(none: prose ValueError \"PUF support ID remap would overflow int64: ...\")" + ], + "encodes": "The int64 storage width of the structural id columns after the decimal clone remap (id + clone_index * 10**digits(max_id)).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "max remapped id 10**6 + 356,501 = 1,356,501 (stacked persons set the pre-clone maximum)", + "bytes_at_full_source": "max remapped id 10**7 + 3,565,013 = 13,565,013; width 9.2 x 10**18 (11.8 orders of magnitude of headroom)", + "measurement_basis": "Derived from _id_multiplier_for_values (:2334-2345, `10 ** max(1, len(str(max_id)))`) computed once on the pre-clone frame (:578) and the census-brief stacked-person counts.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "graph_combined_clone (nodes combined_survey_puf_support_clone / .owned) -> CombinedCloneExpandKernel.run (graph_combined_clone.py:527-531) -> puf_support.clone_us_frame_for_puf_support (:491) -> _clone_preassembled_entity_table (:583, :2082-2103) and _clone_link_table (:598) -> _remap_ids (:2348-2369). Also graph_survey_population._verify_cloned_frame (graph_survey_population.py:797-808) -> puf_support._id_multiplier_for_frame (no bound).", + "classification": "must-not-move", + "notes": "An encoder width. The same 19-node chain continues into validate_puf_clone_attachment (fraction 1.0, graph_combined_clone.py:149 and :533) -> _validate_full_clone_identity (:1128-1179) -> _source_ids_sha256 (:1182-1187), which materialises a JSON list of every detail household source id (8.0 B/id: 1,269,896 B at 1/10, 12,699,008 B at full source) for a digest with no cap, twice per clone (also at :636-641 inside clone_us_frame_for_puf_support); not a bound, so no row (open_questions)." + }, + { + "constant": "operator_column_contracts.py:64 PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID (re-exported at puf_support.py:53-55)", + "value": "10**15 - 1 = 999,999,999,999,999", + "base_value": "10**15 - 1", + "head_value": "10**15 - 1", + "enforced_at": [ + "spine_assembly.py:630-638 (`oversized_ids = np.unique(ids[ids > PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID])`, prose ValueError naming \"the clone-safe bound\"; module outside this family, grep-verified at this head)", + "spine_assembly.py:763-766 (post-remap re-check, prose ValueError)", + "puf_support.py:2362-2368 names it in the _remap_ids overflow message only (not a comparison)" + ], + "refusal_codes": [ + "(none: prose ValueErrors in spine_assembly.py:632-641 and :764-767)" + ], + "encodes": "The largest assembled structural id value the clone remap may shift (an id-space width so that id + clone_index * 10**16 stays inside int64).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "largest assembled id about 356,501 (stacked persons)", + "bytes_at_full_source": "about 3,565,013; the bound is 2.8 x 10**8 times larger", + "measurement_basis": "Census-brief counts; the comment at puf_support.py:2324-2329 gives the derivation.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "The enforcing module spine_assembly is not imported by any of the three runners or by graph_combined_clone (which imports only spine_assembly_manifest from support_provenance); the 19-node chain reaches only the message at puf_support.py:2367.", + "classification": "must-not-move", + "notes": "An int64 arithmetic-width argument; the definition is in operator_column_contracts (another family's module); recorded here because the row-ceilings census keyed it to puf_support.py:54." + }, + { + "constant": "puf_support.py:1079 ", + "value": "len(tail_sha) != 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_support.py:1077-1082 (validate_puf_clone_attachment, version-2 manifests: `not isinstance(tail_sha, str) or len(tail_sha) != 64 or any(character not in \"0123456789abcdef\" ...)` -> ValueError(f\"{boundary}: tail-descendant manifest SHA-256 is malformed.\"))" + ], + "refusal_codes": [ + "(prose ValueError \"...tail-descendant manifest SHA-256 is malformed.\")" + ], + "encodes": "One sha256 hex string in the tail-descendant attachment manifest.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "The 19-node combined clone uses fraction 1.0 with no manifest, so validate_puf_clone_attachment returns from _validate_full_clone_identity (:913-917) before this branch; only a version-2 (tail descendant, bind_puf_clone_attachment_tail_descendant) manifest reaches :1053-1082.", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "post_transfer_calibration.py:1189 ", + "value": "len(value) != 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "post_transfer_calibration.py:1187-1195 (validate_post_transfer_calibration_receipt: nine scope/weights digests `not isinstance(value, str) or len(value) != 64 or any(character not in \"0123456789abcdef\" ...)` -> ValueError(f\"{boundary}: post-transfer calibration context digest is invalid.\"))" + ], + "refusal_codes": [ + "(prose ValueError \"...post-transfer calibration context digest is invalid.\")" + ], + "encodes": "sha256 hex strings in a calibration receipt.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports post_transfer_calibration (importers: graph_full_puf_enrichment, stacked_spine).", + "classification": "must-not-move", + "notes": "The only byte-shaped check in the module; its value arrays are digested whole via `np.ascontiguousarray(values).tobytes()` (:565-574) with no cap, which is a digest, not a bound." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/other/measure_roles_status_puf.py (run with /Users/maxghenis/PolicyEngine/_worktrees/microcosm-native-byte-transports/.venv/bin/python, sys.path set to packages/*/src, every imported module checked to resolve under the worktree)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/other/roles-status-puf-measurements.json (receipt: roles_projection, roles_receipt, person_status_projection, person_status_receipt, id_lists, puf_headers, growth_documents)" + ], + "open_questions": [ + "byte-census-measurements.json roles_projection (324.0 B/person, 1,155,064,212 at full source) is not the document at this head: byte_census_measure.roles_projection builds a 7-column DataFrame directly (omitting UNBOUND_REASON, HOUSEHOLD_VERDICT, VALUE_KNOWN, ALLOCATION_KNOWN, ORIGIN, OBSERVATION_YEAR and VALUE) with placeholder strings (\"observed_reference_person\" in the int64 ROLE_STATE column, \"housing_unit\" for UNIVERSE). Through roles._observation_table with all 13 COLUMN_TOKENS and the real ORIGIN/UNIVERSE literals (55 and 27 characters) the projection is 650.0 B/person: 231,725,650 at 1/10 and 2,317,258,450 at full source; the 64 MiB cap admits 103,244 persons, not 207,126. The row-ceilings census's computed 648 B/row agrees. The lane JSON should be re-measured.", + "Unbounded whole-roster materialisation on the 19-node path: puf_support._source_ids_sha256 (:1182-1187) json.dumps's the list of every detail household source id (8.0 B/id: 1,269,896 B at 1/10, 12,699,008 B at full source) for a sha256, twice per clone (clone_us_frame_for_puf_support :636-641 and graph_combined_clone.py:533, both via _validate_full_clone_identity :1178). No cap exists so it is not a row; a streaming digest would remove the copy.", + "Unbounded materialisations beside the person-status projection (reachable only from person_status=True graphs): graph_current_survey_person_status._columns_payload (:391-401) computes `_sha(canonical_json(_table_document(table)))` for the 45-column literal table (816,423,843 B at full source) and the 121-column recoded table (1,351 B per ACS person, 1,606 per ASEC person: 485,377,090 B at 1/10 and 4,853,789,741 B at full source), each on every _expected_artifacts/_result call; and projection_payload re-encodes the 1.55 GB projection on every _check_artifacts. Likewise current_survey_household_roles._table_stamp (:638-642) and the roles graph _seal_result (:452-469) materialise `to_json()` per column (the bound is_household_head column spans 7,130,026 clone persons at full source) for digests with no cap.", + "current_survey_amounts materialises five roster-sized to_json(orient=\"table\") strings for digests with no cap (:139 origins, :219 uc.person, :345 donor_columns, :366 native, :367 reports); the UC_PROJECTION_CHANGED row's bytes were not measured because uc.person's schema is defined in current_asec_unemployment_source (another family). Off every runner path.", + "Cross-family bounds this family calls into without owning them: current_survey_person_status_source.py:387 passes `budget=[original.asec._BODY_MAX]` (asec_coverage_authentication.py:39, 268,435,456) to that module's _capture (:309-330, \"COVERAGE_BODY_BYTES\" at :286) on the ASEC pppub25.csv member; roles :1075 and status_source :406-410 call acs_housing_universe_source._copy(..., exact_size=size) (\"FILE_TOO_LARGE\"/\"SOURCE_SIZE\", :131-140) on the ACS person archive; roles :769-770 and status_source :80-81 stream records through acs_person_coverage_authentication._records/_decode_record, whose MAX_RECORD_BYTES 400,000 / MAX_TOKEN_BYTES 65,536 / MAX_CSV_HEADER_BYTES 65,536 (:48-50) apply first. Those rows belong to the coverage families; the MAX_PAYLOAD_BYTES row here lists only this family's two enforcement sites and must be merged into the survey-population family's definition row.", + "Read chunk sizes that are not caps and were not given rows: current_survey_household_roles.py:1091 and current_survey_person_status_source.py:426 `while stream.read(65536)` (exhausting auxiliary zip members for CRC), puf_qrf_chain.py:1063 and puf_raw_source.py:523 1 MiB chunks. Digest length prefixes `to_bytes(8, ...)` at graph_full_puf_enrichment.py:113 and multispine_pool.py:3484, and `struct.pack(\" MAX_PAYLOAD_BYTES: segments.append(bytes(payload)); payload = bytearray()` then `_require(len(piece) <= MAX_PAYLOAD_BYTES - len(payload), \"TRANSPORT_LIMIT\")` \u2014 SurveyOriginBudgetError(ValueError) (:79-85)", + "survey_origin_budget.py:92-93 `_json = graph._bounded_json(value, MAX_PAYLOAD_BYTES)` \u2192 graph_survey_population.py:272 (limit itself \u2264 64 MiB), :281, :283 \"TRANSPORT_LIMIT\" \u2014 SurveyPopulationGraphError(ValueError) (graph_survey_population.py:116-122); pieces built with it at :645 (one id/group entry), :653 (key), :659 (one origin record), :664 (one fixed header value)", + "survey_origin_budget.py:277 `_json([asdict(record) for record in population.mass_ledger])` in _population_identity (fixed-shape ledger, one record per node transition)", + "survey_financial_successor.py:116-131 `budget._json({...})` for the fixed-shape financial successor document (reachable_from none; see that row)" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT" + ], + "encodes": "The sampling-origin budget document: one origin record per selected household (`origins`, 709-742 B each through _json at :659) plus two header lists with one entry per clone household row (`household_ids`, `group_indices`, streamed element by element at :637-646), plus a fixed header (:594-616; 26,258 B with the 1/1000 receipt's producer/native/catalogues). At this head the constant bounds one piece and one accumulation (segment) of that stream; the whole document is bounded by MAX_ROSTER_BYTES (:633, :668).", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "117,809,350 = 742.0 B/group \u00d7 158,737 + 26,258 B preamble (measured, variant 'measurements_json_params'); byte-census-measurements.json origin_budget.tenth_bytes = 117,782,854 = 742 \u00d7 158,737 without the preamble \u2014 same encoder and same record fields at this head (the diff moved only the stream, not the fields). Tenth-like fraction widths (p=[4221,42211] / [5570,55697]) measure 764-775 B/group \u2192 121,301,564-123,047,671.", + "bytes_at_full_source": "1,177,861,631 = 742.0 B/group \u00d7 1,587,376 + preamble (byte-census-measurements.json origin_budget.full_source_bytes = 1,177,832,992). ASEC-like widths (d=37601/100) give 747 B/group \u2192 1,185,798,511. Group indices measured at 7-digit widths; the true average digit count over 0..1,587,375 is 6.30 (ids 1..3,174,752: 6.65), so the true per-group cost is ~2 B lower.", + "measurement_basis": "measure_budget_document.py: an exact replica of _document's append/append_list stream (survey_origin_budget.py:625-668) over invented records built with the module's own _reference (:105-140) and _json at full-source id widths, asserted byte-identical to graph._segmented_json({**header, 'origins': records}, segment=MAX_PAYLOAD_BYTES, maximum=MAX_ROSTER_BYTES); per-group cost = (bytes at 2002 groups \u2212 bytes at 2 groups)/2000 so the fixed header cancels; header fields taken from the 1/1000 preparation.json. Record bytes via _json: 709 (JSON params), 714 (ASEC-like), 731/742 (tenth-like).", + "binds_at_tenth": true, + "binds_at_full_source": true, + "base_value": "64 MiB on the WHOLE document: the base append() was `_require(len(piece) <= MAX_PAYLOAD_BYTES - len(payload), \"TRANSPORT_LIMIT\")` into a single bytearray with no segment close, and household_ids/group_indices were single _json pieces (25.4 MB each at full source) \u2014 git diff origin/native-row-ceilings, \u2212 side of :619-635 and :661-662. At that scope: 1.76\u00d7 at 1/10, 17.55\u00d7 at full source (90,443 groups admitted, 5.7% of source).", + "head_value": "64 MiB on one piece and one accumulation (:629-632); the largest piece measured is one origin record, 742 B; each accumulation closes at 64 MiB (~18 segments at full source); the whole document is under MAX_ROSTER_BYTES.", + "reachable_from": "age", + "reach_chain": "survey_age_calibration.run_survey_age_calibration | run_survey_age_development \u2192 _run_survey_age_calibration \u2192 budgets.freeze_survey_origin_budget (:529; again with candidate at :660-666) \u2192 _document (:1005) \u2014 and budget.checked_view (:535, :855 via verify_survey_origin_budget) \u2192 _validate_budget \u2192 _document (:777). The 19/45-node runners touch survey_origin_budget only at graph_atomic_survey_financial.py:721/:1034/:1353/:1890 (_config_payload, _live()), which never reach _initial or _document.", + "classification": "segment", + "notes": "The shared encoder's own ceiling is also 64 MiB \u2014 graph_survey_population.py:272 `_require(type(limit) is int and 0 < limit <= 64 * 1024**2, \"TRANSPORT_LIMIT\")` \u2014 so MAX_PAYLOAD_BYTES cannot be raised above 64 MiB without changing _bounded_json/_canonical_pieces. It is part of the _live() producer-seal contract tuple (:198), so moving it changes the seal. Test pins: test_us_native_row_ceilings.py:153 asserts == 64 MiB and :154 divides by 764 B/group (the row-ceilings lane's tenth-like widths; this head measures 742 at full-source widths, 764-775 at tenth-like widths). The comment at :55-62 quotes 742 B/group, 1,177,832,992 B and 90,443 households \u2014 all reproduced here." + }, + { + "constant": "survey_origin_budget.py:63 MAX_ROSTER_BYTES [budget document total]", + "value": "64 * MAX_PAYLOAD_BYTES = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "survey_origin_budget.py:633 append(): `_require(total + len(piece) <= MAX_ROSTER_BYTES, \"TRANSPORT_ROSTER_LIMIT\")` \u2014 SurveyOriginBudgetError", + "survey_origin_budget.py:666-668 after `document = b\"\".join(segments)`: `_require(len(document) == total <= MAX_ROSTER_BYTES, \"TRANSPORT_ROSTER_LIMIT\")`", + "survey_origin_budget.py:975 freeze_survey_origin_budget: `_require(candidate is None or len(candidate) <= MAX_ROSTER_BYTES, \"CANDIDATE_LIMIT\")` (candidate = the same budget bytes, passed at survey_age_calibration.py:664)", + "graph_survey_budget.py:49-53 _document: `_require(type(payload) is bytes and 0 < len(payload) <= budgets.MAX_ROSTER_BYTES, \"PAYLOAD\")` \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_PAYLOAD\") (:40-42), then json.loads(payload) at :55; reached from numeric_survey_budget_payload :71, SurveySamplingBudgetKernel.__init__ :201 and .run :217" + ], + "refusal_codes": [ + "TRANSPORT_ROSTER_LIMIT", + "CANDIDATE_LIMIT", + "SURVEY_BUDGET_TRANSPORT_PAYLOAD" + ], + "encodes": "The whole sampling-origin budget document (the same bytes as the accumulation row above): one origin record per selected household plus two per-clone-row header lists, joined once from the segments.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "117,809,350 (see the MAX_PAYLOAD_BYTES accumulation row; byte-census-measurements.json 117,782,854 without preamble)", + "bytes_at_full_source": "1,177,861,631 (byte-census-measurements.json 1,177,832,992 without preamble) = 0.274 of 4 GiB", + "measurement_basis": "measure_budget_document.py (same replica stream, same difference-of-two-counts); the joined length equals the streamed total by :668.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none \u2014 the definition was added on this branch (git diff origin/native-row-ceilings: `+MAX_ROSTER_BYTES = 64 * MAX_PAYLOAD_BYTES` at :63); the base's ceiling on this document was MAX_PAYLOAD_BYTES (recorded in that row). CANDIDATE_LIMIT at :975 was `len(candidate) <= MAX_PAYLOAD_BYTES` in the base; graph_survey_budget.py:51 was `<= budgets.MAX_PAYLOAD_BYTES` in the base.", + "head_value": "4,294,967,296; full-source document is 0.274 of it (headroom 3.65\u00d7); also in the _live() contract tuple (:199)", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration \u2192 budgets.freeze_survey_origin_budget (:529, :660 with candidate=budget_payload) \u2192 _document :633/:668 and :975; \u2192 transport.numeric_survey_budget_payload (:537) and SurveySamplingBudgetKernel (:557, run inside run_graph :697) \u2192 graph_survey_budget._document :51/:55.", + "classification": "stays", + "notes": "This is the segmented transport's explicit total (comment :59-62: 'the total a process will materialise for the document is 64 accumulations, the same multiple survey_population_preparation uses'). The document is nonetheless held whole several times per run: the join at :667, the store artifact (graph_survey_budget.py:262 artifacts={'budget': payload}), json.loads at :917 (checked_view) and :1126, graph_survey_budget.py:55, and two more whole encodings compared byte-for-byte at :786 (`expected == payload`, BUDGET_RECONSTRUCTION) and :1125 \u2014 at full source each is ~1.18 GB. Test pin: test_us_native_row_ceilings.py:158-161 (== 64 \u00d7 MAX_PAYLOAD_BYTES; 764 \u00d7 STACKED_HOUSEHOLDS \u2264 it)." + }, + { + "constant": "survey_origin_budget.py:63 MAX_ROSTER_BYTES [consumer-side re-encoding of the issued budget bytes at :934-937]", + "value": "4,294,967,296 (head); the base used MAX_PAYLOAD_BYTES 67,108,864 here", + "enforced_at": [ + "survey_origin_budget.py:934-937 SamplingOriginBudget.checked_view: `_require(graph._json_matches(document, entry[1], maximum=MAX_ROSTER_BYTES), \"FINAL_BUDGET_VIEW_DOCUMENT\")` \u2192 graph_survey_population.py:356-375: each canonical token (\u2264 64 MiB via _canonical_pieces :303/:305 \"TRANSPORT_LIMIT\") is compared against a memoryview slice of the payload; :371 `_require(end <= maximum, \"TRANSPORT_ROSTER_LIMIT\")` \u2014 SurveyPopulationGraphError", + "survey_origin_budget.py:786 `_require(expected == payload, \"BUDGET_RECONSTRUCTION\")` and :1125 `_require(payload == entry[1], \"FINANCIAL_BUDGET_RECONSTRUCTION\")` \u2014 producer-side whole re-materialisations through _document (a second 1.18 GB copy at full source), under the same MAX_ROSTER_BYTES" + ], + "refusal_codes": [ + "FINAL_BUDGET_VIEW_DOCUMENT", + "TRANSPORT_ROSTER_LIMIT", + "TRANSPORT_LIMIT", + "BUDGET_RECONSTRUCTION", + "FINANCIAL_BUDGET_RECONSTRUCTION" + ], + "encodes": "The decoded budget document (`document = json.loads(entry[1])`, :917) re-encoded by its consumer to prove it is the issued bytes \u2014 the whole roster document.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "117,809,350 (same document as the rows above)", + "bytes_at_full_source": "1,177,861,631 (0.274 of MAX_ROSTER_BYTES)", + "measurement_basis": "measure_budget_document.py; _json_matches walks the same _canonical_pieces stream whose total the replica measured (graph_survey_population.py:369).", + "binds_at_tenth": true, + "binds_at_full_source": true, + "base_value": "MAX_PAYLOAD_BYTES 64 MiB, materialised: `_require(_json(document) == entry[1], \"FINAL_BUDGET_VIEW_DOCUMENT\")` (git diff \u2212 side at :930) \u2014 a second whole encoding under 64 MiB: 1.76\u00d7 at 1/10, 17.55\u00d7 at full source", + "head_value": "MAX_ROSTER_BYTES 4 GiB, streamed token-by-token with no second copy. NOTE: this hunk (:927-937) is an UNCOMMITTED working-tree modification at the head read \u2014 git status shows `M survey_origin_budget.py` and `git diff` (vs HEAD 77bafbb3a) contains only this hunk; HEAD itself still has the base form.", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration \u2192 budget.checked_view (:535), verify_survey_origin_budget (:855), and every checked_view inside admit_survey_weight_only_population (:1362, :1376) and SamplingOriginSuccessor.checked_view (:1303, :1306) \u2192 :934-937.", + "classification": "consumer", + "notes": "The consumer takes the producer's total (maximum=MAX_ROSTER_BYTES). :917 `document = json.loads(entry[1])` still materialises the decoded object tree of the whole document before the streamed comparison; _check_view_constraint (:938) then walks it again (see the digest-payload row)." + }, + { + "constant": "survey_origin_budget.py:63 MAX_ROSTER_BYTES [selection_sha256 stream at :605-607]", + "value": "4,294,967,296 (head); the base used MAX_PAYLOAD_BYTES 67,108,864 here", + "enforced_at": [ + "survey_origin_budget.py:605-607 `\"selection_sha256\": graph._json_sha256(view.receipt[\"selection\"], maximum=MAX_ROSTER_BYTES)` \u2192 graph_survey_population.py:341-353: sha256 over _canonical_pieces(value, 64 MiB); :350 `_require(total + len(encoded) <= maximum, \"TRANSPORT_ROSTER_LIMIT\")`; each token \u2264 64 MiB at :303/:305 \"TRANSPORT_LIMIT\" \u2014 SurveyPopulationGraphError" + ], + "refusal_codes": [ + "TRANSPORT_ROSTER_LIMIT", + "TRANSPORT_LIMIT" + ], + "encodes": "The preparation receipt's `selection` block: `selected` \u2014 one row per selected household (key {native_id, source, source_year, survey_year}, domain, inclusion_probability pair, original_design_weight pair, share pair, statistical_unit) \u2014 plus fixed `cells` (5), `excluded` (65 at 1/1000), fraction, seed, supplied_households. Digested only; never held whole by this module (the same rows are held whole in the preparation receipt, another family's row).", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "39,643,296 = 250.0 B/ACS row (p=[4221,42211]) \u00d7 153,161 + 241.0 B/ASEC row (p=[5570,55697]) \u00d7 5,576 + 8,040 B invented preamble; byte-census-measurements.json selection_block.tenth_bytes_SCALED = 38,594,135 (scaled from 1/1000 widths)", + "bytes_at_full_source": "385,250,455 = 243.0 B/ACS row (p=[1,1]) \u00d7 1,531,614 + 234.0 B/ASEC row \u00d7 55,762 + preamble; byte-census-measurements.json selection_block.full_source_bytes_SCALED = 385,942,815. That is 5.74\u00d7 64 MiB and 0.090 of 4 GiB.", + "measurement_basis": "measure_constraint_digest_and_selection.py: selected rows shaped from the 1/1000 receipt at full-source and tenth widths (13-char ACS native ids, 5-digit ASEC, p reduced to [1,1] at full source), encoded through graph._bounded_json as a difference of 2 vs 2002 rows; the streamed _canonical_pieces total asserted equal to the materialised length (the bytes _json_sha256 digests).", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "MAX_PAYLOAD_BYTES 64 MiB, materialised: `\"selection_sha256\": _sha(_json(view.receipt[\"selection\"]))` (git diff \u2212 side at :595) \u2014 0.59\u00d7 at 1/10, 5.74\u00d7 at full source (276,017 rows admitted per byte-census-measurements.json)", + "head_value": "MAX_ROSTER_BYTES 4 GiB, streamed digest (0.090 of it at full source)", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration \u2192 freeze_survey_origin_budget \u2192 _document :605-607 (and again in _validate_budget :777 and _financial_budget_values :1116 on every checked_view).", + "classification": "segment", + "notes": "Comment at :603-604: 'One row per supplied household: digested from the stream, never held whole. The digest equals _sha(_json(...)) wherever _json admits.' The 1/1000 receipt's selection block is 393,395 B for 1,584 rows (243.13 B/row at those widths)." + }, + { + "constant": "survey_origin_budget.py:54 MAX_PAYLOAD_BYTES [weight-only successor document via _json at :1242 and :1316]", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "survey_origin_budget.py:1242 _checked_successor_document: `payload = _json(_successor_document(state, budget_view))` \u2192 graph._bounded_json(value, MAX_PAYLOAD_BYTES) \"TRANSPORT_LIMIT\" \u2014 SurveyPopulationGraphError", + "survey_origin_budget.py:1316 SamplingOriginSuccessor.checked_view: `_require(_json(document) == entry[1], \"FINAL_SUCCESSOR_VIEW_DOCUMENT\")` \u2014 consumer-side whole re-encoding under the same 64 MiB", + "survey_origin_budget.py:1305 `_require(expected == payload, \"SUCCESSOR_RECONSTRUCTION\")`, :1307-1310 and :1377-1379 \"FINAL_SUCCESSOR_SEAL\" \u2014 further whole re-encodings compared byte-for-byte" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT", + "FINAL_SUCCESSOR_VIEW_DOCUMENT", + "SUCCESSOR_RECONSTRUCTION", + "FINAL_SUCCESSOR_SEAL" + ], + "encodes": "The weight-only successor document (:1210-1222): fixed fields (protocol, budget_sha256, previous_binding_sha256, versions, two frame digests, constraint_digest, release_eligible), `mass_ledger` (one record per node transition, fixed) and `group_total_float64_hex` \u2014 `[float(v).hex() for v in totals]` (:1219), one 23-25 B token per group, i.e. per selected household.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "3,778,465 = 23.8 B/group \u00d7 158,737 + 1,080 B fixed", + "bytes_at_full_source": "37,775,073 = 23.8 B/group \u00d7 1,587,376 + fixed = 0.563 of 64 MiB (headroom 1.78\u00d7); worst case with every token at 25 B: 39,685,480 (0.59)", + "measurement_basis": "measure_successor_and_geography.py: the :1210-1222 dict with uniform(10, 5000) calibrated-looking totals (tokens measured 23-24 B) through survey_origin_budget._json, difference of 2 vs 2002 groups.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 MiB, whole document (unchanged: no diff at :1242 or :1316)", + "head_value": "64 MiB, whole document (unchanged)", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration observe() :616 budgets.admit_survey_weight_only_population \u2192 :1375 _checked_successor_document \u2192 :1242; then :1376-1379 (second encoding); verify_survey_weight_only_successor (:856) \u2192 to_bytes \u2192 SamplingOriginSuccessor.checked_view \u2192 :1304, :1308 and :1316 (three more whole encodings).", + "classification": "stays", + "notes": "Not segmented on this branch and not binding at full source. group_bounds.check (:1199-1202) and the per-row loop over `rows[int(group)][\"design_bound_float64_hex\"]` (:1204-1209) walk Python objects of the decoded budget, not bytes. At least six whole encodings of ~38 MB happen per run through these seals." + }, + { + "constant": "survey_origin_budget.py:54 MAX_PAYLOAD_BYTES [geography-binding semantic_population via _json at :390-392]", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "survey_origin_budget.py:390-392 `\"postclone_geography_population_semantic_sha256\": _sha(_json(semantic_population))` \u2192 graph._bounded_json(value, MAX_PAYLOAD_BYTES): graph_survey_population.py:281 `_require(len(piece) <= limit - len(result), \"TRANSPORT_LIMIT\")` on the single hex-string piece \u2014 SurveyPopulationGraphError", + "survey_origin_budget.py:383-401 the enclosing `_json({...})` binding document and :386 `_sha(_json(semantic_receipt))` \u2014 fixed-shape (digests and the decoded validation receipt), same bound" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT" + ], + "encodes": "`semantic_population` (:370-382): `\"design_weights\": [(entity, str(values.dtype), values.shape, values.tobytes().hex()) for entity, values in population.design_weights.items()]` \u2014 16 hex characters per float64 design weight, one per household row of `value.population`, the reconstructed postclone geography population (module docstring :7-8: 'independently reconstructs the complete postclone geography Frame'; :324-328 bind expanded_population to the COMBINED_CLONE_CLAIM_NODE stage), i.e. the clone household roster; plus fixed frame_sha256/version/owners/weight_kind/mass_ledger.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "5,088,502 = 16.0 B/clone household \u00d7 317,475 + 8,426 B invented preamble", + "bytes_at_full_source": "50,809,220 = 16.0 B \u00d7 3,174,752 + preamble = 0.757 of 64 MiB (headroom 1.32\u00d7)", + "measurement_basis": "measure_successor_and_geography.py: the :370-382 dict with np.full(n, 137.5).tobytes().hex() through survey_origin_budget._json, difference of 2 vs 2002 households.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 MiB (unchanged; no diff in _geography_binding)", + "head_value": "64 MiB (unchanged)", + "reachable_from": "age", + "reach_chain": "Only when the runner is given geography_config: survey_age_calibration.run_survey_age_calibration|run_survey_age_development(geography_config=...) \u2192 freeze_survey_origin_budget(geography_config=...) (:533, :665) \u2192 _initial :449-456 \u2192 _geography_binding; re-encoded at :495-501 (FINAL_GEOGRAPHY_RECONSTRUCTION_SEAL) and on every _validate_budget (:768-776, :796-804). With geography_config None, :448 sets geography_binding = None and the site never runs. Not reached from the 19/45 runners (they call _config_payload only).", + "classification": "stays", + "notes": "The tightest non-binding roster document in this family (1.32\u00d7 headroom at full source), and it is a single encoder piece \u2014 one Python str of 16\u00d7N chars \u2014 so segmenting would not help; the hex string itself is the document. The budget header embeds `json.loads(geography_binding)` (:618), which carries only the sha256 of this document, so the budget document does not grow by it." + }, + { + "constant": "survey_origin_budget.py:68 MAX_GROUPS", + "value": "7_000_000", + "enforced_at": [ + "survey_origin_budget.py:421 _initial: `_require(0 < len(instructions) <= MAX_GROUPS, \"GROUP_COUNT_BOUND\")` \u2014 SurveyOriginBudgetError(ValueError)", + "graph_survey_budget.py:75-86 numeric_survey_budget_payload: `and 0 < len(records) <= budgets.MAX_GROUPS` inside \"COMPLETE_CLONE_SHAPE\" \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_COMPLETE_CLONE_SHAPE\")" + ], + "refusal_codes": [ + "GROUP_COUNT_BOUND", + "SURVEY_BUDGET_TRANSPORT_COMPLETE_CLONE_SHAPE" + ], + "encodes": "A record cap: allocation instructions = groups = selected households (graph.allocation_instructions requires one instruction per selected household \u2014 docs/us-native-row-ceilings.md \u00a73a quotes `_require(len(household_origins) == len(plan.selected), \"ORIGIN_COUNT\")`).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "n/a \u2014 a row count: 158,737 groups at 1/10 (0.023 of the cap)", + "bytes_at_full_source": "n/a \u2014 1,587,376 groups at full source (0.227 of the cap; headroom 4.41\u00d7)", + "measurement_basis": "Counts from \u00a73 of the brief (roster-census.json); no bytes behind this bound.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "7_000_000 (unchanged on this branch; the row-ceilings lane had already moved it from 1_000_000 \u2014 docs/us-native-row-ceilings.md \u00a73a and the comment at :64-67)", + "head_value": "7_000_000", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration \u2192 freeze_survey_origin_budget \u2192 _initial :418-421; and \u2192 transport.numeric_survey_budget_payload :81.", + "classification": "not-a-transport", + "notes": "Row count wearing no byte name; the row-count rule is docs/us-native-row-ceilings.md \u00a73a (4\u00d7 the catalogue's selected-household count, rounded up to the next million). In the _live() contract tuple (:200). Test pins: test_us_native_row_ceilings.py:58 and :156." + }, + { + "constant": "survey_origin_budget.py:69 MAX_SCALAR_CHARS", + "value": "4096", + "enforced_at": [ + "survey_origin_budget.py:569-572 records(): `_require(type(key.native_id) is str and len(key.native_id) <= MAX_SCALAR_CHARS, \"NATIVE_KEY_BOUND\")` per origin record \u2014 SurveyOriginBudgetError" + ], + "refusal_codes": [ + "NATIVE_KEY_BOUND" + ], + "encodes": "A single token: the raw native id of one selected household (`raw_native_id`). Observed widths in the 1/1000 receipt: 13 chars for ACS (`2024GQ0000926`, `2024HU\u2026`), 4-5 chars for ASEC.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "\u2264 13 B per token; per-token width, independent of the fraction", + "bytes_at_full_source": "\u2264 13 B per token (0.3% of the cap)", + "measurement_basis": "Widths counted from preparation.json origins.households (1,529 ACS ids of 13 chars; 48 ASEC of 5, 7 of 4).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4096 (unchanged)", + "head_value": "4096", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration \u2192 freeze_survey_origin_budget \u2192 _document \u2192 records() :569.", + "classification": "must-not-move", + "notes": "Protects a token width; in the _live() contract tuple (:201)." + }, + { + "constant": "survey_origin_budget.py:99 128 (FRACTION_BOUND bit_length)", + "value": "value.numerator.bit_length() <= 128 and value.denominator.bit_length() <= 128 (each integer < 2**128, \u2264 39 decimal digits)", + "enforced_at": [ + "survey_origin_budget.py:96-102 _pair: `_require(value.numerator.bit_length() <= 128 and value.denominator.bit_length() <= 128, \"FRACTION_BOUND\")` \u2014 SurveyOriginBudgetError; reached from _reference :107-108 (d, p, s) and :129-133 (d, p, s, b, a) for every origin record" + ], + "refusal_codes": [ + "FRACTION_BOUND", + "EXACT_FRACTION_REQUIRED" + ], + "encodes": "A fixed-width field: each exact-fraction pair `[numerator, denominator]` in an origin record (original anchor d, inclusion probability p, share s, derived b and a). Observed widths at 1/1000: \u2264 5 digits (e.g. [42, 42211], [37601, 100]); at full source p = [1, 1].", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "\u2264 5 decimal digits per integer observed (e.g. p=[4221,42211]); independent of the roster", + "bytes_at_full_source": "\u2264 5 digits per integer (p reduces to [1,1] at full source); the bound admits 39", + "measurement_basis": "Read from preparation.json selection rows and origin records; the derived b, a pairs are ratios of those.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "128 (unchanged)", + "head_value": "128", + "reachable_from": "age", + "reach_chain": "\u2192 freeze_survey_origin_budget \u2192 _document \u2192 records() \u2192 _reference :561 \u2192 _pair.", + "classification": "must-not-move", + "notes": "Protects the exact-arithmetic token width, not a transport." + }, + { + "constant": "survey_origin_budget.py:871-875 json.dumps(digest_payload, sort_keys=True, separators=(\",\", \":\"), ensure_ascii=True).encode() (cap: none)", + "value": "none \u2014 the encoder takes no limit; the payload is a whole-roster document", + "enforced_at": [ + "survey_origin_budget.py:865-875 _check_view_constraint builds `digest_payload = {\"contract\": ..., \"household_ids\": ids (tuple of every clone row), \"group_indices\": groups (every clone row), \"absolute_bounds_hex\": bounds_hex (one per group)}` and hashes json.dumps(...).encode(); :876-898 `_require(... constraint.constraint_digest == expected_digest, \"FINAL_BUDGET_VIEW_CONSTRAINT\")` \u2014 SurveyOriginBudgetError" + ], + "refusal_codes": [ + "FINAL_BUDGET_VIEW_CONSTRAINT" + ], + "encodes": "A consumer-side re-encoding of the GroupedUpperBounds digest payload that microcosm.calibrate.group_bounds produced (group_bounds.py:71-81): two lists over the clone household roster plus one hex float per group.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "7,761,306 = 48.9 B/group \u00d7 158,737 + 99 B", + "bytes_at_full_source": "77,612,467 = 48.9 B/group \u00d7 1,587,376 + 99 B (1.16\u00d7 64 MiB, for scale; nothing refuses it)", + "measurement_basis": "measure_constraint_digest_and_selection.py: the exact json.dumps call over ids at 7-digit widths, group indices and float.hex() bounds, difference of 2 vs 2002 groups; the replica's sha256 asserted equal to GroupedUpperBounds(...).constraint_digest at both sizes.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (uncapped; unchanged)", + "head_value": "none (uncapped; unchanged)", + "reachable_from": "age", + "reach_chain": "\u2192 budget.checked_view (:535, :855, and inside every admit/verify successor call :1303/:1306/:1362/:1376) \u2192 :938 _check_view_constraint.", + "classification": "stays", + "notes": "No bound to meet; recorded because the brief asks for every consumer-side re-encoding of a roster-shaped document produced by another module. Also materialised at the same call: `immutable_array(...)` at :851-863 builds `np.asarray(expected, dtype).tobytes()` for the whole ids/groups/bounds (another ~50 MB at full source) and `buckets` (:845-849) a Python list per group." + }, + { + "constant": "graph_survey_calibration.py:44 MAX_BYTES [numeric-bounds document: one accumulation]", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "graph_survey_budget.py:146-159 `output = graph._segmented_json({...}, segment=numeric.MAX_BYTES, maximum=numeric.MAX_ROSTER_BYTES)` \u2192 graph_survey_population.py:328-331: segment close, then `_require(len(current) + len(encoded) <= segment, \"TRANSPORT_LIMIT\")`; per-token bound via _canonical_pieces :294/:303/:305 \"TRANSPORT_LIMIT\" \u2014 SurveyPopulationGraphError(ValueError)" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT" + ], + "encodes": "The numeric-bounds document (BOUNDS_PROTOCOL, graph_survey_budget.py:147-156): four lists with one entry per clone household row (household_ids, group_indices, row_upper_hex, incoming_hex), one list with one entry per group (group_upper_hex), plus protocol, budget_sha256, population \u2014 151.97 B per group (75.99 B per clone row).", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "24,123,542 = 151.97 B/group \u00d7 158,737 + 281 B (byte-census-measurements.json numeric_bounds.tenth_bytes = 24,123,261 without the fixed part)", + "bytes_at_full_source": "241,233,811 = 151.97 \u00d7 1,587,376 + 281 (byte-census-measurements.json full_source_bytes = 241,233,530); 3.59\u00d7 64 MiB; 0.056 of MAX_ROSTER_BYTES", + "measurement_basis": "measure_numeric_bounds.py: the :147-156 document shape at full-source id widths through graph._segmented_json(segment=MAX_BYTES, maximum=MAX_ROSTER_BYTES) at this head, asserted byte-identical to graph._bounded_json(doc, MAX_BYTES); bytes at 2 and 2002 groups = 585 and 304,525, exactly the figures in byte-census-measurements.json (so that file's numbers are for this head's encoder); per group = (304,525 \u2212 585)/2000.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "64 MiB on the WHOLE document: graph_survey_budget.py \u2212 side `graph._bounded_json({...}, numeric.MAX_BYTES)` and `_require(len(output) <= numeric.MAX_BYTES, \"NUMERIC_LIMIT\")`; graph_survey_calibration.py:113 \u2212 side `0 < len(payload) <= MAX_BYTES` \"PAYLOAD\". At that scope: 0.36\u00d7 at 1/10, 3.59\u00d7 at full source (883,185 clone rows admitted).", + "head_value": "64 MiB on one segment only (graph_survey_budget.py:157 `segment=numeric.MAX_BYTES`); the total is under MAX_ROSTER_BYTES (:158, :160)", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration :537 transport.numeric_survey_budget_payload(budget_payload) \u2192 :146-159; and SurveySamplingBudgetKernel.run :235 inside run_graph (:697).", + "classification": "segment", + "notes": "Comment at :45-51 states 76 B/row and 241,233,530 B \u2014 reproduced. graph_survey_budget.py:161 immediately re-decodes the fresh output (`numeric.decode_numeric_survey_bounds(output)`), so the consumer re-encoding row below runs on every production." + }, + { + "constant": "graph_survey_calibration.py:52 MAX_ROSTER_BYTES [numeric-bounds document total and consumer PAYLOAD]", + "value": "64 * MAX_BYTES = 4,294,967,296", + "enforced_at": [ + "graph_survey_budget.py:158 `maximum=numeric.MAX_ROSTER_BYTES` \u2192 graph_survey_population.py:332 and :337 \"TRANSPORT_ROSTER_LIMIT\" \u2014 SurveyPopulationGraphError", + "graph_survey_budget.py:160 `_require(len(output) <= numeric.MAX_ROSTER_BYTES, \"NUMERIC_LIMIT\")` \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_NUMERIC_LIMIT\")", + "graph_survey_calibration.py:113 decode_numeric_survey_bounds: `_require(type(payload) is bytes and 0 < len(payload) <= MAX_ROSTER_BYTES, \"PAYLOAD\")` \u2192 ValueError(\"SURVEY_CALIBRATION_PAYLOAD\"), then json.loads at :115 \u2014 reached at graph_survey_budget.py:161 and :236, survey_age_calibration.py:538 and :830, graph_survey_calibration.py:357 and :417 (kernel run), survey_calibration_diagnostics.py:109" + ], + "refusal_codes": [ + "TRANSPORT_ROSTER_LIMIT", + "SURVEY_BUDGET_TRANSPORT_NUMERIC_LIMIT", + "SURVEY_CALIBRATION_PAYLOAD" + ], + "encodes": "The whole numeric-bounds document (same bytes as the accumulation row), joined once and decoded whole by every consumer.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "24,123,542 (see the MAX_BYTES row)", + "bytes_at_full_source": "241,233,811 = 0.056 of 4 GiB (headroom 17.8\u00d7)", + "measurement_basis": "measure_numeric_bounds.py (same document; the join length equals the streamed total by graph_survey_population.py:337).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none \u2014 added on this branch (git diff: `+MAX_ROSTER_BYTES = 64 * MAX_BYTES` at :52); the base's ceiling on this document was MAX_BYTES (that row)", + "head_value": "4,294,967,296", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration :537 \u2192 graph_survey_budget :158/:160; :538/:830 and the kernel/diagnostics decodes \u2192 graph_survey_calibration :113.", + "classification": "stays", + "notes": "MAX_ROWS = MAX_ROSTER_BYTES // 128 (:53) derives from it (its own row). The document is decoded whole at least seven times per run (the decode sites listed), each holding the 241 MB bytes plus the json.loads object tree plus the canonical_json re-encoding (next row)." + }, + { + "constant": "graph_survey_calibration.py:53 MAX_ROWS", + "value": "MAX_ROSTER_BYTES // 128 = 33,554,432 (base: MAX_BYTES // 128 = 524,288)", + "enforced_at": [ + "graph_survey_calibration.py:127 `_require(type(ids) is list and 0 < len(ids) <= MAX_ROWS, \"ROW_COUNT\")` \u2192 ValueError(\"SURVEY_CALIBRATION_ROW_COUNT\")", + "graph_survey_budget.py:75-86 `and 0 < len(ids) <= numeric.MAX_ROWS` inside \"COMPLETE_CLONE_SHAPE\" \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_COMPLETE_CLONE_SHAPE\")" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_ROW_COUNT", + "SURVEY_BUDGET_TRANSPORT_COMPLETE_CLONE_SHAPE" + ], + "encodes": "A record cap on `household_ids` = clone household rows of the numeric-bounds document.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "n/a \u2014 a row count: 317,475 clone rows at 1/10 (0.61 of the base 524,288; 0.0095 of the head value)", + "bytes_at_full_source": "n/a \u2014 3,174,752 clone rows at full source (6.06\u00d7 the base 524,288; 0.095 of the head 33,554,432)", + "measurement_basis": "Counts from \u00a73 of the brief; ratios in measure_numeric_bounds.receipt.json.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "524,288 (MAX_BYTES // 128; git diff \u2212 side `MAX_ROWS = MAX_BYTES // 128`) \u2014 binds at full source (6.06\u00d7), not at 1/10", + "head_value": "33,554,432 (MAX_ROSTER_BYTES // 128) \u2014 binds at neither", + "reachable_from": "age", + "reach_chain": "\u2192 transport.numeric_survey_budget_payload :77 and every decode_numeric_survey_bounds :127 (survey_age_calibration.py:538/:830, kernel :357/:417, diagnostics :109, graph_survey_budget :161/:236).", + "classification": "not-a-transport", + "notes": "A row count in byte clothing: comment at :48-51 \u2014 'MAX_ROWS is the same //128 form the allocation pre-check uses (graph_survey_population.py); at 524,288 it refused a full-source clone of 3,174,752 rows before any byte did.' The row-count rule is the //128 structural pre-check of docs/us-native-scale-transport.md \u00a72d (ALLOCATION_ROSTER_BYTES // 128)." + }, + { + "constant": "graph_survey_calibration.py:119 canonical_json(document) == payload (cap: none in canonical_json; bounded upstream by :113 MAX_ROSTER_BYTES)", + "value": "none \u2014 microcosm.graph.canonical.canonical_json (canonical.py:43-58) is one json.dumps(...).encode() with no limit argument", + "enforced_at": [ + "graph_survey_calibration.py:119 decode_numeric_survey_bounds: `_require(canonical_json(document) == payload, \"CANONICAL\")` \u2192 ValueError(\"SURVEY_CALIBRATION_CANONICAL\"); the payload it compares against is \u2264 MAX_ROSTER_BYTES by :113" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_CANONICAL" + ], + "encodes": "A consumer-side whole re-encoding of the numeric-bounds document that graph_survey_budget produced in segments \u2014 a second whole copy of the roster document alongside the payload and its json.loads tree.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "24,123,542 (same bytes; canonical_json(json.loads(payload)) == payload verified for the invented 2002-group document)", + "bytes_at_full_source": "241,233,811 (a second copy at every decode; seven decodes per run)", + "measurement_basis": "measure_numeric_bounds.py: decode_numeric_survey_bounds run on the invented document (exercising :113, :119, :127, :143-146) and `canonical_json(json.loads(dec)) == dec` asserted.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (uncapped re-encoding; unchanged) \u2014 in the base the payload was capped at MAX_BYTES by :113, so the re-encoding was \u2264 64 MiB by construction", + "head_value": "none (uncapped); the payload cap is now MAX_ROSTER_BYTES 4 GiB, so this re-encoding may be up to 4 GiB", + "reachable_from": "age", + "reach_chain": "Every decode site: survey_age_calibration.py:538, :830; graph_survey_budget.py:161, :236; SurveyAgeCalibrationKernel.run graph_survey_calibration.py:357, :417; survey_calibration_diagnostics.py:109.", + "classification": "consumer", + "notes": "Takes the producer's total (MAX_ROSTER_BYTES). The analogous budget-document check at survey_origin_budget.py:934-937 was converted to a streamed _json_matches on this branch; this one was not." + }, + { + "constant": "graph_survey_calibration.py:88 32 (FLOAT_TOKEN)", + "value": "0 < len(v) <= 32 per hex-float token", + "enforced_at": [ + "graph_survey_calibration.py:86-98 _vector: `_require(all(type(v) is str and 0 < len(v) <= 32 for v in raw), \"FLOAT_TOKEN\")` \u2192 ValueError(\"SURVEY_CALIBRATION_FLOAT_TOKEN\"), then float.fromhex and the round-trip check :93-97 \"FLOAT_VALUES\"; called at :140-142 for group_upper_hex, incoming_hex, row_upper_hex and at graph_survey_budget.py:98-99 for each record's design_bound_float64_hex / upper_float64_hex" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_FLOAT_TOKEN", + "SURVEY_CALIBRATION_VECTOR_LENGTH", + "SURVEY_CALIBRATION_FLOAT_VALUES" + ], + "encodes": "A fixed-width token: one float64 as float.hex() (measured max 21 chars at these magnitudes; \u2264 24 for any finite float64).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "\u2264 24 B per token; independent of the roster", + "bytes_at_full_source": "\u2264 24 B per token", + "measurement_basis": "measure_numeric_bounds.receipt.json float_token_widths (max 21).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "32 (unchanged)", + "head_value": "32", + "reachable_from": "age", + "reach_chain": "\u2192 decode_numeric_survey_bounds :140-142 and numeric_survey_budget_payload :98-99.", + "classification": "must-not-move", + "notes": "Protects the encoder width of a float token." + }, + { + "constant": "graph_survey_calibration.py:123 256 (POPULATION)", + "value": "0 < len(document[\"population\"]) <= 256", + "enforced_at": [ + "graph_survey_calibration.py:122-125 `_require(type(document[\"population\"]) is str and 0 < len(document[\"population\"]) <= 256, \"POPULATION\")` \u2192 ValueError(\"SURVEY_CALIBRATION_POPULATION\")" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_POPULATION" + ], + "encodes": "A single scalar: the population version string (clone.COMBINED_CLONE_NODE, `combined_survey_puf_support_clone`, 33 chars).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "33 B at every fraction", + "bytes_at_full_source": "33 B", + "measurement_basis": "len(clone.COMBINED_CLONE_NODE) in measure_numeric_bounds.py's document.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "256 (unchanged)", + "head_value": "256", + "reachable_from": "age", + "reach_chain": "\u2192 decode_numeric_survey_bounds :123.", + "classification": "must-not-move", + "notes": "Token width." + }, + { + "constant": "graph_survey_calibration.py:80-82 64 (_digest)", + "value": "len(value) == 64 hex characters", + "enforced_at": [ + "graph_survey_calibration.py:78-83 _digest, used at :121 `_require(_digest(document[\"budget_sha256\"]), \"BUDGET_DIGEST\")`, graph_survey_budget.py:65 \"SOURCE_EDGE_DIGEST\" (preparation_sha256, allocation_sha256) and :166 \"BUDGET_DIGEST\"" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_BUDGET_DIGEST", + "SURVEY_BUDGET_TRANSPORT_SOURCE_EDGE_DIGEST", + "SURVEY_BUDGET_TRANSPORT_BUDGET_DIGEST" + ], + "encodes": "A single token: a sha256 hex digest.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 B at every fraction", + "bytes_at_full_source": "64 B", + "measurement_basis": "Fixed by sha256.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 (unchanged)", + "head_value": "64", + "reachable_from": "age", + "reach_chain": "\u2192 decode_numeric_survey_bounds :121; graph_survey_budget._document :64-65 and survey_sampling_budget_node :166.", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "graph_survey_calibration.py:274 65536 (ACTIVATION_TEXT)", + "value": "0 < len(value.encode()) <= 65536", + "enforced_at": [ + "graph_survey_calibration.py:273-274 calibration_activation_binding: `_require(type(value) is str and 0 < len(value.encode()) <= 65536, \"ACTIVATION_TEXT\")` \u2192 ValueError(\"SURVEY_CALIBRATION_ACTIVATION_TEXT\"); :277-280 `encoded = canonical_json(binding).decode()` / `_require(encoded == value, \"ACTIVATION_CANONICAL\")` (fixed-shape re-encoding equality)" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_ACTIVATION_TEXT", + "SURVEY_CALIBRATION_ACTIVATION_JSON", + "SURVEY_CALIBRATION_ACTIVATION_CANONICAL" + ], + "encodes": "A fixed-shape receipt: the survey age activation binding carried in node params \u2014 profile, the two fixed S0101 requests, two digests, 18 bands, convention text (survey_age_activation.py:47-75).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "3,766 B (canonical_json(activation_binding(SurveyAgeActivation('a'*64,'b'*64)))) at every fraction \u2014 5.7% of the cap", + "bytes_at_full_source": "3,766 B", + "measurement_basis": "Encoded through survey_age_activation.activation_binding and canonical_json in the final check step (see transcript); placeholder digests are the same width as real ones.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65536 (unchanged)", + "head_value": "65536", + "reachable_from": "age", + "reach_chain": "survey_age_calibration.run_survey_age_development \u2192 _run_survey_age_calibration :412 numerical.calibration_activation_binding(calibration.params); SurveyAgeCalibrationKernel.run :335-339. run_survey_age_calibration (invented registry) has no 'activation' param, so :274 is skipped (:271-272).", + "classification": "must-not-move", + "notes": "Does not move with the fraction; 18 bands are asserted by national_age_activation.py:309." + }, + { + "constant": "graph_survey_budget.py:132 16 (WEIGHT_BYTES)", + "value": "len(weight_bytes) == 16 hex characters (one float64)", + "enforced_at": [ + "graph_survey_budget.py:130-135 `_require(type(weight_bytes) is str and len(weight_bytes) == 16 and all(c in \"0123456789abcdef\" for c in weight_bytes), \"WEIGHT_BYTES\")` \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_WEIGHT_BYTES\"), then np.frombuffer(bytes.fromhex(...)) :136" + ], + "refusal_codes": [ + "SURVEY_BUDGET_TRANSPORT_WEIGHT_BYTES" + ], + "encodes": "A fixed-width field: each `incoming_clone_float64_bytes` entry of an origin record (survey_origin_budget.py:588-591 `clone_weights[...].tobytes().hex()`), two per group.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "16 B per token at every fraction", + "bytes_at_full_source": "16 B per token", + "measurement_basis": "Fixed by float64.tobytes().hex().", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "16 (unchanged)", + "head_value": "16", + "reachable_from": "age", + "reach_chain": "\u2192 transport.numeric_survey_budget_payload :112-138.", + "classification": "must-not-move", + "notes": "Encoder width; the producer's `original_design_float64_bytes` (survey_origin_budget.py:581-585) has the same 16-char shape but is not checked here." + }, + { + "constant": "graph_survey_budget.py:88 2**63 (HOUSEHOLD_IDS int64 range)", + "value": "-(2**63) <= i < 2**63 for every household id", + "enforced_at": [ + "graph_survey_budget.py:87-91 `_require(all(type(i) is int and -(2**63) <= i < 2**63 for i in ids) and len(set(ids)) == len(ids), \"HOUSEHOLD_IDS\")` \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_HOUSEHOLD_IDS\")", + "graph_survey_calibration.py:128-132 the same range plus strict ascending order, \"ORDERED_IDS\" \u2192 ValueError(\"SURVEY_CALIBRATION_ORDERED_IDS\")", + "graph_national_age_counts.py:340-363 _ids: 'must fit signed int64 exactly' (ValueError), and graph_survey_age_artifact.py:79-84 dtype ' 256 (POPULATION)", + "value": "0 < len(header[\"population\"]) <= 256", + "enforced_at": [ + "graph_survey_age_artifact.py:72-75 _validate: `_require(type(header[\"population\"]) is str and 0 < len(header[\"population\"]) <= 256, \"POPULATION\")` \u2192 ValueError(\"SURVEY_AGE_ARTIFACT_POPULATION\")" + ], + "refusal_codes": [ + "SURVEY_AGE_ARTIFACT_POPULATION" + ], + "encodes": "A single scalar: the population version string in the header (33 chars).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "33 B at every fraction", + "bytes_at_full_source": "33 B", + "measurement_basis": "len(clone.COMBINED_CLONE_NODE).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "256 (unchanged)", + "head_value": "256", + "reachable_from": "age", + "reach_chain": "\u2192 _validate from the kernel :207 and decode :128.", + "classification": "must-not-move", + "notes": "Token width." + }, + { + "constant": "survey_financial_successor.py:26 MAX_PAYLOAD_BYTES", + "value": "256 * 1024 = 262,144", + "enforced_at": [ + "survey_financial_successor.py:211-215 admit_survey_financial_population: `_require(candidate is None or (type(candidate) is bytes and len(candidate) <= MAX_PAYLOAD_BYTES), \"CANDIDATE_TYPE_OR_BOUND\")` \u2014 SurveyFinancialSuccessorError(ValueError) (:30-36)", + "survey_financial_successor.py:232 `_require(len(payload) <= MAX_PAYLOAD_BYTES, \"PAYLOAD_BOUND\")` after `payload = _document(...)`", + "survey_financial_successor.py:176 checked_view: `_require(_document(state, budget_view, run_view) == entry[1], \"RECONSTRUCTION\")` \u2014 whole re-encoding equality (fixed shape)", + "the encoder: survey_financial_successor.py:116 `budget._json({...})` = graph._bounded_json(value, survey_origin_budget.MAX_PAYLOAD_BYTES 64 MiB)" + ], + "refusal_codes": [ + "SURVEY_FINANCIAL_SUCCESSOR_CANDIDATE_TYPE_OR_BOUND", + "SURVEY_FINANCIAL_SUCCESSOR_PAYLOAD_BOUND", + "SURVEY_FINANCIAL_SUCCESSOR_RECONSTRUCTION" + ], + "encodes": "A fixed-shape receipt (:116-131): protocol, budget_sha256, financial_run_sha256, the nested `financial_run` document (`json.loads(state.run_entry[1])`, the checked financial run's receipt from graph_atomic_survey_financial \u2014 outside this family), previous/current version strings, two frame digests, owned_columns (runner.values.OUTPUTS), three booleans. No per-household member.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "No measured number in the permitted inputs; the document is O(graph nodes \u00d7 frame columns) through the nested run receipt and does not move with the fraction", + "bytes_at_full_source": "same as at 1/10 (fixed shape)", + "measurement_basis": "Read :116-131; no per-row field. Not measured (needs a live financial run).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "262,144 (unchanged; no diff)", + "head_value": "262,144", + "reachable_from": "none", + "reach_chain": "No caller of admit_survey_financial_population or verify_survey_financial_successor exists outside packages/microcosm-build/tests (grep of packages/ and tools/). survey_origin_budget imports the module only for the _live() contract (:251-256: PROTOCOL, FINANCIAL_SUCCESSOR_TYPE, MAX_PAYLOAD_BYTES) and accepts a SamplingOriginFinancialSuccessor as an optional previous_binding (:1143-1165), which survey_age_calibration never passes (:616-620). graph_atomic_survey_financial and graph_survey_completion_host do not import it. The function that would reach it: survey_financial_successor.admit_survey_financial_population.", + "classification": "must-not-move", + "notes": "The nested `financial_run` at :121 is bounded by the runner's own cap on that receipt, not here. Listed in survey_origin_budget's _live() contract, so moving it changes the budget producer seal." + }, + { + "constant": "survey_calibration_diagnostics.py:24 MAX_DIAGNOSTIC_BYTES", + "value": "8 * 1024**2 = 8,388,608", + "enforced_at": [ + "survey_calibration_diagnostics.py:83-86 validate_survey_calibration_diagnostics: `_require(type(payload) is bytes and 0 < len(payload) <= MAX_DIAGNOSTIC_BYTES, \"PAYLOAD_LIMIT\")` \u2192 ValueError(\"SURVEY_DIAGNOSTICS_PAYLOAD_LIMIT\") (:33-35)", + "survey_age_calibration.py:141-145 _load_diagnostics: `limit = diagnostic_check.MAX_DIAGNOSTIC_BYTES`; `_require(stat.S_ISREG(before.st_mode) and 0 < before.st_size <= limit, \"DIAGNOSTIC_STORE_LIMIT\")` \u2192 ValueError(\"SURVEY_AGE_RUN_DIAGNOSTIC_STORE_LIMIT\")", + "survey_age_calibration.py:153 `payload = stream.read(limit + 1)` (read cap) and :160-166 `len(payload) == record[\"size\"] <= limit and _sha(payload) == record[\"sha256\"]` \"DIAGNOSTIC_STORE_CHANGED\"", + "consumer re-encodings (canonical_json, no cap): survey_calibration_diagnostics.py:92 `_require(canonical_json(document) == payload, \"CANONICAL\")`, :194 `_require(canonical_json(expected) == payload, \"RECOMPUTED_VALUES\")`, survey_age_calibration.py:933-936 `canonical_json(result.diagnostics) == diagnostic_identity` \"FINAL_DIAGNOSTICS_VALUES\"" + ], + "refusal_codes": [ + "SURVEY_DIAGNOSTICS_PAYLOAD_LIMIT", + "SURVEY_AGE_RUN_DIAGNOSTIC_STORE_LIMIT", + "SURVEY_AGE_RUN_DIAGNOSTIC_STORE_CHANGED", + "SURVEY_DIAGNOSTICS_CANONICAL", + "SURVEY_DIAGNOSTICS_RECOMPUTED_VALUES", + "SURVEY_AGE_RUN_FINAL_DIAGNOSTICS_VALUES" + ], + "encodes": "The schema-8 calibration diagnostics artifact (microcosm.calibrate.diagnostics.diagnostics_payload :343-431): one row per target (18 age bands), `loss_trajectory` (one float per epoch; epochs \u2264 1000 by graph_survey_calibration.py:218), options (grouped_upper_bounds diagnostics = four scalars, group_bounds.py:178-184; `target_records: None` :163), target_surface and past_cap_census (scalars; `n_records` is a scalar :409 / diagnostics.py:190), skipped, build anchors. No per-household row.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "No diagnostics artifact among the permitted inputs; grows with targets (18) and epochs (\u2264 1000 floats), not with the roster", + "bytes_at_full_source": "same shape as at 1/10", + "measurement_basis": "Read diagnostics_payload :343-431 and survey_calibration_diagnostics :138-187; not measured.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "8,388,608 (unchanged)", + "head_value": "8,388,608", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration :760 _load_diagnostics \u2192 :785 diagnostic_check.validate_survey_calibration_diagnostics \u2192 :83-86, :92, :194; :933-936.", + "classification": "must-not-move", + "notes": "The producer graph_survey_calibration.py:402-404 `payload = canonical_json(diagnostics_payload(...))` has no cap; the 8 MiB is enforced only on the consumer side. `_history` :50 `len(trajectory) == epochs` and graph_survey_calibration.py:218 `1 <= epochs <= 1000` cap the trajectory record count \u2014 a solver parameter, not recorded as a byte row." + }, + { + "constant": "demographic_calibration_graph.py:146 1_048_576 (registry declaration)", + "value": "len(text.encode()) > 1_048_576 refuses (1 MiB)", + "enforced_at": [ + "demographic_calibration_graph.py:145-147 _registry_from_json: `if not isinstance(text, str) or len(text.encode()) > 1_048_576: raise ValueError(\"A bounded canonical registry declaration is required.\")`; :160-163 `if text != _registry_json(registry): raise ValueError(\"Registry declaration must have its canonical typed representation.\")` (whole re-encoding equality, encoder _json :76-79 has no cap); reached from graph_survey_calibration.py:169 (survey_age_calibration_node), :332 (SurveyAgeCalibrationKernel.run), survey_age_calibration.py:784, survey_age_activation.py:101 (validate_survey_age_registry), demographic_calibration_graph.py:220" + ], + "refusal_codes": [ + "ValueError(\"A bounded canonical registry declaration is required.\")", + "ValueError(\"Registry declaration must have its canonical typed representation.\")" + ], + "encodes": "A fixed-shape declaration carried in node params: `{\"country\": ..., \"specs\": [asdict(spec) ...]}` for the 18 demographic targets (one per band, national_age_activation.py:309), each with metadata and a hierarchy dict.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "Not measured (needs a TargetRegistry; no I/O-free factory found in national_age_activation); 18 specs at every fraction", + "bytes_at_full_source": "same as at 1/10", + "measurement_basis": "Read :82-85 and :145-165; _validate_registry :88-142 fixes the spec shape.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1_048_576 (unchanged)", + "head_value": "1_048_576", + "reachable_from": "age", + "reach_chain": "survey_age_calibration.run_survey_age_calibration \u2192 numerical.survey_age_calibration_node :169 \u2192 demographic._registry_from_json; kernel run :332; :784.", + "classification": "must-not-move", + "notes": "Does not move with the fraction. DemographicCalibrationKernel.run (:278-351, with its uncapped `_json(diagnostics_payload(...))` at :330) is the national kernel and is not reached by survey_age_calibration (only its implementation_hash is, graph_survey_calibration.py:324)." + }, + { + "constant": "graph_national_age_counts.py:137 65_536 (age-band declaration)", + "value": "len(text.encode()) > 65_536 refuses", + "enforced_at": [ + "graph_national_age_counts.py:137-138 _bands_from_param: `if not isinstance(text, str) or len(text.encode()) > 65_536: raise ValueError(\"A bounded canonical age-band declaration is required.\")`; :165-166 canonical re-encoding equality `if _canonical([[c, lo, hi] for c, lo, hi in bands]) != text: raise ValueError(...)` (encoder _canonical :121-122 has no cap); reached via _node_from_schema :173 (graph_survey_age_artifact.survey_age_count_artifact_node :139 \u2192 national_age_count_node :209-211) and NationalAgeCountKernel.run :253 (called from SurveyAgeCountArtifactKernel.run graph_survey_age_artifact.py:181)" + ], + "refusal_codes": [ + "ValueError(\"A bounded canonical age-band declaration is required.\")", + "ValueError(\"The age-band declaration must be in canonical form.\")" + ], + "encodes": "A fixed-shape declaration: the canonical band schema `[[column, low, high], ...]` for the 18 activated bands.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "483 B (`_bands_param(NATIONAL_AGE_ACTIVATION.bands)`) at every fraction \u2014 0.74% of the cap", + "bytes_at_full_source": "483 B", + "measurement_basis": "Encoded through graph_national_age_counts._bands_param in the final check step.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65_536 (unchanged)", + "head_value": "65_536", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._measurement :102 ages.survey_age_count_artifact_node \u2192 :139 \u2192 _node_from_schema :173; kernel run :253.", + "classification": "must-not-move", + "notes": "MAX_SUPPORTED_AGE = 120 (:95) is a value bound on ages, not a byte bound, and is not recorded as a row. The count body `np.bincount(..., minlength=len(bands) * len(index))` (:302-306) allocates 18 \u00d7 households \u00d7 8 B (457 MB at full-source clone households) with no ceiling \u2014 a memory allocation with no document behind it, not a bound." + }, + { + "constant": "survey_age_sources.py:19 MAX_RESPONSE_BYTES", + "value": "8 * 1024**2 = 8,388,608", + "enforced_at": [ + "survey_age_sources.py:128 capture_survey_age_response: `_require(type(payload) is bytes and 0 < len(payload) <= MAX_RESPONSE_BYTES, \"SIZE\")` \u2192 ValueError(\"SURVEY_AGE_SOURCE_SIZE\") (:36-38)", + "survey_age_sources.py:89-109 _read(path, limit): :98 `_require(0 < before.st_size <= limit, \"SIZE\")`, :99 `payload = stream.read(limit + 1)` (read cap), :101-106 \"CHANGED_FILE\" \u2014 with limit = MAX_RESPONSE_BYTES at :144 (_write \u2192 _read, \"IMMUTABLE\" :115) and :175 (the raw response read)", + "survey_age_sources.py:170-174 read_survey_age_response: `_require(type(descriptor[\"size_bytes\"]) is int and 0 < descriptor[\"size_bytes\"] <= MAX_RESPONSE_BYTES, \"SIZE\")`; :176-180 \"RESPONSE_PIN\"" + ], + "refusal_codes": [ + "SURVEY_AGE_SOURCE_SIZE", + "SURVEY_AGE_SOURCE_CHANGED_FILE", + "SURVEY_AGE_SOURCE_IMMUTABLE", + "SURVEY_AGE_SOURCE_RESPONSE_PIN", + "SURVEY_AGE_SOURCE_FILE" + ], + "encodes": "An upstream file: the captured Census API S0101 national response (`for=us:*`, one geography row of group(S0101); metadata and data kinds, :41-57) \u2014 identical at every survey fraction.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "No captured response among the permitted inputs; the request is one national row, so its size does not depend on the survey selection", + "bytes_at_full_source": "same file", + "measurement_basis": "Read :41-57 (the fixed request surface) and :149-181.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "8,388,608 (unchanged)", + "head_value": "8,388,608", + "reachable_from": "age", + "reach_chain": "survey_age_calibration.run_survey_age_development \u2192 age_activation.activate_survey_age_targets (survey_age_activation.py:117-125: read_survey_age_response \u00d72) and :860 age_activation.verify_survey_age_targets (survey_age_activation.py:209-219). run_survey_age_calibration (invented registry) never reaches it. capture_survey_age_response is capture-time only (no runner caller).", + "classification": "must-not-move", + "notes": "Protects an upstream file's real size. strict_json (cd_reference_sources.py:51-66) applied at :129/:158/:181 has no size bound of its own." + }, + { + "constant": "survey_age_sources.py:20 MAX_DESCRIPTOR_BYTES", + "value": "4096", + "enforced_at": [ + "survey_age_sources.py:143 `_require(_read(descriptor_path, MAX_DESCRIPTOR_BYTES) == encoded, \"IMMUTABLE\")`", + "survey_age_sources.py:145 `_write(descriptor_path, encoded, MAX_DESCRIPTOR_BYTES)` (\u2192 _read :98-99 SIZE / read(limit + 1))", + "survey_age_sources.py:158-159 `descriptor = strict_json(_read(root / \"requests\" / f\"{key}.json\", MAX_DESCRIPTOR_BYTES))`" + ], + "refusal_codes": [ + "SURVEY_AGE_SOURCE_SIZE", + "SURVEY_AGE_SOURCE_IMMUTABLE", + "SURVEY_AGE_SOURCE_DESCRIPTOR" + ], + "encodes": "A fixed-shape descriptor: the nine _DESCRIPTOR_FIELDS (table, year, dataset, geography, kind, url, sha256, size_bytes, path), canonical_json.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "a few hundred bytes (nine short fields; not measured) at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "Read :21-33 and :132-146.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4096 (unchanged)", + "head_value": "4096", + "reachable_from": "age", + "reach_chain": "As for MAX_RESPONSE_BYTES (read_survey_age_response :158-159 via activate_survey_age_targets / verify_survey_age_targets).", + "classification": "must-not-move", + "notes": "Fixed-shape." + }, + { + "constant": "survey_age_sources.py:70-77 64 (_digest PIN)", + "value": "len(value) == 64 hex characters", + "enforced_at": [ + "survey_age_sources.py:70-77 _digest: `_require(type(value) is str and len(value) == 64 and all(c in \"0123456789abcdef\" for c in value), \"PIN\")`; used at :154 and by survey_age_activation.py:19/:43-44 (SurveyAgeActivation.__post_init__)" + ], + "refusal_codes": [ + "SURVEY_AGE_SOURCE_PIN" + ], + "encodes": "A single token: a sha256 content pin.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 B at every fraction", + "bytes_at_full_source": "64 B", + "measurement_basis": "Fixed by sha256.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 (unchanged)", + "head_value": "64", + "reachable_from": "age", + "reach_chain": "run_survey_age_development \u2192 activation_binding \u2192 SurveyAgeActivation.__post_init__ \u2192 _digest; read_survey_age_response :154.", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "cd_reference_sources.py:15 MAX_RESPONSE_BYTES", + "value": "32 * 1024 * 1024 = 33,554,432", + "enforced_at": [ + "cd_reference_sources.py:86 fetch_census: `payload = response.read(MAX_RESPONSE_BYTES + 1)` (read cap)", + "cd_reference_sources.py:91-92 `if len(payload) > MAX_RESPONSE_BYTES: raise ValueError(\"Census response exceeds size limit\")`" + ], + "refusal_codes": [ + "ValueError(\"Census response exceeds size limit\")" + ], + "encodes": "An upstream file: a Census API response for one of the eight declared requests (four tables \u00d7 metadata/data at congressional-district level, :18-48).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "No captured response among the permitted inputs; independent of the survey fraction", + "bytes_at_full_source": "same file", + "measurement_basis": "Read :18-48 and :74-93.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "33,554,432 (unchanged)", + "head_value": "33,554,432", + "reachable_from": "none", + "reach_chain": "fetch_census is reached only through capture_request (:166), called from cd_reference.py:458 (a CLI capture tool); none of the three runners reaches it. The age lane imports only strict_json (:51-66) from this module (survey_age_sources.py:17; national_age_activation.py:58), which has no size bound. The function that would reach it: cd_reference_sources.capture_request / fetch_census.", + "classification": "must-not-move", + "notes": "The cached-replay branch of capture_request (:145-163) reads the raw file with `(root / expected_path).read_bytes()` (:156) and checks `len(payload) != descriptor[\"size_bytes\"]` \u2014 no cap on that read. write_json (:115-128) has no cap (fixed descriptor)." + }, + { + "constant": "group_bounds.py:77-81 json.dumps(payload, sort_keys=True, separators=(\",\", \":\"), ensure_ascii=True).encode() (cap: none)", + "value": "none \u2014 no limit argument; the digest payload is a whole-roster document", + "enforced_at": [ + "group_bounds.py:71-81 GroupedUpperBounds.__post_init__: `payload = {\"contract\": ..., \"household_ids\": ids, \"group_indices\": groups.tolist(), \"absolute_bounds_hex\": [float(x).hex() for x in bounds]}`; `digest = hashlib.sha256(json.dumps(...).encode()).hexdigest()` \u2014 materialised whole on every construction: survey_origin_budget.py:669 (every _document), :918-922 (every checked_view), graph_survey_calibration.py:143 (every decode); graph_fiscal_dense_calibration.py:140 is off-path" + ], + "refusal_codes": [ + "none at this site (ValueError from the shape checks at :41-56 only)" + ], + "encodes": "The producer-side encoding of the constraint digest payload: every clone household id (tuple), every group index, one hex float per group. survey_origin_budget.py:865-875 re-encodes the same payload as a consumer (its own row).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "7,761,306 = 48.9 B/group \u00d7 158,737 + 99 B", + "bytes_at_full_source": "77,612,467 = 48.9 B/group \u00d7 1,587,376 + 99 B (1.16\u00d7 64 MiB for scale; nothing refuses)", + "measurement_basis": "measure_constraint_digest_and_selection.py: the exact json.dumps call over the :71-76 payload at full-source widths, difference of 2 vs 2002 groups; the replica sha256 asserted equal to GroupedUpperBounds.constraint_digest at both sizes.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (uncapped; unchanged \u2014 no diff)", + "head_value": "none (uncapped)", + "reachable_from": "age", + "reach_chain": "survey_age_calibration \u2192 freeze_survey_origin_budget \u2192 _document :669; checked_view :918; decode_numeric_survey_bounds graph_survey_calibration.py:143 (seven decodes per run); calibrate(... grouped_upper_bounds=bounds.grouped ...) graph_survey_calibration.py:383 consumes the object.", + "classification": "stays", + "notes": "Also materialised per construction: `groups.tolist()` (:74), one immutable numpy array per group in `_members` (:64-70; 1,587,376 small arrays at full source) and the `ids` tuple (:40). These are memory, not bounds. `check`/`totals` (:109-131) do a Python-level math.fsum per group." + } + ], + "open_questions": [ + "Row keying: the brief keys one row per constant definition, but survey_origin_budget.MAX_PAYLOAD_BYTES (:54) and MAX_ROSTER_BYTES (:63) each sit in front of several different documents with different shapes and verdicts (the budget accumulation, the successor document, the geography-binding population, the selection digest, the consumer re-encoding). I split them into per-document rows with the constant name repeated and a bracketed qualifier so each verification agent gets one document; a reviewer may prefer to merge.", + "Classification set: no bucket names a fixed-shape document cap (financial successor 256 KiB, diagnostics 8 MiB, registry 1 MiB, bands 64 KiB, activation 64 KiB, descriptor 4 KiB) or an uncapped roster re-encoding (the two constraint-digest json.dumps sites). I used must-not-move for the fixed-shape caps (with the reason) and stays with 'cap: none' for the uncapped re-encodings.", + "The FINAL_BUDGET_VIEW_DOCUMENT conversion to graph._json_matches (survey_origin_budget.py:927-937) is an UNCOMMITTED working-tree modification at the head I read (git status: M; git diff vs HEAD 77bafbb3a contains only that hunk). HEAD still has `_json(document) == entry[1]` under 64 MiB, which binds at 1/10 and at full source. The row records both; whoever commits should confirm it lands.", + "byte-census-measurements.json age_artifact encoded the header with age_convention 'integer_years_completed'; this head's graph_national_age_counts.SUPPORTED_AGE_CONVENTION is 'observed_interview_age_completed_years', so the header is 549 B not 534 B and the artifact 482,562,866 B not 482,562,851 B (15 B). Row and body figures are identical; the row uses the head figure.", + "The geography-binding semantic_population (50.8 MB at full source under 64 MiB, headroom 1.32\u00d7) is the tightest non-binding roster document in this family and is a single encoder piece; it is reachable only when a geography_config is supplied to the age runner. Its household count assumes value.population is the postclone (clone-roster) population per the module docstring (:7-8) and the stage bindings (:324-328); a verifier should confirm that against survey_atomic_geography.reconstruct_atomic_survey_geography, outside this family.", + "The registry declaration (demographic_calibration_graph.py:146, 1 MiB) and the diagnostics artifact (8 MiB) were not measured: no I/O-free registry factory exists in national_age_activation and no diagnostics artifact is among the permitted inputs. Both are fixed-shape by reading the code.", + "Household-id widths at full source are assumed 7-digit (as in the existing measurement); the 1/1000 receipt shows non-sequential ids (ACS from 1, ASEC up to 55,002 at 1/1000), so the clone-id scheme at full source is not observed. The measured average digit count over 0..1,587,375 is 6.30 and over 1..3,174,752 is 6.65, so the true per-group cost of the budget document is ~2 B below 742.", + "graph_survey_calibration.py:218 `1 <= epochs <= 1000` caps the loss_trajectory record count of the diagnostics artifact (survey_calibration_diagnostics.py:50); it is a solver parameter and is noted under MAX_DIAGNOSTIC_BYTES rather than given its own row.", + "survey_age_calibration.py defines no byte bound of its own; it appears only as enforcement sites (MAX_DIAGNOSTIC_BYTES at :141-166; CANDIDATE_LIMIT via :664) and as the reach chain for every row marked age.", + "The brief's `_recovered/pilot-runs/.../preparation.json` path is relative; the file lives at /Users/maxghenis/PolicyEngine/_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json (outside the worktree) and was read by absolute path as a \u00a74 input. No other recovered artifact was read." + ] + }, + { + "family": "ASEC native population, catalogue and prepared source", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py (full; no byte bound found: schema/identity validation only, HDF read via load_frame_checkpoint)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py (full; no byte bound found: _hash_file streams via hashlib.file_digest, _read_numeric_household reads selected int64 blocks whole with no cap)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py (full; no own constant; enforces asec_housing_status.HEADER_MAX_BYTES via _parse at :287)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py (full; no own constant; enforces asec_housing_universe.HEADER_MAX_BYTES via _parse at :143 and :221)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_original_household_weights.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py (full; no module-level byte constant; its canonical_json re-encodings at :189, :407-409, :651-652 are on digest-shaped documents)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py (full; no byte constant; re-encodings at :159 and :243 are on digest-shaped documents)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_coverage_fields.py (lines 150-320, for the nested field-record shape and its enforcement sites at :209, :265-266, :270, :403)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py (lines 495-520, 540-570, 1150-1185: _file_limits, _source_files enforcement, native receipt consumer)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py (lines 230-330: _Kernel.implementation_hash and _qualified)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py (lines 180-260: qualify_current_asec_demographics)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py (grep: :85 demographics call, :359 demographic_conditioning gate)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py (grep: :47-49 HEADER_MAX_BYTES/MAX_PERSONS/MAX_HOUSEHOLDS, :77-97 _json/_parse)", + "packages/microcosm-build/src/microcosm/build/us_runtime/education_assistance_source.py (grep: :112/:134/:156 member_size_bytes pins, :117/:139/:161 rows)", + "packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py (grep: :127 MAX_SOURCE_BYTES, :146/:156/:166 size_bytes pins)", + "packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py (grep: :183-186 origin_key is a sha256 hex)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py (lines 104-160: _row_identity and encode_us_frame_context are digest-shaped)", + "packages/microcosm-build/src/microcosm/build/frame_sampling.py (grep: :313 receipt carries selected_household_ids_sha256, not ids)", + "packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py (grep: :558-569 receipt carries digests only)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_domains.py (grep: dataclass constructors)", + "_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/graph.json (node ids; survey_predictors params demographic_conditioning=true)", + "_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json (structure and id widths only)" + ], + "rows": [ + { + "constant": "asec_2024_native_population.py:41 _MAX_EVIDENCE_BYTES", + "value": "128 * 1024**2 = 134,217,728", + "enforced_at": [ + "asec_2024_native_population.py:65 (_encode: `_require(len(result) + len(part) <= _MAX_EVIDENCE_BYTES, \"EVIDENCE_SIZE\")` charged per iterencode part; AsecNativePopulationError)", + "asec_2024_native_population.py:141-145 (`len(candidate) <= _MAX_EVIDENCE_BYTES`, \"CANDIDATE\"; AsecNativePopulationError)", + "roster document encoded under it: asec_2024_native_population.py:760-795 (the population payload); fixed-shape documents under the same encoder: :223-230 _frame_identity, :409/:422/:528/:701 _implementation()" + ], + "refusal_codes": [ + "EVIDENCE_SIZE", + "CANDIDATE" + ], + "encodes": "The issued 2024 native-population payload: `households` = one row per selected 2024 ASEC household (:336-347), each nesting the whole hhpub anchor record as `source` (asec_original_household_weights._record_fields :274-316 plus the pins added at :364-373) and the whole household-fields record as `household_fields` (asec_household_coverage_fields._project :207-254 plus pins); `persons` = one 9-key dict per selected ASEC person (`projection.loc[mask].to_dict(orient=\"records\")` :348 over asec_coverage_authentication.COLUMNS); `selection.keys` = one [2024, H_SEQ] per household; plus a fixed preamble (coverage receipt header ~12.5 kB, parent identity, digests).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "12,256,009 (5,576 households + 14,212 persons)", + "bytes_at_full_source": "122,530,878 (55,762 households + 142,125 persons); headroom 1.095x", + "measurement_basis": "Encoded through asec_2024_native_population._encode on invented rows at full-source widths (6-digit parent receiving ids, 5-digit H_SEQ, 22-digit PERIDNUM, 64-hex digests, PRPERTYP \"2\"/\"observed_code\", a representative coverage header built as _reconstruct :673-702 with the real _implementation() = 12,454 B via _bounded_header, a 30-key parent-identity placeholder). Difference of two counts (1,000 hh/2,549 p -> 2,209,446 B; 2,000/5,097 -> 4,404,786 B) gives 2,195.34 B per household-with-2.5487-persons; a household row is 1,667 B incl. comma and selection key, a person row 205 B, preamble 16,118 B; then the whole document was encoded at both census counts (tenth 0.28 s, full 2.85 s, no refusal). Receipt: scratchpad/census/asec/asec-byte-census.json#native_population_payload.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "134,217,728", + "head_value": "134,217,728 (module unchanged vs origin/native-row-ceilings)", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial -> survey_population.create -> graph_survey_population create kernel -> survey_population_preparation.py:1965 asec_native.load_authenticated_asec_2024_native_population(**kwargs, selected_households=asec_keys) -> :760 _encode(payload); re-borrowed by replay/verification through _checked (:614-632, whole-payload equality) and consumed whole by survey_population_preparation.py:1160-1162 and current_asec_demographics.py:216/240 via json.loads (also reachable from 45, which contains the nineteen).", + "classification": "stays", + "notes": "Headroom at full source is only 9.5%. The width comes from :344-345 `\"source\": row, \"household_fields\": field_rows[key[1]]` nesting two full ~700 B records into every household row (1,667 B per household vs 205 B per person). The row-ceilings census estimated ~117 MB; this head measures 122.5 MB through the real encoder. The 1/10 figure is 12.3 MB. Three consumers materialise the whole payload again: `receipt` (:660-661 json.loads), survey_population_preparation.py:1162 (`asec_rows = {row[\"person_id\"]: row for row in native_receipts[\"asec\"][\"persons\"]}`), current_asec_demographics.py:240 (dict of all household rows). `_frame_identity` also copies the weights vector (`vector.values.tobytes()` :219) under the same encoder but that document is fixed-shape." + }, + { + "constant": "asec_2024_native_population.py:42 _MAX_CHECKPOINT_BYTES", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "asec_2024_native_population.py:162-165 (_file_identity: `stat.S_ISREG(before.st_mode) and 0 < before.st_size <= maximum`, \"SOURCE_FILE_BOUNDS\"), with this maximum supplied for P/H/T at :720-732 and rechecked on every borrow at :418-419", + "asec_2024_native_population.py:168-170 (read cap `os.read(descriptor, min(1024**2, before.st_size - count + 1))`, `count <= before.st_size` \"SOURCE_FILE_CHANGED\")", + "asec_population_catalogue.py:593-611 (`native._MAX_CHECKPOINT_BYTES` for the same three files; enforced via native._file_identity at :454)", + "survey_population_preparation.py:506 (_file_limits gives every _SOURCE_ROSTER entry this cap; enforced at :558 `_regular_reader(path, limits[name])` (:425 `0 <= before.st_size <= maximum`) and :561 `_require(size <= limits[name], \"SOURCE_FILE_LIMIT\")`, SurveyPopulationPreparationError)" + ], + "refusal_codes": [ + "SOURCE_FILE_BOUNDS", + "SOURCE_FILE_CHANGED", + "SOURCE_FILE_LIMIT" + ], + "encodes": "An upstream file: each of the three ASEC checkpoint HDFs (raw-stage parent P, household attachment H, restored person-income attachment T), hashed block by block.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: T = 666,333,922 (row-ceilings census, measured on the recovered restoration checkpoint); P and H not established", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Not re-measured (gated data); the T size is cited from experiments/native-row-ceilings/census.json (`_ATTACHMENT_MAX_BYTES` row). Fraction-independent by construction: the files are the whole three-cohort source.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "8,589,934,592", + "head_value": "8,589,934,592 (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> survey_population_preparation:1965 -> load_authenticated_asec_2024_native_population -> _validate_state :418-419 -> _file_identity; and :1942 issue_asec_source_catalogue -> _validate_state :453-454.", + "classification": "must-not-move", + "notes": "Protects an upstream file's real size; the 1 MiB `os.read` chunk at :168 is a chunk width, not a document bound." + }, + { + "constant": "education_assistance_source.py:112/134/156 member_size_bytes (pppub23 281,065,733; pppub24 277,250,415; pppub25 277,882,549), carried as _MEMBER_PINS in asec_coverage_authentication.py:34-37, asec_person_income_source.py:35-38, asec_demographic_source.py:388-391, asec_student_controls.py:46-49, asec_income_observations.py:53-56", + "value": "exact pins: 281,065,733 / 277,250,415 / 277,882,549", + "enforced_at": [ + "asec_coverage_authentication.py:323 (`before.st_size == size`, \"COVERAGE_SOURCE_SIZE\"), :329-331 (read cap `src.read(min(_CHUNK, size - count + 1))`, `count <= size` \"COVERAGE_SOURCE_SIZE\"), :337-340 (\"COVERAGE_SOURCE_CHANGED\"); AsecCoverageAuthenticationError", + "asec_2024_native_population.py:734-742 (`source[\"member_bytes\"]` passed as both maximum and size to _file_identity -> :163 \"SOURCE_FILE_BOUNDS\", :166 \"SOURCE_FILE_SIZE\")", + "asec_population_catalogue.py:613-622 (same, via native._file_identity)", + "asec_person_income_source.py:83 (`path.stat().st_size == size`, \"MEMBER_SIZE\", MoneyRefusalError)", + "asec_student_controls.py:103 (_snapshot `before.st_size == size` \"STUDENT_SOURCE_SIZE\"), :109 (read cap `src.read(min(1024 * 1024, size - count + 1))`), :111, :116-120 (\"STUDENT_SOURCE_CHANGED\"); reused by asec_demographic_source.py:1189 (\"DEMOGRAPHIC_SOURCE_BYTES\") and :537 (\"MEMBER_SIZE\"), asec_income_observations.py:196/:403 (_capture), asec_student_controls.py:254" + ], + "refusal_codes": [ + "COVERAGE_SOURCE_SIZE", + "COVERAGE_SOURCE_CHANGED", + "SOURCE_FILE_BOUNDS", + "SOURCE_FILE_SIZE", + "MEMBER_SIZE", + "STUDENT_SOURCE_SIZE", + "STUDENT_SOURCE_CHANGED", + "DEMOGRAPHIC_SOURCE_BYTES" + ], + "encodes": "An upstream file: one official pppub CSV member per cohort, captured whole to a private temp copy and hashed.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 281,065,733 / 277,250,415 / 277,882,549 (pinned)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "The pins themselves (education_assistance_source.py:112/134/156); no gated read.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "exact pins (unchanged)", + "head_value": "exact pins (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> load_authenticated_asec_2024_native_population -> restoration.load_authenticated_restored_current_money_source (:702) and coverage_owner.authenticate_asec_coverage (:706) -> _reconstruct -> _capture; and survey_predictors.* -> current_asec_demographics:222 -> asec_demographic_source._reconstruct :1186-1192 -> _snapshot.", + "classification": "must-not-move", + "notes": "Definition lives outside this family's module list (education_assistance_source.py); recorded here so the family's enforcement sites are not lost \u2014 merge on the definition key if another family lists it. Every capture writes a whole private copy of the ~278 MB member to a temp directory (three members per reconstruction, repeated by the coverage, restoration, student, income and demographic owners)." + }, + { + "constant": "native_household_origin.py:146/156/166 size_bytes (hhpub23 30,259,450; hhpub24 30,448,089; hhpub25 33,298,479), the _ASEC_MEMBER_PINS read by asec_original_household_weights._registry :166-185", + "value": "exact pins: 30,259,450 / 30,448,089 / 33,298,479", + "enforced_at": [ + "asec_original_household_weights.py:172-177 (`pin.size_bytes <= _MAX_MEMBER_BYTES`, \"MEMBER_REGISTRY\")", + "asec_original_household_weights.py:336 (`before.st_size == pin.size_bytes`, \"CAPTURE_SIZE\"), :246-250 (_DigestReader.readinto cap `min(len(buffer), self.size - self.count + 1)`, `self.count <= self.size` \"CAPTURE_SIZE\"), :376-380 (\"CAPTURE_CHANGED\"); HouseholdWeightSourceError", + "asec_original_household_weights.py:485-486 (`_capture_owner._snapshot(paths[...], capture, size=pin.size_bytes)` -> asec_student_controls.py:103/109/111)", + "asec_household_coverage_fields.py:265-266, :306, :425 (same pattern for the field owner)", + "asec_2024_native_population.py:743-746 (`pin[\"size_bytes\"]` as maximum and size to _file_identity :163/:166); asec_population_catalogue.py:623-626", + "native_household_origin.py:126-127 (pin dataclass: `0 < self.size_bytes <= MAX_SOURCE_BYTES`; outside this family)" + ], + "refusal_codes": [ + "MEMBER_REGISTRY", + "CAPTURE_SIZE", + "CAPTURE_CHANGED", + "STUDENT_SOURCE_SIZE", + "SOURCE_FILE_BOUNDS", + "SOURCE_FILE_SIZE" + ], + "encodes": "An upstream file: the hhpub household CSV member; the 19-path reads only hhpub25 (`{2024: member}` at asec_2024_native_population.py:709-712).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 33,298,479 (hhpub25)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "The pins themselves (native_household_origin.py:166-167); no gated read.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "exact pins (unchanged)", + "head_value": "exact pins (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> load_authenticated_asec_2024_native_population :709 -> anchor_owner.load_authenticated_asec_household_weights -> _registry / _snapshot / _read_capture; also :710 field_owner.", + "classification": "must-not-move", + "notes": "Definition outside the family list (native_household_origin.py); merge on the definition key." + }, + { + "constant": "asec_population_catalogue.py:26 _MAX_ENVELOPE_BYTES", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "asec_population_catalogue.py:53 (_encode: `_require(len(result) + len(part) <= _MAX_ENVELOPE_BYTES, \"ENVELOPE_SIZE\")` per iterencode part; AsecSourceCatalogueError)", + "asec_population_catalogue.py:556-560 (`len(candidate) <= _MAX_ENVELOPE_BYTES`, \"CANDIDATE_SIZE_OR_TYPE\")", + "documents under this encoder: :172 one catalogue record `_encode([kind, _record(row)])`; :333 one bound record `_encode([kind, row])`; :82-89 _source_authority(); :440/:455/:573 _implementation(); :644-679 the catalogue receipt payload" + ], + "refusal_codes": [ + "ENVELOPE_SIZE", + "CANDIDATE_SIZE_OR_TYPE" + ], + "encodes": "A single record (one ASEC household with its nested persons, one ledger record, one native household row nesting the anchor and field records, or one person row) or the fixed catalogue receipt; never a whole roster (:45-46: \"The complete catalogue is hashed incrementally, never encoded as another body\").", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: largest household catalogue record with 16 persons 1,476 B (:172 form); largest bound household row 1,671 B (:333 form); catalogue receipt ~4,539 B", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Measured through asec_population_catalogue._encode on a 16-person invented household in both record forms and on a receipt shaped as :644-679 (receipt #catalogue_records_stream).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,048,576", + "head_value": "1,048,576 (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> survey_population_preparation.py:1942 issue_asec_source_catalogue -> :589 _records_identity / :659 _bound_records_identity / :644 payload.", + "classification": "must-not-move", + "notes": "A per-record envelope width; grows only with persons per household (~76 B per nested person), not with the selection fraction." + }, + { + "constant": "asec_population_catalogue.py:27 _MAX_RECORD_BYTES", + "value": "128 * 1024**2 = 134,217,728", + "enforced_at": [ + "asec_population_catalogue.py:174 (_records_identity: `size += len(data); _require(size <= _MAX_RECORD_BYTES, \"RECORD_SIZE\")` over every catalogue record; AsecSourceCatalogueError)", + "asec_population_catalogue.py:335 (_bound_records_identity: same accumulation over every native household row and person row of the complete 2024 cohort)" + ], + "refusal_codes": [ + "RECORD_SIZE" + ], + "encodes": "Two whole-cohort canonical streams, both fraction-independent because :587 calls `native._roster(parent, coverage, anchors, fields, None)` (the complete 2024 cohort, `selected=None`): (a) :159-194 the catalogue \u2014 55,762 household records with 142,125 nested persons plus 33,170 unrepresented-household ledger records; (b) :328-338 the native binding \u2014 the same 55,762 household rows (each nesting the anchor and field records) and 142,125 person rows that the population payload carries at full source, as `_encode([kind, row]) + b\"\\n\"`.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: (a) 26,535,585; (b) 124,133,900 (92.5% of the cap, headroom 1.081x)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "(a) asec_population_catalogue._records_identity run on 55,762 invented domains.AsecHousehold records nesting 142,125 domains.AsecPerson records (22-digit PERIDNUM, string A_LINENO/A_AGE, \"observed_code\") plus 33,170 UnrepresentedAsecHousehold records: identity[1] = 26,535,585 (389 B per 1,000 household-with-persons bytes-per-1000 389,000; 209,000 per 1,000 ledger records). (b) asec_population_catalogue._bound_records_identity run on the full-source rows/roster of the native payload measurement: canonical_bytes = 124,133,900 over 197,887 records. Receipt #catalogue_records_stream and #catalogue_bound_records_stream.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "134,217,728", + "head_value": "134,217,728 (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> survey_population_preparation.py:1942 issue_asec_source_catalogue -> :589 `_records_identity(households, ledger, memo=True)` and :659 `_bound_records_identity(rows, roster)`; every later borrow (`_checked` -> _validate_state :447/:462 -> _current_records_identity) re-streams (a) unless the leaf memo :299-325 hits.", + "classification": "stays", + "notes": "The tightest bound in the family and it does not shrink with the fraction: stream (b) is the same 124 MB at 1/1000 as at full source, so the recovered 1/1000 run's success (preparation.json /catalogues/asec) is the proof it is under in reality; the exact real figure is the catalogue receipt's `native_binding.canonical_bytes` (:659), which the recovered run kept only as a sha256. Widths here are invented (6-digit parent ids, 5-digit H_SEQ). A lift of _MAX_EVIDENCE_BYTES for the population payload would not help this stream, and vice versa." + }, + { + "constant": "asec_coverage_authentication.py:38 _HEADER_MAX", + "value": "262_144", + "enforced_at": [ + "asec_coverage_authentication.py:455-460 (_bounded_header: `remaining = _HEADER_MAX` charged per visited token, `_require(remaining >= 0, \"COVERAGE_HEADER_BYTES\")`; depth cap :463 \"COVERAGE_HEADER_DEPTH\")", + "asec_coverage_authentication.py:514-517 (validate: `0 < len(self._header) <= _HEADER_MAX`, \"COVERAGE_HEADER_BYTES\"); AsecCoverageAuthenticationError" + ], + "refusal_codes": [ + "COVERAGE_HEADER_BYTES", + "COVERAGE_HEADER_DEPTH" + ], + "encodes": "The coverage receipt header (:673-702): fixed keys, three `sources`, partition counts, the literal reader receipt (contract + three cohorts + two digests) and the `implementation` dict (7,931 B measured).", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 12,454 B representative", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "coverage._bounded_header on a header shaped exactly as :673-702 with the real _implementation() and literal.coverage_field_contract(), digests as placeholders (receipt #coverage_body.header_bytes_representative).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "262,144", + "head_value": "262,144 (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> load_authenticated_asec_2024_native_population :706 -> authenticate_asec_coverage -> _reconstruct :703 `_bounded_header(header)`.", + "classification": "must-not-move", + "notes": "Charged token-at-a-time before the single `_json(header)` materialisation at :497; row-independent (`rows` and `compared_rows` are integers)." + }, + { + "constant": "asec_coverage_authentication.py:39 _BODY_MAX", + "value": "268_435_456", + "enforced_at": [ + "asec_coverage_authentication.py:281-286 (_CsvBounds.feed: per CSV record `self.budget[0] -= 4 + _NUMBERS.size + 12 + 22 + 32` then `-= 1` per PRPERTYP token byte, `_require(self.budget[0] >= 0, \"COVERAGE_BODY_BYTES\")`; one budget list `[_BODY_MAX]` seeded at :632 and shared by the three cohorts at :643-645)", + "asec_coverage_authentication.py:414 (_body: `_require(len(result) + _LENGTH.size + size <= _BODY_MAX, \"COVERAGE_BODY_BYTES\")`)", + "asec_coverage_authentication.py:424 (_decode_rows: `len(body) <= _BODY_MAX`)", + "asec_coverage_authentication.py:518-521 (validate: `len(self._body) <= _BODY_MAX`)", + "out-of-module callers seeding their own `budget=[..._BODY_MAX]` for _capture over the same members: current_asec_income_routing_source.py:1319, current_asec_dividend_source.py:353, current_asec_interest_source.py:329, current_asec_retirement_detail_source.py:408, current_social_security_source.py:239, current_survey_health_source.py:270, current_survey_housing.py:294, puf55_survey_recipients.py:156 (their reachability belongs to those families)" + ], + "refusal_codes": [ + "COVERAGE_BODY_BYTES" + ], + "encodes": "The coverage body: one length-prefixed record per ASEC person of all three cohorts (432,523): ` load_authenticated_asec_2024_native_population :706 -> authenticate_asec_coverage -> _reconstruct :632-668 (_capture x3, then _body).", + "classification": "stays", + "notes": "Three-cohort whole source, so no selection fraction moves it. The body is retained in `_attached_evidence` (asec_2024_native_population.py:399-400) and compared whole (`current[1:] == state.attached_evidence[1:]` :436-440) on every borrow; `table()` :552-559 materialises a 432,523-row DataFrame from it each call (`_roster` :234)." + }, + { + "constant": "asec_coverage_authentication.py:40 _ROW_MAX", + "value": "1_048_576", + "enforced_at": [ + "asec_coverage_authentication.py:272-275 (`self.row <= (_CSV_HEADER_MAX if self.header else _ROW_MAX)`, \"COVERAGE_CSV_RECORD_BYTES\" \u2014 raw CSV record bytes incl. quotes and delimiters)", + "asec_coverage_authentication.py:413 (_body: `size <= _ROW_MAX`, \"COVERAGE_ROW_BYTES\")", + "asec_coverage_authentication.py:432-435 (_decode_rows: `_NUMBERS.size + 12 <= size <= _ROW_MAX`, \"COVERAGE_ROW_BYTES\")", + "puf55_survey_recipients.py:162-164 (`stream.readline(owner._ROW_MAX + 1)`, `len(chunk) <= owner._ROW_MAX`; another family's module, not on these paths)" + ], + "refusal_codes": [ + "COVERAGE_CSV_RECORD_BYTES", + "COVERAGE_ROW_BYTES" + ], + "encodes": "A width: one pppub CSV record on capture, one encoded coverage record on encode/decode.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: ~1,955 B per pppub25 CSV record (277,882,549 / 142,125 from the pins); 100 B per encoded record", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Pins arithmetic and the _body measurement; no gated read.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,048,576", + "head_value": "1,048,576 (unchanged)", + "reachable_from": "19", + "reach_chain": "authenticate_asec_coverage -> _reconstruct -> _capture (feed) and _body.", + "classification": "must-not-move", + "notes": "The ` _reconstruct -> _capture / _body.", + "classification": "must-not-move", + "notes": "Token width; not a document." + }, + { + "constant": "asec_coverage_authentication.py:42 _CSV_HEADER_MAX", + "value": "65_536", + "enforced_at": [ + "asec_coverage_authentication.py:272-275 (the `self.header` branch of `self.row <= (_CSV_HEADER_MAX if self.header else _ROW_MAX)`, \"COVERAGE_CSV_RECORD_BYTES\"); the header is buffered at :278 and decoded once at :256-262 to locate PRPERTYP" + ], + "refusal_codes": [ + "COVERAGE_CSV_RECORD_BYTES" + ], + "encodes": "A width: the pppub CSV header line.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: not measured (gated CSV; a header line of several hundred short column names, a few kB)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Not measured; the value is a property of the upstream CSV header, not of any roster.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "19", + "reach_chain": "authenticate_asec_coverage -> _reconstruct -> _capture -> _CsvBounds.feed.", + "classification": "must-not-move", + "notes": "The header bytes are the only part of the CSV decoded inside _CsvBounds (:258-262)." + }, + { + "constant": "asec_coverage_authentication.py:44 _CHUNK", + "value": "65_536", + "enforced_at": [ + "asec_coverage_authentication.py:329 (`src.read(min(_CHUNK, size - count + 1))` \u2014 the cap is the pin `size`, the chunk is this constant)", + "asec_coverage_authentication.py:715 (_compare_candidate: `handle.read(min(_CHUNK, len(payload) - offset + 1))`)" + ], + "refusal_codes": [ + "none of its own; the surrounding checks raise COVERAGE_SOURCE_SIZE (:331) and COVERAGE_CANONICAL_BYTES (:716-719)" + ], + "encodes": "A read-chunk width for the streaming capture; no document is bounded by it.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 65,536 per read", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "The constant itself.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "19", + "reach_chain": "authenticate_asec_coverage -> _reconstruct -> _capture :329.", + "classification": "not-a-transport", + "notes": "A memory chunk size with no document behind it; listed in the implementation `limits` (:204-212) so it is code identity, not a bound." + }, + { + "constant": "asec_person_coverage_source.py:33 MAX_MEMBER_BYTES", + "value": "1_000_000_000", + "enforced_at": [ + "asec_person_coverage_source.py:168 (_regular_member_opener: `snapshot.st_size <= MAX_MEMBER_BYTES`, \"MEMBER_BYTES\"; CoverageSourceRefusalError)", + "asec_person_coverage_source.py:181 and :189-191 (_BoundedMember: `_remaining = MAX_MEMBER_BYTES`, `readinto(memoryview(buffer)[: self._remaining + 1])`, `_require(self._remaining >= 0, \"MEMBER_BYTES\")`)", + "asec_coverage_authentication.py:315-318 (_capture: `0 < size <= literal.MAX_MEMBER_BYTES`, \"COVERAGE_MEMBER_BYTES\"), :219 (recorded in the implementation identity)", + "survey_population_preparation.py:508 (_file_limits cap for asec/pppub23-25.csv -> :558 _regular_reader and :561 \"SOURCE_FILE_LIMIT\")" + ], + "refusal_codes": [ + "MEMBER_BYTES", + "COVERAGE_MEMBER_BYTES", + "SOURCE_FILE_LIMIT" + ], + "encodes": "An upstream file: one pppub CSV member (largest pinned 281,065,733).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 281,065,733 (largest member)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "The pins (education_assistance_source.py:112/134/156); no gated read.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,000,000,000", + "head_value": "1,000,000,000 (unchanged)", + "reachable_from": "19", + "reach_chain": "authenticate_asec_coverage -> _reconstruct :658 read_asec_person_coverage_source -> _regular_member_opener / _BoundedMember.", + "classification": "must-not-move", + "notes": "`_digest` (:150-158) streams `json.dumps(row)` per person into sha256 with no cap and no whole copy (two passes over 432,523 rows); it is a stream, not a bound." + }, + { + "constant": "asec_person_income_source.py:39 _ATTACHMENT_MAX_BYTES", + "value": "2_000_000_000", + "enforced_at": [ + "asec_person_income_source.py:272-275 (_candidate_snapshot: `0 < before.st_size == expected_size <= _ATTACHMENT_MAX_BYTES`, \"RESTORATION_SIZE\"; MoneyRefusalError)", + "asec_person_income_source.py:277-279 (`src.read(1024 * 1024)` chunks, `count <= expected_size` \"RESTORATION_SIZE\"), :283 (`count == expected_size`), :284-300 (\"RESTORATION_BYTES_CHANGED\")" + ], + "refusal_codes": [ + "RESTORATION_SIZE", + "RESTORATION_BYTES_CHANGED" + ], + "encodes": "An upstream file: the candidate restored person-income checkpoint (asec_person_income_source.checkpoint.h5), whose expected size is the freshly written canonical checkpoint's (:323-333).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 666,333,922 (row-ceilings census, measured on the recovered restoration)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Cited from experiments/native-row-ceilings/census.json; not re-measured (gated).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "2,000,000,000", + "head_value": "2,000,000,000 (unchanged)", + "reachable_from": "19", + "reach_chain": "load_authenticated_asec_2024_native_population :702 -> load_authenticated_restored_current_money_source :455 -> verify_asec_person_income_source -> _verify :330 _candidate_snapshot (after write_frame_checkpoint :324 writes the expected copy).", + "classification": "must-not-move", + "notes": "Each verification rewrites the whole 666 MB canonical checkpoint to a temp directory and copies the candidate beside it (:320-337). `_read_member` :83 enforces the pppub pin exactly (see the member_size_bytes row)." + }, + { + "constant": "asec_prepared_source.py:91 RESTORATION_RECEIPT_MAX_BYTES", + "value": "256 * 1024 = 262,144", + "enforced_at": [ + "asec_prepared_source.py:217-219 (`0 < info.st_size <= RESTORATION_RECEIPT_MAX_BYTES`, \"RESTORATION_RECEIPT\"; PreparedSourceRefusalError)", + "asec_prepared_source.py:221 (`stream.read(RESTORATION_RECEIPT_MAX_BYTES + 1)`), :224 (`0 < len(payload) <= RESTORATION_RECEIPT_MAX_BYTES`)", + "asec_prepared_source.py:227 (`_parse(payload[:-1], RESTORATION_RECEIPT_MAX_BYTES)` -> asec_current_money.py:84 \"JSON_SIZE_OR_TYPE\" MoneyRefusalError, re-raised at :228-229 as \"RESTORATION_RECEIPT\")", + "asec_prepared_source.py:230 (consumer re-encoding `_json(receipt) == payload[:-1]`, \"RESTORATION_RECEIPT_CANONICAL\")" + ], + "refusal_codes": [ + "RESTORATION_RECEIPT", + "RESTORATION_RECEIPT_CANONICAL" + ], + "encodes": "restoration.receipt.json \u2014 a fixed receipt (schema keys :126-150, three `sources` :151-165).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: 2,424 B (row-ceilings census, recovered artifact)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Cited from experiments/native-row-ceilings/census.json.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "262,144", + "head_value": "262,144 (unchanged)", + "reachable_from": "none", + "reach_chain": "prepare_asec_current_money_population :421 is called only by graph_asec_prepared.USAsecPreparedCreateKernel.run :309 (node asec_prepared.create) and graph_composed_population.compose_from_sources :406 (node composed_population.create); neither is a node of the 19- or 45-node graphs nor of survey_age_calibration.", + "classification": "must-not-move", + "notes": "The other prepared-source encoders (`receipt_payload` :400, `encode_current_money` :471 from _asec_current_money_codec \u2014 the money family) carry no cap of this module's; the prepared receipt is digest-shaped (`entity_rows`, `dtype_transitions` per column)." + }, + { + "constant": "asec_demographic_source.py:66 _HEADER_MAX", + "value": "262_144", + "enforced_at": [ + "asec_demographic_source.py:1106-1109 (validate: `0 < len(self._header) <= _HEADER_MAX`, \"DEMOGRAPHIC_HEADER_SIZE\"; DemographicSourceRefusalError)", + "asec_demographic_source.py:1118-1119 (consumer re-encodings of fixed sub-documents: `_json(header[\"contract\"]) == _json(contract_document())`, `_json(header[\"implementation\"]) == _json(_implementation())`, \"DEMOGRAPHIC_CONTENT_CHANGED\")" + ], + "refusal_codes": [ + "DEMOGRAPHIC_HEADER_SIZE", + "DEMOGRAPHIC_CONTENT_CHANGED" + ], + "encodes": "The demographic header (:1302-1324): contract document, implementation, household-membership verification, per-cohort diagnostics, three `sources`, column digests \u2014 all fixed-shape.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: ~30-40 kB (contract 9,159 B and implementation 10,154 B measured; membership/diagnostics/sources fixed-shape, not measured)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "`len(demo._json(demo.contract_document()))` and `len(demo._json(demo._implementation()))` offline (receipt #demographic_source).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "262,144", + "head_value": "262,144 (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_predictors.source_projection / .asec_design_donor (recovered graph.json params `demographic_conditioning: true`) -> graph_current_survey_predictors._Kernel._qualified :301 values.qualify_current_survey_predictors(..., demographic_conditioning=...) -> current_survey_predictors.py:359 `_demographic_features(preparation, entry) if demographic_conditioning` -> :85 observed_geography.demographics.qualify_current_asec_demographics -> current_asec_demographics.py:222 load_authenticated_asec_demographic_source(parent, member_paths=...) -> _reconstruct :1325.", + "classification": "must-not-move", + "notes": "Reached only because the pilot conditioned on demographics; with `demographic_conditioning=False` (the default at current_survey_predictors.py:315) the loader is never called and the module is only hashed for code identity (graph_current_survey_predictors.py:275)." + }, + { + "constant": "asec_demographic_source.py:75 _MAX_PERSONS (as the row factor of the width identity at :1115 `len(self._body) == rows * len(COLUMNS) * 8`)", + "value": "14_000_000 rows x 14 columns x 8 = 1,568,000,000 implicit body ceiling", + "enforced_at": [ + "asec_demographic_source.py:1112 (validate: `0 < rows <= _MAX_PERSONS`, \"DEMOGRAPHIC_ROWS\")", + "asec_demographic_source.py:1113-1121 (`len(self._body) == rows * len(COLUMNS) * 8`, \"DEMOGRAPHIC_CONTENT_CHANGED\")", + "asec_demographic_source.py:1153 (_reconstruct: `0 < rows <= _MAX_PERSONS`, \"DEMOGRAPHIC_ROWS\"); row-count-only sites :660 (\"MEMBERSHIP_ROWS\"), :803 (\"CLASSIFY_ROWS\"), :1024 (\"ACS_ROWS\", reached only from current_survey_household_roles \u2014 49/51-node variants)" + ], + "refusal_codes": [ + "DEMOGRAPHIC_ROWS", + "DEMOGRAPHIC_CONTENT_CHANGED" + ], + "encodes": "The demographic body: 14 little-endian int64 columns (COORDINATES + ALIASES + DERIVED, :378) over the three-cohort ASEC persons of the restored money parent (432,523).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: 48,442,576 (432,523 x 112 B)", + "bytes_at_full_source": "same as at tenth (headroom 32x against the implicit ceiling)", + "measurement_basis": "`len(demo.COLUMNS)` = 14 imported, times 8, times 432,523 (146,133 + 144,265 + 142,125 pinned rows); the body is `b\"\".join(buffers)` at :1300-1301 (receipt #demographic_source).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "14,000,000 (already lifted on origin/native-row-ceilings; experiments/native-row-ceilings/census.json predates that lift and still says 600,000)", + "head_value": "14,000,000 (unchanged on this branch)", + "reachable_from": "19", + "reach_chain": "As the _HEADER_MAX row: survey_predictors.* -> current_asec_demographics.py:222 -> _reconstruct :1300-1301.", + "classification": "stays", + "notes": "Not a byte-named constant: the body ceiling exists only through the width identity. `array()` :1123-1128 is a zero-copy frombuffer view; `attach_asec_demographic_source` :1447-1448 copies the nine alias/derived columns into the frame. The three pppub captures (`_snapshot`, ~278 MB each) are written whole to a temp directory per reconstruction." + }, + { + "constant": "asec_engine_evaluation.py:42 EVALUATION_HEADER_MAX_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "asec_engine_evaluation.py:391 (_evaluation_parts: `0 < len(header) <= EVALUATION_HEADER_MAX_BYTES`, \"HEADER_SIZE\"; EngineAdmissionError)", + "asec_engine_evaluation.py:424-427 (decode: `0 < size <= EVALUATION_HEADER_MAX_BYTES and start + size <= len(payload) - 32`, \"HEADER_SIZE\")", + "asec_engine_evaluation.py:431-433 (consumer re-encoding `canonical_json(header) == payload[start : start + size]`, \"HEADER_CANONICAL\")" + ], + "refusal_codes": [ + "HEADER_SIZE", + "HEADER_CANONICAL" + ], + "encodes": "The engine-evaluation header (:356-384): four closures, aggregates, baseline runtime identity, bindings \u2014 fixed-shape.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: not measured (fixed, a few kB)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Row-independent by the key roster at :356-384 (`person_rows` is an integer).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "none", + "reach_chain": "Would be reached by graph_asec_prepared.USAsecPreparedEngineKernel.run :848 encode_engine_evaluation (node asec_prepared.engine_current_money); decode_engine_evaluation has no production caller in packages/*/src.", + "classification": "must-not-move", + "notes": "Header width." + }, + { + "constant": "asec_engine_evaluation.py:43 EVALUATION_MAX_ROWS (times 8 x len(roots) through the length identity at :439 `len(payload) == start + size + 8 * rows * len(roots) + 32`)", + "value": "1_000_000 rows x 32 B = 32,000,000 implicit body ceiling (4 admitted roots x 8)", + "enforced_at": [ + "asec_engine_evaluation.py:345 (materialize_engine_outputs: `0 < rows <= EVALUATION_MAX_ROWS`, \"ENGINE_ROW_BOUND\")", + "asec_engine_evaluation.py:437 (decode: `0 < rows <= EVALUATION_MAX_ROWS`, \"ENGINE_ROW_BOUND\"), :438-440 (\"PAYLOAD_LENGTH\")", + "asec_engine_evaluation.py:398-401 (`array.shape == (rows,)`, \"ENGINE_RESULT_SHAPE\")" + ], + "refusal_codes": [ + "ENGINE_ROW_BOUND", + "PAYLOAD_LENGTH", + "ENGINE_RESULT_SHAPE" + ], + "encodes": "Four float64 arrays (` :558 _regular_reader and :561 \"SOURCE_FILE_LIMIT\")" + ], + "refusal_codes": [ + "MEMBER_REGISTRY", + "SOURCE_FILE_LIMIT" + ], + "encodes": "An upstream file: an hhpub household CSV member (largest pinned 33,298,479).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 33,298,479 (hhpub25)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "The pins (native_household_origin.py:146/156/166).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "67,108,864", + "head_value": "67,108,864 (unchanged)", + "reachable_from": "19", + "reach_chain": "load_authenticated_asec_2024_native_population :709 -> load_authenticated_asec_household_weights :469 _registry.", + "classification": "must-not-move", + "notes": "Registry-level ceiling on the pins; the exact size is enforced by the pin rows." + }, + { + "constant": "asec_original_household_weights.py:43 _MAX_RECORD_CHARS", + "value": "65_536", + "enforced_at": [ + "asec_original_household_weights.py:266 (_RecordLines.__next__: `self.text.readline(_MAX_RECORD_CHARS - self.characters + 1)`), :270 (`self.characters <= _MAX_RECORD_CHARS`, \"MEMBER_RECORD_SIZE\")", + "asec_household_coverage_fields.py:270 (`shared._RecordLines(text)` \u2014 the same class for the field owner)" + ], + "refusal_codes": [ + "MEMBER_RECORD_SIZE" + ], + "encodes": "A width: one logical hhpub CSV record including embedded newlines.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: ~374 chars mean (33,298,479 / 88,932 from the pins)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Pins arithmetic.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "19", + "reach_chain": "load_authenticated_asec_household_weights :492 _read_capture -> _RecordLines.", + "classification": "must-not-move", + "notes": "Record width." + }, + { + "constant": "asec_original_household_weights.py:44 _MAX_FIELD_CHARS", + "value": "128", + "enforced_at": [ + "asec_original_household_weights.py:276 (_record_fields: `all(len(t) <= _MAX_FIELD_CHARS for t in tokens)`, \"MEMBER_FIELD_SIZE\"; HouseholdWeightSourceError)", + "asec_household_coverage_fields.py:208-210 (`len(token) <= shared._MAX_FIELD_CHARS`, \"MEMBER_FIELD_SIZE\"; HouseholdCoverageSourceError)", + "current_asec_demographics.py:60 (`len(token) <= household._MAX_FIELD_CHARS` on the current-state tokens; on the 19 path via qualify_current_asec_demographics :236 _load_current_state)" + ], + "refusal_codes": [ + "MEMBER_FIELD_SIZE" + ], + "encodes": "A width: one H_SEQ / H_HHTYPE / HSUP_WGT (and field/state) token.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: <= 8 (HSUP_WGT printed_length 8, _FIELD_CONTRACT :53-77)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "_FIELD_CONTRACT literals.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "128", + "head_value": "128 (unchanged)", + "reachable_from": "19", + "reach_chain": "load_authenticated_asec_household_weights -> _read_capture :360 _record_fields; and survey_predictors.* -> current_asec_demographics :236.", + "classification": "must-not-move", + "notes": "Token width." + }, + { + "constant": "asec_original_household_weights.py:45 _MAX_PAYLOAD_BYTES", + "value": "512 * 1024**2 = 536,870,912", + "enforced_at": [ + "asec_original_household_weights.py:95-97 (_issue_capsule: `len(payload) <= _MAX_PAYLOAD_BYTES`, \"SOURCE_BYTES\"; HouseholdWeightSourceError)", + "asec_original_household_weights.py:116-121 (_checked_capsule_payload: `len(payload) <= _MAX_PAYLOAD_BYTES and _sha(payload) == entry[1]`, \"SOURCE_MUTATION\")", + "asec_original_household_weights.py:161 (_encode: `len(result) + len(encoded) <= _MAX_PAYLOAD_BYTES` per iterencode fragment, \"ENCODING_SIZE\")", + "asec_original_household_weights.py:471-475 (`len(candidate) <= _MAX_PAYLOAD_BYTES`, \"CANDIDATE_SIZE\")", + "asec_household_coverage_fields.py:403 (`len(candidate) <= shared._MAX_PAYLOAD_BYTES` for the field owner's candidate)" + ], + "refusal_codes": [ + "SOURCE_BYTES", + "SOURCE_MUTATION", + "ENCODING_SIZE", + "CANDIDATE_SIZE" + ], + "encodes": "The household-weights payload (:498-525) with `records` = one record per hhpub25 row (88,932; `_record_fields` :274-316 plus the pins added at :364-373), and two further whole materialisations under the same cap: `_encode(records)` for `projection_sha256` (:518) and `_encode([[income_year, native_household_id] ...])` for `native_roster_sha256` (:515-517).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: payload 45,437,948; records-only 45,433,147; roster list 1,156,117 (510 B per record)", + "bytes_at_full_source": "same as at tenth (headroom 11.8x)", + "measurement_basis": "anchors._encode on 88,932 invented records built by anchors._record_fields([str(H_SEQ), \"1\", \"199603\"]) plus pins, with the real _implementation() as `producer`; per-record width from the 1,000 -> 2,000 difference (receipt #household_weights_payload).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "536,870,912", + "head_value": "536,870,912 (unchanged)", + "reachable_from": "19", + "reach_chain": "load_authenticated_asec_2024_native_population :709 -> load_authenticated_asec_household_weights :498 _encode(payload).", + "classification": "stays", + "notes": "Whole member at every fraction. Consumers re-parse the 45 MB payload on every `.document` access (`verify_asec_household_weights_source` :454 `json.loads(payload)`): asec_2024_native_population.py:241, :321 (original_weight_fractions :419), asec_population_catalogue.py:373, and every borrow via :416/:451; `_encode(doc[\"producer\"]) == _encode(_implementation())` at :455 re-encodes only the fixed producer sub-document." + }, + { + "constant": "graph_asec_income.py:81 ACCOUNTING_HEADER_MAX", + "value": "65536", + "enforced_at": [ + "graph_asec_income.py:482-483 (derive_reported_income: `encoded = _json(header); _require(len(encoded) <= ACCOUNTING_HEADER_MAX, \"ACCOUNTING_HEADER_SIZE\")`; MoneyRefusalError)", + "factor of ACCOUNTING_PAYLOAD_MAX_BYTES at graph_asec_income.py:86" + ], + "refusal_codes": [ + "ACCOUNTING_HEADER_SIZE" + ], + "encodes": "The reported-income accounting header (:450-481): 28 column descriptors (name, dtype, offset, bytes, sha256), input digests, source digests \u2014 fixed-shape.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: not measured (28 descriptors with a sha256 each, roughly 3-4 kB)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Row-independent by the key roster.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "none", + "reach_chain": "USAsecReportedIncomeKernel.run :628 / USComposedAsecReportedIncomeKernel.run :455 -> derive_reported_income \u2014 nodes asec_prepared.reported_income / composed_population.asec_reported_income, not in the 19/45 graphs.", + "classification": "must-not-move", + "notes": "Header width." + }, + { + "constant": "graph_asec_income.py:82 ACCOUNTING_ROW_BYTES", + "value": "8 * 9 + len(EVIDENCE_COLUMNS) = 72 + 19 = 91", + "enforced_at": [ + "graph_asec_income.py:445-449 (`cursor == rows * ACCOUNTING_ROW_BYTES`, \"ACCOUNTING_LENGTH\")", + "factor of ACCOUNTING_PAYLOAD_MAX_BYTES at graph_asec_income.py:87" + ], + "refusal_codes": [ + "ACCOUNTING_LENGTH" + ], + "encodes": "A width: bytes per accounting row (3 int64 + 6 float64 + 19 uint8 columns, ACCOUNTING_DTYPES :78).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: 91", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Imported constant.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "91", + "head_value": "91 (unchanged)", + "reachable_from": "none", + "reach_chain": "As ACCOUNTING_HEADER_MAX.", + "classification": "must-not-move", + "notes": "Encoder width." + }, + { + "constant": "graph_asec_income.py:83-89 ACCOUNTING_PAYLOAD_MAX_BYTES", + "value": "len(ACCOUNTING_MAGIC) + 4 + ACCOUNTING_HEADER_MAX + source._MAX_PERSONS * ACCOUNTING_ROW_BYTES + 32 = 9 + 4 + 65,536 + 600,000 * 91 + 32 = 54,665,581", + "enforced_at": [ + "graph_asec_income.py:495 (encode_reported_income: `len(payload) + 32 <= ACCOUNTING_PAYLOAD_MAX_BYTES`, \"ACCOUNTING_SIZE\"; MoneyRefusalError)", + "graph_asec_income.py:501-504 (read_reported_income: `len(payload) <= ACCOUNTING_PAYLOAD_MAX_BYTES`, \"ACCOUNTING_SIZE\")", + "graph_asec_income.py:506 (consumer re-encoding `payload == encode_reported_income(expected)`, \"ACCOUNTING_RECONSTRUCTION\")" + ], + "refusal_codes": [ + "ACCOUNTING_SIZE", + "ACCOUNTING_RECONSTRUCTION" + ], + "encodes": "The reported-income accounting payload: header + 28 buffers, 91 B per selected ASEC person (the prepared slice's sample, or the composed population's ASEC arm).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "3,935,932 at 1/10 of the prepared population (43,252 persons x 91) \u2014 the prepared slice's own fraction", + "bytes_at_full_source": "39,359,593 over the whole prepared population (432,523 x 91) + header + 45 framing; headroom 1.39x", + "measurement_basis": "Width x count (receipt #fixed_width_bodies.reported_income_accounting).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "54,665,581", + "head_value": "54,665,581 (unchanged)", + "reachable_from": "none", + "reach_chain": "USAsecReportedIncomeKernel.run :650 encode_reported_income; USComposedAsecReportedIncomeKernel.run :479 \u2014 not in the 19/45 graphs or the age runner.", + "classification": "stays", + "notes": "The row ceiling is source._MAX_PERSONS = 600,000, 1.39x the whole three-cohort roster; `read_reported_income` has no production caller in packages/*/src." + }, + { + "constant": "asec_current_money.py:47 HEADER_MAX_BYTES (imported at graph_composed_asec_binding.py:78; definition owned by the current-money family)", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "graph_composed_asec_binding.py:1016 (_arm_rows_parts: `len(header) <= HEADER_MAX_BYTES`, \"ARM_ROWS_HEADER_SIZE\"; PreparedGraphError)", + "graph_composed_asec_binding.py:1056-1059 (bind_composed_asec_arm_rows: `0 < size <= HEADER_MAX_BYTES and start + size < len(payload) - 32`, \"ARM_ROWS_HEADER_SIZE\")", + "graph_composed_asec_binding.py:1061 (`_parse(header, HEADER_MAX_BYTES)` -> asec_current_money.py:84 \"JSON_SIZE_OR_TYPE\")", + "graph_composed_asec_binding.py:1063 (consumer re-encoding `_json(data) == header`, \"ARM_ROWS_HEADER_CANONICAL\")", + "term of ARM_ROWS_PAYLOAD_MAX_BYTES at graph_composed_asec_binding.py:959" + ], + "refusal_codes": [ + "ARM_ROWS_HEADER_SIZE", + "JSON_SIZE_OR_TYPE", + "ARM_ROWS_HEADER_CANONICAL" + ], + "encodes": "The arm-rows header (:995-1015): seven keys, a four-entry buffer roster with byte counts and digests \u2014 fixed-shape.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: roughly 600 B (four buffer descriptors with a sha256 each)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Row-independent by the key roster (`rows` are two integers).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "none", + "reach_chain": "USComposedAsecBindKernel.run :1268 encode_composed_asec_arm_rows; the measure kernels' bind_composed_asec_arm_rows \u2014 node composed_population.asec_bind only.", + "classification": "must-not-move", + "notes": "Only this family's enforcement sites are listed; merge on the definition key with the current-money family's row (asec_current_money.py:47), which also carries MAX_PERSONS :48 and MAX_HOUSEHOLDS :49." + }, + { + "constant": "graph_composed_asec_binding.py:956-962 ARM_ROWS_PAYLOAD_MAX_BYTES", + "value": "len(ARM_ROWS_MAGIC) + 4 + HEADER_MAX_BYTES + 16 * (MAX_PERSONS + MAX_HOUSEHOLDS) + 32 = 9 + 4 + 65,536 + 16 * (1,000,000 + 400,000) + 32 = 22,465,581 (row factors from asec_current_money.py:48-49)", + "enforced_at": [ + "graph_composed_asec_binding.py:1033 (encode_composed_asec_arm_rows: `len(payload) + 32 <= ARM_ROWS_PAYLOAD_MAX_BYTES`, \"ARM_ROWS_SIZE\"; PreparedGraphError)", + "graph_composed_asec_binding.py:1047-1051 (bind_composed_asec_arm_rows: `len(ARM_ROWS_MAGIC) + 4 + 32 < len(payload) <= ARM_ROWS_PAYLOAD_MAX_BYTES`, \"ARM_ROWS_SIZE\")", + "graph_composed_asec_binding.py:1093-1106 (width identity `width = rows[entity] * 8` per buffer, \"ARM_ROWS_BUFFER_HEADER:...\" / \"ARM_ROWS_LENGTH\")" + ], + "refusal_codes": [ + "ARM_ROWS_SIZE", + "ARM_ROWS_BUFFER_HEADER", + "ARM_ROWS_LENGTH" + ], + "encodes": "Four int64 buffers (source positions and original ids for the ASEC-arm persons and households of the composed population), 16 B per arm person row + 16 B per arm household row (:984-1017, :1030-1031).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "963,536 at 1/10 of the prepared arm (43,252 persons + 16,969 households) \u2014 the composed graph's own sample_fraction", + "bytes_at_full_source": "9,635,552 for the whole prepared arm (432,523 persons + 169,699 households, the latter from the row-ceilings derivation); headroom 2.33x", + "measurement_basis": "Width x count (receipt #fixed_width_bodies.arm_rows).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "22,465,581", + "head_value": "22,465,581 (unchanged)", + "reachable_from": "none", + "reach_chain": "USComposedAsecBindKernel.run :1268 (composed_population.asec_bind); bound again by graph_composed_asec_measures :325/:446 \u2014 not in the 19/45 graphs or the age runner.", + "classification": "stays", + "notes": "The arm can never exceed the prepared population (`ARM_OVERFLOW` :636-639), so the bound is fixed at 2.33x headroom whatever the composed fraction." + }, + { + "constant": "graph_composed_population.py:322-333 canonical_json(channels) / canonical_json(spine_ids) / canonical_json([[channel, spine_id, source_id, output_id] ...]) in _source_origin (cap: none)", + "value": "none (no byte cap; each list is materialised whole by microcosm.graph.canonical.canonical_json before `_sha`)", + "enforced_at": [ + "producer graph_composed_population.py:311-334 (_source_origin: three whole-roster encodings per entity for all six US entities, hashed into `ordered_channels_sha256`, `ordered_spine_source_ids_sha256`, `ordered_source_to_output_sha256`)", + "consumer graph_composed_asec_binding.py:448-470 (_origin_entity_digests recomputes the same three encodings per entity from the composed tables) compared at :613-616 (`_origin_entity_digests(tables) == origin[\"assembled\"]`, \"ORIGIN_DOES_NOT_DESCRIBE_THIS_FRAME\"; PreparedGraphError)" + ], + "refusal_codes": [ + "ORIGIN_DOES_NOT_DESCRIBE_THIS_FRAME" + ], + "encodes": "Whole-roster provenance lists over every row of the composed (stacked) population: per entity a list of channel strings, a list of pre-remap spine ids and a list of four-element rows; also `.tolist()` copies at :311-316 / :448-453.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "four-tuple list (largest of the three): households 5,085,952; persons 11,422,301 (158,737 / 356,501 rows); channels 6.04 B/row, spine ids 8.0 B/row, tuples 32.04 B/row", + "bytes_at_full_source": "four-tuple list: households 50,859,527; persons 114,223,016 (1,587,376 / 3,565,013 rows); channels 9,587,751 / 21,532,678; spine ids 12,699,008 / 28,520,104; the four other entities scale by the 1/1000 roster ratios (SCALED)", + "measurement_basis": "canonical_json on invented lists at 100,000 and 200,000 rows with 7-digit ids and a 4% \"asec\" share, per-row width by difference, times the stacked counts (receipt #composed_origin_rosters).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none", + "head_value": "none (unchanged)", + "reachable_from": "none", + "reach_chain": "graph_composed_population.compose_from_sources :438 / USComposedPopulationCreateKernel.run :487 (composed_population.create) and USComposedAsecBindKernel.run :1238 -> resolve_composed_asec_binding :614 \u2014 not in the 19/45 graphs or the age runner.", + "classification": "segment", + "notes": "No cap, so it cannot bind, but each of the eighteen lists exists whole before hashing (114 MB for the persons' four-tuple list at full source) and the consumer builds all of them again; a per-row digest stream keeps the bytes and removes the whole copies. Both `binds` flags are false only because there is nothing to exceed." + }, + { + "constant": "graph_composed_population.py:252-259 canonical_json([None if value is None or value is pd.NA else str(value) for value in series.tolist()]) in _arm_origin (cap: none)", + "value": "none (no byte cap; one whole-column list per native identity column, materialised then `_sha`)", + "enforced_at": [ + "graph_composed_population.py:241-260 (_arm_origin, called at :382-385 for both arms inside _source_origin); the ACS arm's SERIALNO / source_household_id / source_person_id and the ASEC arm's asec_H_SEQ / PERIDNUM / PH_SEQ per NATIVE_IDENTITY_COLUMNS :151-162" + ], + "refusal_codes": [ + "none (ARM_NATIVE_IDENTITY:* only refuses a missing column at :246)" + ], + "encodes": "Whole-arm identity lists at every fraction: the sources are read whole (:406 prepare_asec_current_money_population, :419 load_graph_acs) before prepare_stacked_spine samples (:428-430), so the lists cover the whole prepared ASEC arm and the whole ACS source.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: ASEC PERIDNUM 10,813,075 (432,523 x 25); PH_SEQ 3,460,184; asec_H_SEQ 1,357,592 (169,699 households, DERIVED); ACS SERIALNO 24,505,824 (1,531,614 x 16); ACS source_household_id and source_person_id 34,228,880 each (3,422,888 x 10)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "canonical_json on invented string lists at two sizes (22-digit PERIDNUM, 5-digit H_SEQ, 13-char SERIALNO, 7-digit ints as str), per-element width by difference, times the whole-source counts (receipt #composed_origin_rosters.arm_origin). The SERIALNO figure agrees with byte-census-measurements.json acs_household_archive.serialno_list_bytes_full_source = 24,505,825 measured by the ACS family through its own encoder.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none", + "head_value": "none (unchanged)", + "reachable_from": "none", + "reach_chain": "graph_composed_population.compose_from_sources :438 -> _source_origin :382-385 -> _arm_origin (composed_population.create only).", + "classification": "segment", + "notes": "Whole ACS-source identity lists (24.5 MB + 2 x 34.2 MB) are built even at 1/1000 because the arm is read whole before sampling; the `.tolist()` at :256 and the str() conversion double the transient copy." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/asec/measure_asec.py", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/asec/asec-byte-census.json" + ], + "open_questions": [ + "Native population payload headroom is 9.5% at full source (122,530,878 vs 134,217,728) on invented widths: 6-digit parent receiving ids, 5-digit H_SEQ, 1-digit A_LINENO, \"observed_code\" states, and a 30-key placeholder for `parent.source.identity` in the fixed preamble (16,118 B total). A real full-source encode could move the figure by roughly +/-1%; the verdict `stays` holds unless real rows are ~10% wider than these.", + "The catalogue's native-binding stream (asec_population_catalogue.py:328-338) measures 124,133,900 B against 134,217,728 at every fraction (92.5%). The only exact figure is the catalogue receipt's `native_binding.canonical_bytes` (:659), which the recovered 1/1000 run kept only as `catalogues.asec.receipt_sha256`; reading `asec.to_bytes()` from any live 1/1000 preparation would settle the real margin. Because it is fraction-independent, no selection fraction can relieve it; only the nesting of `source`/`household_fields` into each household row (asec_2024_native_population.py:344-345) drives it.", + "The three-cohort household count 169,699 (housing status / universe / arm-rows / asec_H_SEQ figures) is the row-ceilings lane's derivation, not re-measured (gated data); those figures scale linearly with it.", + "Rows keyed on definitions outside this family's module list \u2014 education_assistance_source.py:112/134/156 member_size_bytes, native_household_origin.py:146/156/166 size_bytes, asec_current_money.py:47 HEADER_MAX_BYTES \u2014 carry only this family's enforcement sites and should be merged with the owning family's rows on the definition key.", + "asec_coverage_authentication._BODY_MAX is seeded as a fresh capture budget by eight modules outside this family (current_asec_income_routing_source.py:1319, current_asec_dividend_source.py:353, current_asec_interest_source.py:329, current_asec_retirement_detail_source.py:408, current_social_security_source.py:239, current_survey_health_source.py:270, current_survey_housing.py:294, puf55_survey_recipients.py:156) and _ROW_MAX by puf55_survey_recipients.py:162-164; whether the 45-node property-income nodes reach the dividend/interest/retirement/routing callers is the property family's call chain to confirm.", + "The demographic-source rows are reachable from the nineteen only because the recovered 19-node graph.json carries `demographic_conditioning: true` on the survey_predictors nodes; the 45-node run's graph.json is not under _recovered/pilot-runs/native45-v5/required or /probe, so the same flag on the 45 is inferred from \u00a72 (\"the nineteen above\"), not read.", + "Consumer-side canonical_json re-encodings in graph_asec_prepared (:189, :407-409, :651-652), graph_asec_income (:588-589), graph_composed_asec_binding (:413, :422, :579, :811), graph_composed_asec_measures (:159, :243) and graph_composed_population (:577) were judged not roster-shaped after reading graph_context.py:104-160 (per-entity `columns` list + `rows` + `ordered_ids_sha256`), frame_sampling.py:313 (`selected_household_ids_sha256`) and stacked_spine.py:558-569 (digests only); if any of those documents carries per-row content these checks become rows with cap `none`.", + "asec_checkpoint.py and asec_household_observations.py contain no byte bound (no rows): asec_household_observations._read_numeric_household reads the selected int64 HDF block whole (`values[:, wanted]` :177) with no cap, and both are on the raw-stage assembly path rather than the native 19/45 path.", + "Row-count-only constants without a byte name and without a width product (asec_2024_native_population._MAX_HOUSEHOLDS :39 and _MAX_PERSONS :40, asec_coverage_authentication._MAX_PERSONS :43, asec_person_coverage_source.MAX_PERSONS :32, asec_original_household_weights._MAX_ROWS :42, asec_housing_status/universe.MAX_HOUSEHOLDS as factors) are covered by experiments/native-row-ceilings/census.json and were not given separate rows here; where they multiply a width (demographic, student, engine, income, housing, accounting, arm rows) the product row above names them." + ] + }, + { + "family": "ACS coverage and housing source", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py (full, 832 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py (full, 749 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py (full, 301 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py (full, 933 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py (full, 874 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_sources.py (full, 309 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py (full, 613 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py (full, 365 lines; no byte bound)", + "packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py (full, 31 lines; no byte bound)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py (full, 203 lines; no byte bound)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py (full, 230 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py (full, 698 lines; reachability note only)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_1yr_sources.json (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py (lines 15-50, the constants graph_housing_universe enforces)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py (partial: 500-560, 590-612, 1157-1200, 1430-1460, 1695-1720, 1760-1800, 1880-1990, 2035-2050, def inventory)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py (partial: 1275-1360)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py (partial: 137-175)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py (partial: 949-1010)", + "packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py (partial: 230-300, 355-385)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_sources.py (partial: 280-310)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py (partial: 90-105)", + "packages/microcosm-graph/src/microcosm/graph/canonical.py (partial: 1-80, canonical_json)", + "packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py, test_us_acs_housing_source.py, test_us_acs_population_catalogue.py, test_us_acs_pums.py (partial; fixture builders reused by the measurement script, not run under pytest)", + "experiments/native-byte-transports/census-brief.md (full)", + "experiments/native-byte-transports/byte-census-measurements.json (full)", + "experiments/native-byte-transports/byte_census_measure.py (full)", + "experiments/native-row-ceilings/census.json (this family's 38 rows and verdicts)", + "docs/us-native-scale-transport.md and docs/us-native-row-ceilings.md (ACS-related sections)", + "_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/graph.json and preparation.json (nodes/sources; /catalogues/acs, /native/acs, /producer/acs, /selection counts)" + ], + "rows": [ + { + "constant": "acs_person_coverage_authentication.py:36 MAX_HEADER_BYTES", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_person_coverage_authentication.py:84 default cap of _json_size -> :93 ACSCoverageAuthenticationError(\"CANONICAL_SIZE\")", + "acs_person_coverage_authentication.py:132/:141/:147 default cap of _json_chunks -> :147 \"CANONICAL_SIZE\"", + "acs_person_coverage_authentication.py:151 default cap of _json (no runner-path call relies on the default; every site passes a cap)", + "acs_person_coverage_authentication.py:676-684 native_binding receipt, explicit MAX_HEADER_BYTES", + "acs_person_coverage_authentication.py:804 `raw_header = _json(header, MAX_HEADER_BYTES)` (coverage envelope header)", + "recorded, not enforced: acs_person_coverage_authentication.py:268 producer ceilings" + ], + "refusal_codes": [ + "CANONICAL_SIZE (ACSCoverageAuthenticationError, :93 pre-charge pass and :147 encoder pass)" + ], + "encodes": "Fixed-shape header of the coverage envelope (protocol, producer closure 1,490 B, archives, six-member inventory, field_contract, original_literal_receipt whose row-dependent parts are integers and digests, columns, body_bytes/body_sha256/rows integers, native_consistency) plus the 589-byte UngrantedACSNativeBinding receipt; O(members) and O(columns), never O(rows).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "5,915 (header with the real six-member inventory; 5,138 with the invented three); native_binding receipt 589", + "bytes_at_full_source": "5,915 (identical: row counts enter as integers)", + "measurement_basis": "measured through auth._json at this head by driving the real issuance on invented archives (scratchpad measure_acs_family.py part 2: coverage_header_bytes_invented, coverage_header_bytes_REALISTIC_members with the pilot's 3+3 member names/sizes, native_binding_receipt_bytes)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1275) -> run_atomic_survey_population (:1355; graph_atomic_survey_population.py:137) -> run_authenticated_survey_population (:158; graph_survey_population.py:949) -> survey_population_preparation.prepare_authenticated_survey_population (:1892) -> acs_native.issue_acs_native_coverage(root/'acs', serialnos=acs_keys) (:1962) -> binding.issue_acs_native_coverage (:558) -> coverage.load_authenticated_acs_person_coverage (:614; auth.py:712) -> :804; native_binding at binding.py:627 and auth.py:695 on every verify", + "classification": "stays", + "notes": "Fixed-shape, 177x headroom. Base and head identical (no diff at :36). The default cap only matters for `_json(value)` calls without a cap and there are none on the runner path (auth :597,:676,:760,:804; binding :477; catalogue :205,:234,:443,:452,:486,:516,:590 all pass one)." + }, + { + "constant": "acs_person_coverage_authentication.py:37 MAX_BODY_BYTES [coverage body: one NDJSON line per selected ACS person]", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "HEAD acs_person_coverage_authentication.py:763-766: `if current and len(current) + len(raw) > MAX_BODY_BYTES:` start a new segment; `_require(len(current) + len(raw) <= MAX_BODY_BYTES, \"BODY_SIZE\")` (one line larger than one accumulation)", + "HEAD acs_person_coverage_authentication.py:767 `_require(size + len(raw) <= MAX_ROSTER_BYTES, \"BODY_LIMIT\")` and :772 `_require(len(body) == size, \"BODY_LIMIT\")` (the total, under MAX_ROSTER_BYTES)", + "HEAD acs_person_coverage_authentication.py:163/:178 `_json_roster(segment=MAX_BODY_BYTES)` -> \"CANONICAL_SIZE\" when one token exceeds a segment (used by binding.py:724)", + "BASE acs_person_coverage_authentication.py:661 `_require(size <= MAX_BODY_BYTES, \"BODY_SIZE\")` on the whole body (removed by this branch)", + "recorded: :269 producer ceilings" + ], + "refusal_codes": [ + "BODY_SIZE (ACSCoverageAuthenticationError, :766)", + "BODY_LIMIT (:767, :772)", + "CANONICAL_SIZE (:178 through _json_roster)" + ], + "encodes": "Per-row roster: ACS persons (selected). Each line is `_json([SERIALNO, SPORDER, AGEP, MIL, ESR, MIL_state, ESR_state, source_member, source_row_ordinal], MAX_RECORD_BYTES - 1) + b\"\\n\"` (auth.py:759-762).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "30,894,098 (342,288 lines x measured mean 90.26 B; scaled from the measured full-source mean)", + "bytes_at_full_source": "308,941,698 MEASURED (byte-census-measurements.json acs_person_archive.body_bytes, 3,422,888 lines, longest 102) = 4.60 x 64 MiB, 5 segments; 0.072 x MAX_ROSTER_BYTES", + "measurement_basis": "cited byte-census-measurements.json (whole captured archive through auth._json); cross-checked on invented rows at full-source widths through auth._json at this head: 89 B (observed_code, 1-digit SPORDER, 7-digit ordinal), 100 B (outside_age_universe), 90 B (2-digit SPORDER)", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "64 MiB on the whole body (base auth.py:661 `_require(size <= MAX_BODY_BYTES, \"BODY_SIZE\")`): binds at full source (4.60x), not at 1/10 (0.46x)", + "head_value": "64 MiB per accumulation segment (:763-766); total under MAX_ROSTER_BYTES 4 GiB (:767) = 0.072x at full source", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1275) -> run_atomic_survey_population (:1355; graph_atomic_survey_population.py:137) -> run_authenticated_survey_population (:158; graph_survey_population.py:949) -> survey_population_preparation.prepare_authenticated_survey_population (:1892) -> acs_native.issue_acs_native_coverage(root/'acs', serialnos=acs_keys) (:1962) -> binding.issue_acs_native_coverage (:558) -> coverage.load_authenticated_acs_person_coverage (:614; auth.py:712) -> literal.read_acs_person_coverage_columns (:747) -> body loop :757-771", + "classification": "segment", + "notes": "Already segmented on this branch (brief section 5). The bytes are still one string: `body = b\"\".join(segments)` (:771) and `payload = _MAGIC + len(raw_header).to_bytes(4, \"big\") + raw_header + body` (:805); the payload is retained in `_Owned.literal.payload` and digested at binding.py:515/:720. Consumer-side re-parse with no cap: `AuthenticatedACSPersonCoverage.table` (auth.py:664-671, `rows = [json.loads(row) for row in self._parts()[1].splitlines()]` into a DataFrame) runs at issuance (auth.py:702 via binding.py:522 `_verify`) and at every verify_acs_native_coverage (preparation.py:1789, current_survey_predictors.py:99)." + }, + { + "constant": "acs_person_coverage_authentication.py:47 MAX_ROSTER_BYTES [pre-allocation charge SELECTED_BODY_BUDGET]", + "value": "64 * MAX_BODY_BYTES = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "HEAD acs_person_coverage_authentication.py:475-479 `selected_budget += _body_row_bound(cells, member.filename, row_count)`; `_require(selected_budget <= MAX_ROSTER_BYTES, \"SELECTED_BODY_BUDGET\")`, per selected row, fresh per _inventory call (role household and role person)", + "BASE acs_person_coverage_authentication.py:377-382 `selected_budget += 6 * len(raw) + 1024`; `_require(selected_budget <= MAX_BODY_BYTES, \"SELECTED_BODY_BUDGET\")`", + "callers: acs_native_coverage_binding.py:337 and :351 (_preflight, both roles) and acs_person_coverage_authentication.py:733 and :736 (load_authenticated, both roles): four charges per issuance; acs_population_catalogue.py:155 passes frozenset() so its charge is 0", + "recorded: :270 producer ceilings" + ], + "refusal_codes": [ + "SELECTED_BODY_BUDGET (ACSCoverageAuthenticationError, :479)" + ], + "encodes": "A running upper bound on the coverage body's bytes, per selected row: `len(json.dumps([*cells, member, ordinal], sort_keys, separators, ensure_ascii=True)) + _STATE_BYTES + 1` (:201-213). Person role: roster ACS persons; household role: roster ACS households (cells [SERIALNO, NP]). No document is allocated; the sum guards the DataFrame the low-level reader allocates.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "person role 38,652,211 (scaled); household role ~15,142,084 (ESTIMATE). Base charge: person 1,793,778,148 (26.7 x 64 MiB), household 709,514,338 (10.6x)", + "bytes_at_full_source": "person role 386,523,011 (DERIVED exactly: measured cells_json_bytes 198,264,171 + 54 x 3,422,888 + 3,422,888) = 0.090 x 4 GiB; household role ~151,421,240 (ESTIMATE, 93 B + ordinal digits per household). Base charge: person 17,937,823,406 MEASURED (267 x 64 MiB), household occupied 7,095,161,910 MEASURED (106x)", + "measurement_basis": "auth._body_row_bound on invented rows at full-source widths (113 B per person row; 93 B + ordinal digits per household row) and the measured components in byte-census-measurements.json (cells_json_bytes, old_charge_bytes, old_charge_bytes_occupied); household-role total sums ordinal digits exactly over the two member files (827,133 + 804,836 rows) scaled to the non-vacant share", + "binds_at_tenth": true, + "binds_at_full_source": true, + "base_value": "MAX_BODY_BYTES 64 MiB with charge 6*len(raw)+1024 (base auth.py:377-382): binds at 1/10 (person 26.7x, household 10.6x) and at full (267x, 106x)", + "head_value": "MAX_ROSTER_BYTES 4 GiB with the exact line bound: 0.090x (person), 0.035x (household) at full source", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1275) -> run_atomic_survey_population (:1355) -> run_authenticated_survey_population (graph_survey_population.py:949) -> prepare_authenticated_survey_population (survey_population_preparation.py:1892) -> acs_native.issue_acs_native_coverage (:1962) -> binding._preflight (:585 -> :337/:351 coverage._inventory) and coverage.load_authenticated_acs_person_coverage (:614 -> auth.py:733/:736 _inventory) -> :475-479", + "classification": "stays", + "notes": "Moved on this branch: both the charge formula and its ceiling. At head it is a counter that takes the body's ceiling and needs no transport of its own; 11.1x headroom (person role) and 28x (household role). The charge is at least the body line it guards: SPORDER is charged as the record's string (2 bytes more than the int the line carries) and both states as 27 bytes each (`_STATE_BYTES = 2 * (2 + 24 + 1)`, :187) against the 16/23 the measured file needs." + }, + { + "constant": "acs_person_coverage_authentication.py:47 MAX_ROSTER_BYTES [selected SERIALNO list pre-check, binding.py:578]", + "value": "64 * MAX_BODY_BYTES = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "HEAD acs_native_coverage_binding.py:578 `coverage._json_size(serialnos, coverage.MAX_ROSTER_BYTES)` -> acs_person_coverage_authentication.py:93 \"CANONICAL_SIZE\" (sized, never encoded)", + "BASE acs_native_coverage_binding.py:573-575 `coverage._json(serialnos, min(MAX_EVIDENCE_BYTES, housing.ACS_HU_RECEIPT_MAX_BYTES))` = 1 MiB, materialised" + ], + "refusal_codes": [ + "CANONICAL_SIZE (ACSCoverageAuthenticationError :93, surfaced by binding.py:749 as ACSNativeCoverageBindingError(\"NATIVE_ISSUANCE_REFUSED\") with the cause chained)" + ], + "encodes": "Per-row roster: selected ACS households, the tuple of 13-character SERIALNO keys the runner passes (`serialnos=acs_keys`, survey_population_preparation.py:1946-1963); 16 canonical bytes per key (`\"2024HU0000001\",`) + 1.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "2,450,577 (= 16 x 153,161 + 1)", + "bytes_at_full_source": "24,505,825 (= 16 x 1,531,614 + 1; matches byte-census-measurements.json serialno_list_bytes_full_source) = 0.0057 x 4 GiB", + "measurement_basis": "auth._json_size on invented 13-char keys at 1,000 and 2,000 rows (per entry 16.0, fixed 1.0) at this head; full-source figure cited from byte-census-measurements.json", + "binds_at_tenth": true, + "binds_at_full_source": true, + "base_value": "min(MAX_EVIDENCE_BYTES 2 MiB, ACS_HU_RECEIPT_MAX_BYTES 1 MiB) = 1,048,576 (base binding.py:573-575): binds at 1/10 (2.34x) and at full (23.4x); 65,535 keys was the most it admitted", + "head_value": "MAX_ROSTER_BYTES 4 GiB, sized without encoding: 0.0057x at full source", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial -> ... -> prepare_authenticated_survey_population (survey_population_preparation.py:1892) -> acs_native.issue_acs_native_coverage(root/'acs', serialnos=acs_keys) (:1962) -> acs_native_coverage_binding.py:578 (before any capture or producer work)", + "classification": "segment", + "notes": "The list is the roster part of the issuance receipt (row below); this pre-check takes that receipt's ceiling (`coverage.MAX_ROSTER_BYTES`, comment :574-577). The same bytes are encoded whole and uncapped at housing.py:605 `_sha(_json(chosen))` (separate row)." + }, + { + "constant": "acs_person_coverage_authentication.py:47 MAX_ROSTER_BYTES [raw_person_keys_sha256 digest, binding.py:656-659]", + "value": "64 * MAX_BODY_BYTES = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "HEAD acs_native_coverage_binding.py:656-659 `coverage._json_sha256(sorted(zip(keys.SERIALNO, map(int, keys.SPORDER), strict=True)), coverage.MAX_ROSTER_BYTES)` -> acs_person_coverage_authentication.py:155-160 (_json_sha256) -> :93/:147 \"CANONICAL_SIZE\"", + "BASE acs_native_coverage_binding.py:641-647 `coverage._sha(coverage._json(sorted(...), coverage.MAX_BODY_BYTES))` (materialised, 64 MiB)" + ], + "refusal_codes": [ + "CANONICAL_SIZE (ACSCoverageAuthenticationError :93/:147, surfaced as NATIVE_ISSUANCE_REFUSED at binding.py:749)" + ], + "encodes": "Per-row roster: selected ACS persons, one `[\"2024HU0000001\",1]` pair per person (20 bytes with a 1-digit SPORDER, 21 with two digits).", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "6,846,175 (20 x 342,288 + 414 two-digit rows + 1)", + "bytes_at_full_source": "68,461,900 MEASURED (byte-census-measurements.json acs_person_archive.key_list_bytes; 4,139 two-digit SPORDER rows) = 1.02 x 64 MiB, 0.016 x 4 GiB", + "measurement_basis": "auth._json_size on invented pairs at 1,000 and 2,000 rows (20.0 per pair, fixed 1.0) at this head; full-source figure cited from byte-census-measurements.json", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "MAX_BODY_BYTES 64 MiB, whole list materialised then hashed (base binding.py:641-647): binds at full (1.020x), not at 1/10 (0.10x)", + "head_value": "MAX_ROSTER_BYTES 4 GiB, digested token-at-a-time (`for part in _json_chunks(value, cap): digest.update(part)`, auth.py:158-159): 0.016x", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial -> ... -> prepare_authenticated_survey_population (survey_population_preparation.py:1892) -> acs_native.issue_acs_native_coverage (:1962) -> receipt construction acs_native_coverage_binding.py:644-721 -> :656-659", + "classification": "segment", + "notes": "Already streamed on this branch (a digest; no whole copy). `keys` comes from coverage._native_roster(prepared.frame) (binding.py:604), which itself streams one `_json(row, MAX_RECORD_BYTES - 1) + b\"\\n\"` per person into a sha256 with no total cap (auth.py:584-598)." + }, + { + "constant": "acs_person_coverage_authentication.py:47 MAX_ROSTER_BYTES [native issuance receipt, binding.py:724 _json_roster]", + "value": "64 * MAX_BODY_BYTES = 4,294,967,296 (4 GiB); segments of MAX_BODY_BYTES 64 MiB", + "enforced_at": [ + "HEAD acs_native_coverage_binding.py:724 `payload = coverage._json_roster(receipt)` -> acs_person_coverage_authentication.py:163-181: :174 `_json_chunks(value, maximum)` (\"CANONICAL_SIZE\" above 4 GiB, :93/:147) and :178 `_require(len(current) + len(part) <= segment, \"CANONICAL_SIZE\")` (one token larger than 64 MiB)", + "BASE acs_native_coverage_binding.py:711 `payload = coverage._json(receipt, MAX_EVIDENCE_BYTES)` (2 MiB, materialised)", + "consumers (no cap): acs_native_coverage_binding.py:388 `json.loads(_owned(self).payload)` (.receipt), :491 and :503 (`_verify_frame`, `_verify`), :411 payload equality; survey_population_preparation.py:1983 `native_receipts = {\"acs\": actual_acs.receipt, ...}` (parsed whole, only the asec side is read by _origins :1157-1162); current_survey_predictors.py:100-101 `json.loads(acs_payload)[\"vintage\"]`; candidate copy binding.py:728-737 (candidate_path is never passed by the runner)" + ], + "refusal_codes": [ + "CANONICAL_SIZE (ACSCoverageAuthenticationError, auth.py:93/:147/:178; surfaced as NATIVE_ISSUANCE_REFUSED at binding.py:749 with the cause chained)" + ], + "encodes": "Per-row roster: selected ACS households. The receipt (binding.py:644-721) embeds `\"requested_serialnos\": serialnos` (:651, 16 B per key) and `\"vacant_serialnos\"` (:660-662, `[]` on the runner path because the catalogue selection draws only non-vacant records) inside a fixed preamble of 20,265 bytes at this head (producer closure 17,532 of it).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "2,470,841 (= 20,265 + 16 x 153,161)", + "bytes_at_full_source": "24,526,089 (= 20,265 + 16 x 1,531,614) = 11.7 x 2 MiB, 0.0057 x 4 GiB; one segment", + "measurement_basis": "the real issuance receipt issued on invented archives at this head, re-encoded through auth._json_roster with the requested list swapped for 1,000 and 2,000 invented keys (per key 16.0; fixed preamble 20,265) -- scratchpad measure_acs_family.py part 2 `issuance_receipt`", + "binds_at_tenth": true, + "binds_at_full_source": true, + "base_value": "MAX_EVIDENCE_BYTES 2 MiB (base binding.py:711): binds at 1/10 (1.18x) and at full (11.7x); 65,535 selected households (4.28% of source) was the most the issuer admitted", + "head_value": "MAX_ROSTER_BYTES 4 GiB in 64 MiB segments: 0.0057x at full source", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1275) -> run_atomic_survey_population (:1355) -> run_authenticated_survey_population (graph_survey_population.py:949) -> prepare_authenticated_survey_population (survey_population_preparation.py:1892) -> acs_native.issue_acs_native_coverage (:1962) -> acs_native_coverage_binding.py:724; parsed whole again at preparation.py:1983 and, at graph time, current_survey_predictors.py:99-101 (survey_predictors.source_projection node)", + "classification": "segment", + "notes": "Already segmented on this branch; the bytes are one string (`return b\"\".join(segments)`, auth.py:181) retained as `_Owned.payload` and as `AuthenticatedACSNativeCoverage.payload`, and `json.loads` of the whole 24.5 MB string happens on every borrow that reads `.receipt`. Base-time preamble differed slightly (the 1/1000 artifact's producer closure was 17,610 B; this head's is 17,532 B)." + }, + { + "constant": "acs_person_coverage_authentication.py:48 MAX_RECORD_BYTES", + "value": "400_000", + "enforced_at": [ + "acs_person_coverage_authentication.py:290 `cap = MAX_CSV_HEADER_BYTES if first else MAX_RECORD_BYTES` -> :298 and :302 `_require(length < cap, \"CSV_RECORD_BYTES\")` (raw logical CSV record, replayed only above :291's fast path)", + "acs_person_coverage_authentication.py:597 `_json(row, MAX_RECORD_BYTES - 1)` per native-roster row (streamed digest) -> :93/:147 \"CANONICAL_SIZE\"", + "acs_person_coverage_authentication.py:760 `_json([*row, *lineage[...]], MAX_RECORD_BYTES - 1)` per coverage body line -> \"CANONICAL_SIZE\"", + "recorded: :271" + ], + "refusal_codes": [ + "CSV_RECORD_BYTES (ACSCoverageAuthenticationError, :298/:302)", + "CANONICAL_SIZE (:93/:147 for the :597 and :760 sites)" + ], + "encodes": "Width: one raw logical CSV record of an ACS member, and separately one canonical row (native roster row ~40 B; body line <= 102 B).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "longest raw record 877 (measured raw_max); longest body line 102", + "bytes_at_full_source": "877 / 102, unchanged by fraction (per-record)", + "measurement_basis": "cited byte-census-measurements.json (acs_person_archive.raw_max 877, body_row_max 102); the row-count lane measured the longest logical record in either archive as the 1,882-byte person header (first record, under MAX_CSV_HEADER_BYTES)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding._preflight (:585) and coverage.load_authenticated_acs_person_coverage (:614) -> auth._inventory (:426) -> _records (:447) -> _fence (:366/:373) -> :290-302; body line :760; native roster :597 via binding.py:604/:823 and auth.py:724", + "classification": "must-not-move", + "notes": "Per-record encoder width; does not scale with the roster. `_fence` replays only records longer than `min(cap, MAX_TOKEN_BYTES)` (:291), so with 877-byte records the byte loop never runs." + }, + { + "constant": "acs_person_coverage_authentication.py:49 MAX_CSV_HEADER_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "acs_person_coverage_authentication.py:290 (`cap` for the first record of a member) -> :298/:302 \"CSV_RECORD_BYTES\"", + "recorded: :272" + ], + "refusal_codes": [ + "CSV_RECORD_BYTES (ACSCoverageAuthenticationError, :298/:302)" + ], + "encodes": "Width: the CSV header line of one archive member (1,882 B person, 1,570 B household per the row-count lane's measurement).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "1,882", + "bytes_at_full_source": "1,882, unchanged by fraction (upstream header)", + "measurement_basis": "cited experiments/native-row-ceilings/census.json row acs_person_coverage_authentication.py:39 (measured header widths 1,882 / 1,570)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth._inventory (:426) -> _records (:447) -> _fence(record, quoted, first=True) (:366) -> :290", + "classification": "must-not-move", + "notes": "Protects an upstream file's header width." + }, + { + "constant": "acs_person_coverage_authentication.py:50 MAX_TOKEN_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "acs_person_coverage_authentication.py:291 fast path `if len(record) <= min(cap, MAX_TOKEN_BYTES): return`", + "acs_person_coverage_authentication.py:307 `_require(token_bytes <= MAX_TOKEN_BYTES, \"CSV_TOKEN_BYTES\")`", + "recorded: :273" + ], + "refusal_codes": [ + "CSV_TOKEN_BYTES (ACSCoverageAuthenticationError, :307)" + ], + "encodes": "Width: one raw CSV field including quoting (longest field in either archive is the 13-byte SERIALNO).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13", + "bytes_at_full_source": "13, unchanged by fraction", + "measurement_basis": "cited row-count census (longest field 13; zero quote characters in either archive); serialno_widths {13: 1,631,969} in byte-census-measurements.json", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth._inventory -> _records -> _fence -> :291/:307", + "classification": "must-not-move", + "notes": "Token/field width." + }, + { + "constant": "acs_person_coverage_authentication.py:51 _SCAN_BLOCK", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_person_coverage_authentication.py:332 `while block := stream.read(_SCAN_BLOCK):` (read block size in _records; no refusal)" + ], + "refusal_codes": [ + "none (a read block, not a check)" + ], + "encodes": "Memory: the block size the byte fence scans; a record spans at most two blocks (docstring :326-329). No document behind it.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "1,048,576 per read, unchanged", + "bytes_at_full_source": "1,048,576 per read, unchanged", + "measurement_basis": "code read only", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth._inventory (:426) -> _records (:447) -> :332", + "classification": "not-a-transport", + "notes": "A memory ceiling with no document behind it; listed for completeness." + }, + { + "constant": "acs_person_coverage_authentication.py:187 _STATE_BYTES", + "value": "2 * (2 + 24 + 1) = 54", + "enforced_at": [ + "acs_person_coverage_authentication.py:211 (`+ _STATE_BYTES` inside _body_row_bound, the per-row pre-allocation charge)" + ], + "refusal_codes": [ + "none directly; contributes to SELECTED_BODY_BUDGET (:479)" + ], + "encodes": "Width: the two derived state cells of one body line, charged at the longest state `value_below_age_universe` (24 chars) plus quotes and separator; the measured file's cells are `observed_code` (16 B) and `outside_age_universe` (23 B).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "54 per row, unchanged", + "bytes_at_full_source": "54 per row, unchanged (184,835,952 summed over 3,422,888 rows inside the charge)", + "measurement_basis": "code read; state vocabulary from acs_person_coverage_columns._field_state (:119-126); measured state counts in byte-census-measurements.json (observed_code 5,704,812; outside_age_universe 1,140,964)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth._inventory (:426) -> _body_row_bound (:475) -> :211", + "classification": "must-not-move", + "notes": "Must stay at least the widest state's encoded width or the charge stops bounding the body (test test_body_row_bound_covers_every_body_line)." + }, + { + "constant": "acs_person_coverage_authentication.py:442 member.file_size read cap", + "value": "`stream.read(min(4096, member.file_size - count + 1))` (non-applicable members) and `count <= member.file_size` after every record (applicable members); member.file_size from the ZIP central directory (largest 1,226,543,489)", + "enforced_at": [ + "acs_person_coverage_authentication.py:442-444 `_require(count <= member.file_size, \"ZIP_MEMBER_GREW\")`", + "acs_person_coverage_authentication.py:448-449 (applicable members, per raw record)", + "acs_person_coverage_authentication.py:497 `_require(count == member.file_size, \"ZIP_MEMBER_SIZE_MISMATCH\")`", + "acs_person_coverage_authentication.py:415 `0 <= member.file_size <= custody._MEMBER_MAX` (\"ZIP_MEMBER_SIZE\", see the _MEMBER_MAX row)" + ], + "refusal_codes": [ + "ZIP_MEMBER_GREW (ACSCoverageAuthenticationError, :444/:449)", + "ZIP_MEMBER_SIZE_MISMATCH (:497)" + ], + "encodes": "Upstream: the declared uncompressed size of each ZIP member (psam_pusa 1,226,543,489; psam_pusb 1,178,924,624; psam_husa 495,752,055; psam_husb 481,732,465; README 103,658).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "3,383,159,949 across the six members, unchanged", + "bytes_at_full_source": "3,383,159,949, unchanged", + "measurement_basis": "cited row-count census (member sizes read from the captured archives' central directory); byte-census-measurements.json raw_bytes 977,481,380 + 2,405,464,349 for the four data members", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding._preflight (:585) and coverage.load_authenticated_acs_person_coverage (:614) -> auth._inventory (:426) -> :440-449, :497", + "classification": "must-not-move", + "notes": "The read(N+1) pattern: a member that grows past its declared size refuses. The 4096 is a chunk size for non-applicable members only." + }, + { + "constant": "acs_person_coverage_authentication.py:805 4-byte header length", + "value": "`len(raw_header).to_bytes(4, \"big\")` -> at most 2**32 - 1 = 4,294,967,295", + "enforced_at": [ + "acs_person_coverage_authentication.py:805 (OverflowError from int.to_bytes above 2**32-1; MAX_HEADER_BYTES 1 MiB at :804 precludes it)", + "consumer acs_person_coverage_authentication.py:654-657 `_parts`: `size = int.from_bytes(self.payload[len(_MAGIC) : start], \"big\")`" + ], + "refusal_codes": [ + "none of this module's; an OverflowError would be caught by :831 and re-raised as SOURCE_RECONSTRUCTION_REFUSED" + ], + "encodes": "Width: the length prefix of the coverage envelope header.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "4 (field); header 5,915", + "bytes_at_full_source": "4; header 5,915, unchanged", + "measurement_basis": "code read; header size from part 2 of the measurement script", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> coverage.load_authenticated_acs_person_coverage (:614; auth.py:712) -> :805; read back by `.receipt`/`.table` (:659-671)", + "classification": "must-not-move", + "notes": "Encoder width; the header is fixed-shape (MAX_HEADER_BYTES row)." + }, + { + "constant": "acs_native_coverage_binding.py:28 MAX_ARCHIVE_BYTES", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "acs_native_coverage_binding.py:581 `_require(sum(p[3] for p in pins) <= MAX_ARCHIVE_BYTES, \"ARCHIVE_BUDGET\")`", + "recorded: :274 (producer limits)" + ], + "refusal_codes": [ + "ARCHIVE_BUDGET (ACSNativeCoverageBindingError, :581)" + ], + "encodes": "Upstream: the two pinned archives' declared size_bytes (csv_hus.zip 251,500,587 + csv_pus.zip 602,847,146).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "854,347,733, unchanged", + "bytes_at_full_source": "854,347,733, unchanged (0.099x)", + "measurement_basis": "acs_2024_1yr_sources.json size_bytes, read this session (housing._pins())", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (survey_population_preparation.py:1962) -> :579 housing._pins() -> :581", + "classification": "must-not-move", + "notes": "Protects a real file size; the catalogue has its own twin (_ARCHIVE_BYTES) and survey_population_preparation.py:551 checks the on-disk sizes against that twin." + }, + { + "constant": "acs_native_coverage_binding.py:29 MAX_EXPANDED_BYTES", + "value": "16 * 1024**3 = 17,179,869,184", + "enforced_at": [ + "acs_native_coverage_binding.py:314-319 `expanded += sum(member.file_size for member in members)`; `_require(expanded <= MAX_EXPANDED_BYTES, \"EXPANDED_BUDGET\")` (combined, both archives, before any member is opened)", + "recorded: :275" + ], + "refusal_codes": [ + "EXPANDED_BUDGET (ACSNativeCoverageBindingError, :319)" + ], + "encodes": "Upstream: total declared uncompressed member bytes of both archives.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "3,383,159,949, unchanged", + "bytes_at_full_source": "3,383,159,949, unchanged (0.197x)", + "measurement_basis": "cited row-count census (member sizes from the captured archives' central directories)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (:1962) -> binding._preflight (:585) -> :314-319", + "classification": "must-not-move", + "notes": "Decompression-bomb gate on real upstream sizes." + }, + { + "constant": "acs_native_coverage_binding.py:30 MAX_SOURCE_ROWS", + "value": "6_000_000 (a row count)", + "enforced_at": [ + "acs_native_coverage_binding.py:325-328, :340-343, :354-357 `sum(m[\"rows\"] for m in inventories[role]) <= MAX_SOURCE_ROWS` -> \"SOURCE_ROW_BUDGET\"", + "recorded: :276" + ], + "refusal_codes": [ + "SOURCE_ROW_BUDGET (ACSNativeCoverageBindingError)" + ], + "encodes": "Row count of the upstream archive per role (household 1,631,969; person 3,422,888), not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a (rows: 1,631,969 / 3,422,888, unchanged)", + "bytes_at_full_source": "n/a (rows: 1,631,969 / 3,422,888, unchanged)", + "measurement_basis": "preparation.json /catalogues/acs/counts (households 1631969, people 3422888)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding._preflight (:585) -> :326/:341/:355", + "classification": "not-a-transport", + "notes": "A row count; the row-count lane's upstream-file-size rule applies (census.json row acs_native_coverage_binding.py:30). Same number as acs_person_coverage_columns.MAX_ROWS and acs_population_catalogue._SOURCE_ROWS." + }, + { + "constant": "acs_native_coverage_binding.py:39 MAX_EVIDENCE_BYTES", + "value": "2 * 1024**2 = 2,097,152", + "enforced_at": [ + "HEAD acs_native_coverage_binding.py:477 `return coverage._sha(coverage._json(details, MAX_EVIDENCE_BYTES))` (_frame_sha256) -> auth.py:93/:147 \"CANONICAL_SIZE\"", + "BASE acs_native_coverage_binding.py:573-575 (serialno pre-check, min with ACS_HU_RECEIPT_MAX_BYTES) and :711 (issuance receipt) -- both moved to coverage.MAX_ROSTER_BYTES on this branch", + "recorded: :277" + ], + "refusal_codes": [ + "CANONICAL_SIZE (ACSCoverageAuthenticationError auth.py:93/:147; surfaced as NATIVE_ISSUANCE_REFUSED :749 or NATIVE_VERIFICATION_REFUSED :555)" + ], + "encodes": "Fixed-shape: the frame details document of _frame_sha256 (per entity: index type/dtype, column axis, dtypes list, attrs, allows_duplicate_labels, weight kind/dtype/shape/sha256; strata attrs). O(columns), never O(rows).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "3,745 (this head, the real column set)", + "bytes_at_full_source": "3,745 (weight_shape digits aside), unchanged (0.0018x)", + "measurement_basis": "captured the len of coverage._json(details, MAX_EVIDENCE_BYTES) by wrapping auth._json around binding._frame_sha256 on the invented frame after issuance (measure_acs_family.py part 2 frame_sha256_details_bytes); the frame's columns are the same 43+18+2+2+1+1 the real builder produces", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "2 MiB fronting the issuance receipt (:711) and, through min(), the 1 MiB serialno pre-check (:573-575): those documents bound at 1/10 and at full (recorded on their own rows)", + "head_value": "2 MiB fronting only the fixed _frame_sha256 details: 560x headroom", + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (:1962) -> :716 `\"exact_frame_sha256\": _frame_sha256(prepared.frame)`, :738 _verify -> _verify_frame (:494); survey_population_preparation.py:1970 and :1774 `acs_native._frame_sha256(frame)` (source seals); every verify_acs_native_coverage (:534 -> :548/:550)", + "classification": "stays", + "notes": "Fixed-shape at head; the binding rows carry the base-time bindings. `_require(not frame._link_tables, \"UNEXPECTED_LINK_TABLES\")` (:476) precedes the encode." + }, + { + "constant": "acs_native_coverage_binding.py:54 _COMPILE_CACHE_MAX_SOURCE_BYTES", + "value": "16 * 1024**2 = 16,777,216", + "enforced_at": [ + "acs_native_coverage_binding.py:107 eligibility (`len(source) <= _COMPILE_CACHE_MAX_SOURCE_BYTES`, bypass only)", + "acs_native_coverage_binding.py:152-157 FIFO eviction while retained source bytes + this source exceed it" + ], + "refusal_codes": [ + "none (silent cache bypass / eviction)" + ], + "encodes": "Python module source bytes retained as cache keys for the producer drift check; not survey data.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "1,844,965 retained across 69 entries after one issuance at this head", + "bytes_at_full_source": "1,844,965, unchanged", + "measurement_basis": "read binding._COMPILE_CACHE after the invented issuance (measure_acs_family.py part 2 compile_cache)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding._producer (:582/:739) -> _live_code (:268) -> _compile_source (:215) -> :107/:152-157", + "classification": "not-a-transport", + "notes": "A memory ceiling with no document behind it. Siblings: _COMPILE_CACHE_MAX_ENTRIES = 128 (:53; :108 eligibility, :153 eviction; 69 entries observed) and _COMPILE_CACHE_MAX_ENTRY_BYTES = 1 MiB (:55; :106 eligibility only; largest source 135,278 B observed). All three: no refusal code." + }, + { + "constant": "acs_person_coverage_columns.py:39 MAX_ROWS", + "value": "6_000_000 (a row count)", + "enforced_at": [ + "acs_person_coverage_columns.py:198-199 `if rows > MAX_ROWS: raise ValueError(\"ACS coverage source row bound exceeded\")`", + "acs_person_coverage_authentication.py:465 `_require(rows <= literal.MAX_ROWS, \"SOURCE_ROWS\")`", + "recorded: acs_person_coverage_authentication.py:264" + ], + "refusal_codes": [ + "ValueError(\"ACS coverage source row bound exceeded\") (columns.py:199)", + "SOURCE_ROWS (ACSCoverageAuthenticationError, auth.py:465)" + ], + "encodes": "Upstream person-file rows (3,422,888) and, at auth.py:465, either role's rows; not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a (rows 3,422,888 person / 1,631,969 household, unchanged)", + "bytes_at_full_source": "n/a (same)", + "measurement_basis": "preparation.json /catalogues/acs/counts", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth._inventory (:465) via binding._preflight and load_authenticated; literal._scan_acs_person_coverage (:198) via auth.py:747 read_acs_person_coverage_columns and acs_population_catalogue.py:300", + "classification": "not-a-transport", + "notes": "Row count wearing no byte name; the row-count lane's upstream-file-size rule applies." + }, + { + "constant": "acs_person_coverage_columns.py:40 MAX_SELECTED_ROWS", + "value": "14_000_000 (a row count)", + "enforced_at": [ + "acs_person_coverage_columns.py:239-240 `if not 0 < len(expected) <= MAX_SELECTED_ROWS: raise ValueError(\"ACS coverage selected person count is outside the bound\")`", + "acs_person_coverage_authentication.py:494 `_require(len(selected) < literal.MAX_SELECTED_ROWS, \"SELECTED_ROWS\")` (strict)", + "acs_person_coverage_authentication.py:517 `_require(0 < len(h) <= len(p) <= literal.MAX_SELECTED_ROWS, \"NATIVE_ROWS\")`", + "acs_native_coverage_binding.py:330-333 (\"NATIVE_ROW_BUDGET\", serialnos-is-None route only) and :348 `_require(expected_rows <= coverage.literal.MAX_SELECTED_ROWS, \"NATIVE_ROW_BUDGET\")`", + "acs_population_catalogue.py:315-318 `count <= min(_BATCH_PEOPLE, literal.MAX_SELECTED_ROWS)` (\"BATCH_PERSON_LIMIT\")", + "recorded: acs_person_coverage_authentication.py:265" + ], + "refusal_codes": [ + "ValueError (columns.py:240)", + "SELECTED_ROWS (auth.py:494)", + "NATIVE_ROWS (auth.py:517)", + "NATIVE_ROW_BUDGET (binding.py:333/:348)", + "BATCH_PERSON_LIMIT (catalogue.py:318)" + ], + "encodes": "Selected ACS person rows (3,422,888 at full source); not bytes.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "n/a (rows 342,288)", + "bytes_at_full_source": "n/a (rows 3,422,888; 4.09x headroom)", + "measurement_basis": "brief section 3 counts", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding._preflight (:348) ; auth._inventory (:494) ; auth._native_roster (:517) via binding.py:604/:823 and auth.py:724 ; literal.read_acs_person_coverage_columns (:239) via auth.py:747", + "classification": "not-a-transport", + "notes": "Lifted 1,000,000 -> 14,000,000 on native-row-ceilings (docs/us-native-row-ceilings.md section 3); unchanged on this branch." + }, + { + "constant": "acs_person_coverage_columns.py:41 MAX_CSV_RECORD_CHARS", + "value": "100_000", + "enforced_at": [ + "acs_person_coverage_columns.py:138 `stream.readline(MAX_CSV_RECORD_CHARS - record_chars + 1)` (read cap) and :140-141 `raise ValueError(\"ACS coverage CSV record character bound exceeded\")`", + "recorded: acs_person_coverage_authentication.py:263" + ], + "refusal_codes": [ + "ValueError(\"ACS coverage CSV record character bound exceeded\") (columns.py:141); inside the owners it surfaces as SOURCE_RECONSTRUCTION_REFUSED (auth.py:832) / CATALOGUE_ISSUANCE_REFUSED (catalogue.py:607)" + ], + "encodes": "Width: decoded characters of one logical CSV record including its line ending (record_chars resets per record, :151).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "1,882 worst case (the header)", + "bytes_at_full_source": "1,882, unchanged", + "measurement_basis": "cited row-count census (longest logical record in either archive)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth.load_authenticated_acs_person_coverage (:747) -> literal.read_acs_person_coverage_columns -> _scan_acs_person_coverage (:256) -> _literal_csv_records (:187) -> :138-141; also auth._decode_record (:383) per raw record, and catalogue._collect (:300)", + "classification": "must-not-move", + "notes": "Per-record width." + }, + { + "constant": "acs_person_coverage_columns.py:84 _json [native_roster_sha256, :292-294] (cap: none)", + "value": "none -- `json.dumps(value, sort_keys=True, separators=(\",\", \":\"), allow_nan=False).encode()` with no limit", + "enforced_at": [ + "acs_person_coverage_columns.py:292-294 `\"native_roster_sha256\": hashlib.sha256(_json(expected.to_dict(\"records\"))).hexdigest()` (whole list materialised, then hashed)" + ], + "refusal_codes": [ + "none" + ], + "encodes": "Per-row roster: selected ACS persons as records `{\"SERIALNO\":\"2024HU0000001\",\"SPORDER\":1}` (41 bytes each with a 1-digit SPORDER).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "14,034,223 (41 x 342,288 + 414 + 1)", + "bytes_at_full_source": "140,342,548 (41 x 3,422,888 + 4,139 + 1)", + "measurement_basis": "literal._json on invented records at 1,000 and 2,000 rows (41.0 per row) at this head; two-digit SPORDER share from byte-census-measurements.json", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding.issue_acs_native_coverage (:558) -> coverage.load_authenticated_acs_person_coverage (:614) -> literal.read_acs_person_coverage_columns (auth.py:747) -> columns.py:292-294; the receipt lands in the coverage header as original_literal_receipt (auth.py:791)", + "classification": "segment", + "notes": "No bound exists at all; the whole 140 MB string exists before the digest. Classified segment because the brief has no 'uncapped' class: a digest over iterencode (the shape auth._json_sha256 already has) would remove the copy. Recorded because the brief asks for every whole-roster encoding; the orchestrator may prefer to carry it as a note." + }, + { + "constant": "acs_person_coverage_columns.py:84 _json [projection_sha256, :295-297] (cap: none)", + "value": "none -- uncapped json.dumps", + "enforced_at": [ + "acs_person_coverage_columns.py:295-297 `\"projection_sha256\": hashlib.sha256(_json(result.to_dict(\"records\"))).hexdigest()`" + ], + "refusal_codes": [ + "none" + ], + "encodes": "Per-row roster: selected ACS persons as 7-key records (SERIALNO, SPORDER, AGEP, MIL, ESR, MIL_state, ESR_state); 129 bytes each at observed_code widths, +7 per outside_age_universe cell.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "~44,155,152 ESTIMATE (129 x 342,288; the measured 1/6 outside_age_universe cells add ~0.8 MB)", + "bytes_at_full_source": "~441,552,552 ESTIMATE (129 x 3,422,888; +~8 MB for the wider state cells)", + "measurement_basis": "literal._json on invented 7-key records at 1,000 and 2,000 rows (129.0 per row) at this head", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth.load_authenticated_acs_person_coverage (:747) -> literal.read_acs_person_coverage_columns -> columns.py:295-297", + "classification": "segment", + "notes": "The largest uncapped whole copy in the family (~440 MB at full source), alongside the 3.4M-row `observed` DataFrame (:261) and `result` (:268). Same caveat as the previous row." + }, + { + "constant": "acs_housing_universe_source.py:42 ACS_HU_SOURCE_MAX_BYTES [full lexical projection, whole file]", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "acs_housing_universe_source.py:355-364 streamed sum `selected_bytes += len(_json(row))`; `_require(selected_bytes <= ACS_HU_SOURCE_MAX_BYTES, \"PROJECTION_TOO_LARGE\")` -- every applicable row of one archive, whatever the selection", + "acs_housing_universe_source.py:573-574 `full_bytes = _json(full)`; `_write(private / \"full-projection.json\", full_bytes, ACS_HU_SOURCE_MAX_BYTES)` -> :180 `_require(len(data) <= cap, \"OUTPUT_SIZE\")`", + "acs_population_catalogue.py:497-499 `projection_raw = stream.read(housing.ACS_HU_SOURCE_MAX_BYTES + 1)`; `_require(len(projection_raw) <= housing.ACS_HU_SOURCE_MAX_BYTES, \"PROJECTION_SIZE\")` then `json.loads(projection_raw)` (:508)", + "acs_housing_universe_source.py:203 doubled into the disk demand" + ], + "refusal_codes": [ + "PROJECTION_TOO_LARGE (ACSHousingSourceError, :363)", + "OUTPUT_SIZE (:180)", + "PROJECTION_SIZE (ACSSourceCatalogueError, catalogue.py:499)" + ], + "encodes": "The whole-source lexical projection: one row per household record (1,631,969, incl. vacancies) `[SERIALNO, TYPEHUGQ, NP, TEN, WGTP, PUMA, ST, source_member, source_row_ordinal]` and one per person (3,422,888) `[SERIALNO, SPORDER, PWGTP, source_member, source_row_ordinal]`; identical at every fraction.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "292,484,053 (unchanged: _reconstruct always builds the whole source before selecting)", + "bytes_at_full_source": "292,484,053 MEASURED by the row-count lane on the pilot's full-projection.json = 0.034 x 8 GiB", + "measurement_basis": "cited experiments/native-row-ceilings/census.json row acs_housing_universe_source.py:42; reproduced through housing._json (canonical_json) on invented rows at full-source widths: 65 B + ordinal digits per occupied household row, 62 GQ, 64 vacant, 44 B + digits per person row at 3-digit WGTP/PWGTP -> 292,401,960 materialised (0.03% under the measured file), preamble 270", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> prepare_authenticated_survey_population -> acs_catalogue.issue_acs_source_catalogue (:1939) -> housing._reconstruct(private, paths, pins, None, ...) (catalogue.py:490) -> _archive (:564-565) and :573-574; catalogue reads it back whole at :495-508. Also -> acs_native.issue_acs_native_coverage (:1962) -> housing.prepare_acs_housing_population (binding.py:591) -> _reconstruct (housing.py:833)", + "classification": "stays", + "notes": "Built twice per preparation (catalogue path and native path) and written to each private capture; the catalogue then reads the 292 MB file whole and json.loads it (`del projection_raw` only after parsing, :509). 29x headroom; does not move with the fraction. `_tables(projection)` (:398-494) then materialises pandas frames of the whole source." + }, + { + "constant": "acs_housing_universe_source.py:42 ACS_HU_SOURCE_MAX_BYTES [selected projection]", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "acs_housing_universe_source.py:576-577 `projection = _json(selected)`; `_require(len(projection) <= ACS_HU_SOURCE_MAX_BYTES, \"PROJECTION_TOO_LARGE\")`", + "acs_housing_universe_source.py:641-673 (_run_source readback/publish caps for projection.json: \"RECONSTRUCTION_SIZE\"/\"OUTPUT_SIZE\") -- produce/load_acs_housing_source only, not on a runner path", + "consumers (no cap): acs_native_coverage_binding.py:519-521 `housing.verify_acs_frame_projection(frame, json.loads(prepared.source.projection_json))` on every _verify (issuance :738 and every verify_acs_native_coverage); acs_housing_universe_source.py:834 `json.loads(source.projection_json)` and :852/:889 verify_acs_frame_projection; :551-560 properties re-parse per access; digested at binding.py:513-514/:719", + "graph_acs_housing_universe.py:205 and :309 re-encode it under ACS_HU_PAYLOAD_MAX_BYTES 64 MiB (unreachable; see the _parse_json row)" + ], + "refusal_codes": [ + "PROJECTION_TOO_LARGE (ACSHousingSourceError, :577)" + ], + "encodes": "Per-row roster: selected ACS households (1,531,614 at full source: the full projection minus the 100,355 vacant rows) plus their ACS persons (3,422,888), same row shapes as the full projection, sorted by SERIALNO (:511-528).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "~28,529,285 ESTIMATE (preamble 270 + a tenth of the calibrated selected rows)", + "bytes_at_full_source": "~285,290,419 ESTIMATE (292,484,053 measured full projection minus the vacant household rows at calibrated widths) = 0.033 x 8 GiB", + "measurement_basis": "housing._json on invented rows at 3-digit WGTP/PWGTP widths (the variant that reproduces the measured full projection to 0.03%), vacant rows removed; see measure_acs_family.py part 1 lexical_projection.variants", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (:1962) -> housing.prepare_acs_housing_population (binding.py:591) -> _reconstruct (housing.py:833) -> _select (:575) -> :576-577; parsed whole at binding.py:520 on every verify", + "classification": "stays", + "notes": "30x headroom. Held whole in AuthenticatedACSHousingSource.projection_json and re-parsed into DataFrames (`_tables`, :398-494) at every verification: `json.loads(prepared.source.projection_json)` at binding.py:520." + }, + { + "constant": "acs_housing_universe_source.py:43 ACS_HU_RECEIPT_MAX_BYTES", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_housing_universe_source.py:615-616 `receipt_bytes = _json(receipt)`; `_require(len(receipt_bytes) <= ACS_HU_RECEIPT_MAX_BYTES, \"RECEIPT_TOO_LARGE\")` (source receipt)", + "acs_housing_universe_source.py:232 `_write(private / \"failure.json\", failure, ACS_HU_RECEIPT_MAX_BYTES)` -> :180 \"OUTPUT_SIZE\" (best effort on refusal)", + "acs_housing_universe_source.py:647 and :678 (_run_source candidate/publish caps; not on a runner path)", + "acs_housing_universe_source.py:204 doubled into the disk demand", + "graph_acs_housing_universe.py:202/:218/:294/:308/:310 (`_parse_json(..., ACS_HU_RECEIPT_MAX_BYTES)`; unreachable)", + "BASE acs_native_coverage_binding.py:573-575 `min(MAX_EVIDENCE_BYTES, housing.ACS_HU_RECEIPT_MAX_BYTES)` on the serialno tuple (moved to coverage.MAX_ROSTER_BYTES on this branch)" + ], + "refusal_codes": [ + "RECEIPT_TOO_LARGE (ACSHousingSourceError, :616)", + "OUTPUT_SIZE (:180)", + "JSON_SIZE / CANONICAL_JSON (graph sites, :82/:85)" + ], + "encodes": "Fixed-shape source receipt: parser profile, archives, six-member inventory including each data member's header_hex (3,140 / 3,764 hex chars) and header name list (241 / 286 names), full_counts, selected_counts, digests. No per-row data.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "26,076 (with the real header widths and six members; 3,258 on the invented archives)", + "bytes_at_full_source": "26,076, unchanged (0.025x)", + "measurement_basis": "the real source receipt issued on invented archives at this head, re-encoded through housing._json with the members inventory replaced by six entries carrying synthetic 1,570-byte/241-name and 1,882-byte/286-name headers (measure_acs_family.py part 2 source_receipt_bytes_REALISTIC_headers)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1 MiB, also fronting (through min()) the selected serialno list at base binding.py:573-575: bound at 1/10 (2.34x) and full (23.4x); recorded on that row", + "head_value": "1 MiB fronting only fixed-shape receipts", + "reachable_from": "19", + "reach_chain": "... -> housing._reconstruct (via catalogue.py:490 and binding.py:591 -> housing.py:833) -> :615-616; parsed by binding.py:618 `json.loads(prepared.source.receipt_json)` and digested :511/:718", + "classification": "stays", + "notes": "40x headroom. The graph's prepared-receipt parse at graph_acs_housing_universe.py:202 shares this constant and would refuse the prepared receipt above 65,535 selected households (see the housing.py:913 and _parse_json rows) -- unreachable today." + }, + { + "constant": "acs_housing_universe_source.py:44 _CHUNK", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_housing_universe_source.py:142 `stream.read(min(_CHUNK, cap - count + 1))` (_copy) with :144 \"FILE_GREW\"", + "acs_housing_universe_source.py:168 `stream.read(min(_CHUNK, size - count + 1))` (_persisted_sha) with :170 \"SNAPSHOT_GREW\"", + "acs_housing_universe_source.py:318 `stream.read(min(_CHUNK, member.file_size - count + 1))` (non-applicable members) with :323 \"ZIP_MEMBER_GREW\"" + ], + "refusal_codes": [ + "none from _CHUNK itself; the caps are the pinned sizes (see the size_bytes row) and member sizes" + ], + "encodes": "Memory: the read chunk of the read(N+1) loops; no document.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "1,048,576 per read, unchanged", + "bytes_at_full_source": "1,048,576 per read, unchanged", + "measurement_basis": "code read", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._capture (:190) -> _copy (:213) / _persisted_sha (:215); _archive (:311-318)", + "classification": "not-a-transport", + "notes": "Chunk size only." + }, + { + "constant": "acs_housing_universe_source.py:45 _MEMBER_MAX", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "acs_housing_universe_source.py:293 `_require(0 <= member.file_size <= _MEMBER_MAX, \"ZIP_MEMBER_SIZE\")`", + "acs_person_coverage_authentication.py:415 `_require(0 <= member.file_size <= custody._MEMBER_MAX, \"ZIP_MEMBER_SIZE\")`", + "recorded: acs_person_coverage_authentication.py:274" + ], + "refusal_codes": [ + "ZIP_MEMBER_SIZE (ACSHousingSourceError :293; ACSCoverageAuthenticationError auth.py:415)" + ], + "encodes": "Upstream: one ZIP member's declared uncompressed size (largest psam_pusa.csv 1,226,543,489).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "1,226,543,489, unchanged", + "bytes_at_full_source": "1,226,543,489, unchanged (0.143x)", + "measurement_basis": "cited row-count census (central directory of the captured archives)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._archive (:262, via _reconstruct) and auth._members (:389, via _inventory from binding._preflight/load_authenticated/catalogue._preflight :148)", + "classification": "must-not-move", + "notes": "Real upstream size." + }, + { + "constant": "acs_housing_universe_source.py:46 _EXPANDED_MAX", + "value": "16 * 1024**3 = 17,179,869,184", + "enforced_at": [ + "acs_housing_universe_source.py:294-295 `total_bytes += member.file_size`; `_require(total_bytes <= _EXPANDED_MAX, \"ZIP_EXPANDED_SIZE\")` (per archive)", + "acs_person_coverage_authentication.py:416-417 (per archive)", + "recorded: acs_person_coverage_authentication.py:275" + ], + "refusal_codes": [ + "ZIP_EXPANDED_SIZE (ACSHousingSourceError :295; ACSCoverageAuthenticationError auth.py:417)" + ], + "encodes": "Upstream: one archive's total declared member bytes (person 2,405,571,771; household 977,588,178).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "2,405,571,771 (larger archive), unchanged", + "bytes_at_full_source": "2,405,571,771, unchanged (0.14x)", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._archive (:262) and auth._members (:389)", + "classification": "must-not-move", + "notes": "Real upstream size; the binding's MAX_EXPANDED_BYTES and the catalogue's _EXPANDED_BYTES are the combined twins." + }, + { + "constant": "acs_housing_universe_source.py:47 _MEMBER_COUNT_MAX", + "value": "64 (a count)", + "enforced_at": [ + "acs_housing_universe_source.py:269 `_require(0 < len(members) <= _MEMBER_COUNT_MAX, \"ZIP_MEMBER_COUNT\")`", + "acs_person_coverage_authentication.py:395 (same, \"ZIP_MEMBER_COUNT\")", + "recorded: acs_person_coverage_authentication.py:276" + ], + "refusal_codes": [ + "ZIP_MEMBER_COUNT (ACSHousingSourceError :269; ACSCoverageAuthenticationError auth.py:395)" + ], + "encodes": "Count of ZIP central-directory entries per archive (3), not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a (3 members per archive, unchanged)", + "bytes_at_full_source": "n/a (3, unchanged)", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._archive (:262) and auth._members (:389)", + "classification": "not-a-transport", + "notes": "A count wearing the _MAX_ name; the row-count lane's rule applies." + }, + { + "constant": "acs_housing_universe_source.py:48 _RECORD_MAX", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_housing_universe_source.py:239 `_require(0 < len(raw) <= _RECORD_MAX, \"CSV_RECORD_SIZE\")`", + "acs_housing_universe_source.py:313-316 `stream.readline(min(_RECORD_MAX + 1, member.file_size - count + 1))` (read cap)", + "recorded in the receipt: :596 parser_profile.record_max_bytes" + ], + "refusal_codes": [ + "CSV_RECORD_SIZE (ACSHousingSourceError, :239)" + ], + "encodes": "Width: one physical CSV record including terminator (longest 1,882).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "1,882", + "bytes_at_full_source": "1,882, unchanged", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._reconstruct -> _archive (:564-565) -> :313-316 / _csv_record (:327 -> :239)", + "classification": "must-not-move", + "notes": "Per-record width; also recorded in the receipt's parser_profile (:589-597), so moving it changes issued receipts." + }, + { + "constant": "acs_housing_universe_source.py:49 _FIELD_MAX", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "acs_housing_universe_source.py:256-258 `_require(all(len(v.encode(\"utf-8\")) <= _FIELD_MAX for v in values), \"CSV_FIELD_SIZE\")`", + "recorded in the receipt: :595 parser_profile.field_max_bytes" + ], + "refusal_codes": [ + "CSV_FIELD_SIZE (ACSHousingSourceError, :257)" + ], + "encodes": "Width: one parsed CSV field (longest 13).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13", + "bytes_at_full_source": "13, unchanged", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._archive -> _csv_record (:327) -> :256-258", + "classification": "must-not-move", + "notes": "Field width; part of the receipt's parser_profile." + }, + { + "constant": "acs_2024_1yr_sources.json size_bytes (pinned archive sizes; loaded by acs_sources.py:88 load_acs_source_manifest -> acs_housing_universe_source.py:101-107 _pins)", + "value": "csv_hus.zip 251,500,587; csv_pus.zip 602,847,146 (sum 854,347,733)", + "enforced_at": [ + "acs_housing_universe_source.py:213 `_copy(source / name, target, size, exact_size=size)` -> :137 \"FILE_TOO_LARGE\", :139 \"SOURCE_SIZE\", :142-144 read(N+1) \"FILE_GREW\", :149-154 \"FILE_CHANGED\"; :214 \"SOURCE_SHA256\"", + "acs_housing_universe_source.py:215 `_persisted_sha(target, size)` -> :165-167 \"SNAPSHOT_SIZE\", :168-170 read(N+1) \"SNAPSHOT_GREW\", :172-175 \"SNAPSHOT_CHANGED\"; :218 \"SOURCE_AUTHORITY_CHANGED\"", + "acs_person_coverage_authentication.py:817-821 `custody._persisted_sha(paths[role], size) == digest` -> \"SNAPSHOT_CHANGED\"", + "acs_native_coverage_binding.py:447 `housing._persisted_sha(path, size) == digest` -> \"SOURCE_CHANGED\" (every verify, every snapshot path)", + "acs_population_catalogue.py:201 -> \"SOURCE_CHANGED\"", + "acs_sources.py:198-202 (download exceeds pinned size), :273-276 (_file_matches), :300-304 (_verify_download), :266-269 (manifest validation) -- fetch path, not on a runner path", + "survey_population_preparation.py:551 compares the on-disk sizes to acs_catalogue._ARCHIVE_BYTES" + ], + "refusal_codes": [ + "FILE_TOO_LARGE, SOURCE_SIZE, FILE_GREW, FILE_CHANGED, SOURCE_SHA256, SNAPSHOT_SIZE, SNAPSHOT_GREW, SNAPSHOT_CHANGED, SOURCE_AUTHORITY_CHANGED (ACSHousingSourceError)", + "SNAPSHOT_CHANGED (ACSCoverageAuthenticationError auth.py:817-821)", + "SOURCE_CHANGED (ACSNativeCoverageBindingError binding.py:447; ACSSourceCatalogueError catalogue.py:201)", + "ValueError (acs_sources.py fetch path)" + ], + "encodes": "Upstream: the two Census archives' exact byte sizes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "854,347,733, unchanged", + "bytes_at_full_source": "854,347,733, unchanged", + "measurement_basis": "manifest read this session; equal to the sum the row-count lane confirmed on disk", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._capture (catalogue.py:487, binding.py:583, housing.py:832 via prepare, auth.py:726 via load: four captures per preparation) -> :211-215; re-hashed on every verify (binding._verify_sources :442-448 over every snapshot)", + "classification": "must-not-move", + "notes": "Exact-size read(N+1) caps on real files; every capture copies both archives (854 MB) into a fresh private directory." + }, + { + "constant": "acs_housing_universe_source.py:201-207 disk demand", + "value": "sum(pins) + 2 * ACS_HU_SOURCE_MAX_BYTES + 2 * ACS_HU_RECEIPT_MAX_BYTES + 1024**3 = 854,347,733 + 17,179,869,184 + 2,097,152 + 1,073,741,824 = 19,110,055,893 (17.8 GiB free)", + "enforced_at": [ + "acs_housing_universe_source.py:207 `_require(shutil.disk_usage(root).free >= required, \"INSUFFICIENT_DISK\")`" + ], + "refusal_codes": [ + "INSUFFICIENT_DISK (ACSHousingSourceError, :207)" + ], + "encodes": "An operational free-disk ceiling for one private capture; no document (comment :198-200: 'This bound is operational, not a source-size prediction').", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "19,110,055,893 required, unchanged", + "bytes_at_full_source": "19,110,055,893 required, unchanged", + "measurement_basis": "computed from the constants at this head (measure_acs_family.py part 1 disk_demand)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._capture (:190) at each of the four captures per preparation -> :201-207", + "classification": "not-a-transport", + "notes": "Tests patch `shutil.disk_usage` (test_us_acs_housing_source.py:514-520); the measurement script did the same. Not a byte transport; listed because it is an inline byte literal." + }, + { + "constant": "acs_housing_universe_source.py:386-395 `len(token) <= 5` (_integer)", + "value": "5 ASCII digits for TYPEHUGQ, NP, WGTP, TEN, SPORDER, PWGTP, PUMA tokens; also :456 `len(v) <= 2` (SPORDER) and :473-474 `len(v) == 5` (PUMA)", + "enforced_at": [ + "acs_housing_universe_source.py:387-392 `_require(type(token) is str and re.fullmatch(r\"[0-9]+\", token, flags=re.ASCII) is not None and len(token) <= 5, label)`", + "acs_housing_universe_source.py:441-464 (per field, inside _tables)", + "acs_housing_universe_source.py:473-475 PUMA" + ], + "refusal_codes": [ + "the field label as the code: TYPEHUGQ, NP, WGTP, TEN, SPORDER, PWGTP, PUMA (ACSHousingSourceError)" + ], + "encodes": "Width: one numeric source token.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "<= 5 per token, unchanged", + "bytes_at_full_source": "<= 5, unchanged", + "measurement_basis": "code read", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._tables (via _select :498, verify_acs_frame_projection :709 on every verify, properties :551-560)", + "classification": "must-not-move", + "notes": "Fixed-width field checks. SERIALNO's 13-character width is checked separately (acs_pums row)." + }, + { + "constant": "acs_housing_universe_source.py:605 `_sha(_json(chosen))` (cap: none)", + "value": "none -- canonical_json of the whole selected-serialno tuple, then sha256", + "enforced_at": [ + "acs_housing_universe_source.py:605 `\"selected_serialnos_sha256\": _sha(_json(chosen))`", + "consumer re-encodings: graph_acs_housing_universe.py:174 `receipt[\"selected_serialnos_sha256\"] == source._sha(canonical_json(chosen))` and :609 (unreachable)" + ], + "refusal_codes": [ + "none" + ], + "encodes": "Per-row roster: `chosen` = sorted selected ACS household SERIALNOs (binding path: 1,531,614 at full source) or every household record including vacancies (catalogue path, serialnos=None: 1,631,969 at every fraction); 16 B per key + 1.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "2,450,577 (binding path); 26,111,505 (catalogue path, unchanged)", + "bytes_at_full_source": "24,505,825 (binding path); 26,111,505 (catalogue path)", + "measurement_basis": "housing._json (canonical_json) gives the same 16 B per 13-char key as auth._json_size (measured at 1,000/2,000 rows); counts from brief section 3 and preparation.json", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._reconstruct (:563) via catalogue.py:490 (serialnos None) and via binding.py:591 -> prepare_acs_housing_population -> housing.py:833 (serialnos=acs_keys) -> :605", + "classification": "segment", + "notes": "Uncapped whole copy of the same bytes the binding sizes without encoding at :578; a streamed digest would remove it. Same caveat as the other uncapped rows." + }, + { + "constant": "acs_housing_universe_source.py:913 `_json(receipt)` prepared receipt (cap: none)", + "value": "none -- `PreparedACSHousingPopulation(frame, source, _json(receipt))`, canonical_json with no limit", + "enforced_at": [ + "acs_housing_universe_source.py:891-913 (producer; no cap)", + "consumers: acs_native_coverage_binding.py:481 `json.loads(prepared.receipt_json)[\"frame_sha256\"]` (_verify_prepared_frame, at :594 and inside every _verify_frame :497), :509 `coverage._sha(prepared.receipt_json)` (EVIDENCE_CHANGED), :600 `prepared.receipt[\"requested_serialnos\"] == (None if serialnos is None else list(serialnos))` (parses whole and compares the list), :717; acs_housing_universe_source.py:823-824 `.receipt` property; survey_population_preparation.py:1710 retains `acs.prepared.receipt_json` in the nested seals", + "graph_acs_housing_universe.py:202 `source._parse_json(prepared.receipt_json, source.ACS_HU_RECEIPT_MAX_BYTES)` (1 MiB; JSON_SIZE / CANONICAL_JSON) -- the only capped consumer, unreachable" + ], + "refusal_codes": [ + "none on the reachable path; JSON_SIZE / CANONICAL_JSON (ACSHousingSourceError :82/:85) at the unreachable graph site" + ], + "encodes": "Per-row roster: selected ACS households through `\"requested_serialnos\": serialnos` (:904), inside a fixed preamble of 2,518 bytes at this head (dtype_transitions list of 10 columns, entity_rows, digests).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "2,453,094 (= 2,518 + 16 x 153,161)", + "bytes_at_full_source": "24,508,342 (= 2,518 + 16 x 1,531,614)", + "measurement_basis": "the real prepared receipt issued on invented archives at this head, re-encoded through housing._json with requested_serialnos swapped for 1,000 and 2,000 keys (per key 16.0; preamble 2,518) -- measure_acs_family.py part 2 prepared_receipt", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (survey_population_preparation.py:1962) -> housing.prepare_acs_housing_population (binding.py:591) -> housing.py:913; parsed whole at binding.py:481 on every verify and at :600 at issuance", + "classification": "segment", + "notes": "The document the brief says the row-count lane missed. No bound at head; a whole 24.5 MB string parsed on every verify. If the ACS housing graph were run on a selection, graph_acs_housing_universe.py:202 would refuse it above 65,535 keys (2.34x over 1 MiB at 1/10)." + }, + { + "constant": "acs_pums.py:55 MAX_EXACT_HOUSEHOLDS", + "value": "7_000_000 (a row count)", + "enforced_at": [ + "acs_pums.py:192-208 `snapshot_serialnos`: `0 < len(serialnos) <= MAX_EXACT_HOUSEHOLDS` else ValueError(\"ACS exact selection requires bounded unique raw native keys.\")", + "callers: acs_native_coverage_binding.py:568, acs_housing_universe_source.py:830, acs_pums.py:224", + "recorded: acs_native_coverage_binding.py:278" + ], + "refusal_codes": [ + "ValueError (acs_pums.py:206-208); surfaces as NATIVE_ISSUANCE_REFUSED (binding.py:749) / FRAME_PREPARATION_REFUSED (housing.py:925)" + ], + "encodes": "Count of requested ACS household keys (1,531,614 at full source), not bytes.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "n/a (153,161 keys)", + "bytes_at_full_source": "n/a (1,531,614 keys; 4.57x headroom)", + "measurement_basis": "brief section 3", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (:1962) -> binding.py:568 -> acs_pums.py:192", + "classification": "not-a-transport", + "notes": "Lifted on native-row-ceilings; unchanged here." + }, + { + "constant": "acs_pums.py:56 MAX_EXACT_PERSON_ROWS", + "value": "14_000_000 (a row count)", + "enforced_at": [ + "acs_pums.py:270-273 `if household.NP.sum() > MAX_EXACT_PERSON_ROWS: raise ValueError(\"ACS selected complete roster exceeds native person budget.\")` (serialnos route only)", + "recorded: acs_native_coverage_binding.py:279" + ], + "refusal_codes": [ + "ValueError (acs_pums.py:271-273); surfaces as FRAME_PREPARATION_REFUSED / NATIVE_ISSUANCE_REFUSED" + ], + "encodes": "NP summed over the selected households (3,422,888 at full source), not bytes.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "n/a (342,288)", + "bytes_at_full_source": "n/a (3,422,888; 4.09x headroom)", + "measurement_basis": "brief section 3", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing.prepare_acs_housing_population (binding.py:591) -> build_acs_pums_unit_frame (housing.py:836) -> load_acs_pums_tables (acs_pums.py:333) -> :270", + "classification": "not-a-transport", + "notes": "Row count." + }, + { + "constant": "acs_pums.py:197 SERIALNO width 13", + "value": "`len(key) != 13` refuses; equivalently the regex `2024(?:HU|GQ)[0-9]{7}` at acs_person_coverage_columns.py:104, acs_housing_universe_source.py:432 and acs_population_catalogue.py:264", + "enforced_at": [ + "acs_pums.py:192-208 snapshot_serialnos (ValueError)", + "acs_person_coverage_columns.py:101-107 _keys (ValueError \"ACS coverage native serial is not a 2024 HU/GQ identity\")", + "acs_housing_universe_source.py:429-437 (\"SERIALNO\")", + "acs_population_catalogue.py:262-268 (\"LITERAL_PERSON_KEY\")" + ], + "refusal_codes": [ + "ValueError (acs_pums.py, columns.py)", + "SERIALNO (ACSHousingSourceError)", + "LITERAL_PERSON_KEY (ACSSourceCatalogueError)" + ], + "encodes": "Width: every household key is exactly 13 characters (measured serialno_widths {13: 1,631,969}); this fixes the 16 canonical bytes per key that every serialno roster in this family carries.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13 per key, unchanged", + "bytes_at_full_source": "13 per key, unchanged", + "measurement_basis": "byte-census-measurements.json acs_household_archive.serialno_widths", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding.py:568 snapshot_serialnos; literal._keys via auth.py:579 and columns.py:238/:262; housing._tables; catalogue._collect", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "acs_pums.py:48 DEFAULT_CHUNKSIZE", + "value": "100_000 rows (also acs_person_coverage_columns.py:236 `0 < chunksize <= 100_000` and acs_person_coverage_authentication.py:750 `chunksize=min(1000, len(keys))`)", + "enforced_at": [ + "acs_pums.py:229-230 `if chunksize <= 0: raise ValueError`; :446-452 pd.read_csv(chunksize=...)", + "acs_person_coverage_columns.py:236-237 ValueError(\"ACS coverage chunksize must be in 1..100000\"); :252-253 batch flush", + "acs_person_coverage_authentication.py:750" + ], + "refusal_codes": [ + "ValueError (acs_pums.py:230; columns.py:237)" + ], + "encodes": "Row batch sizes for the pandas reader and the literal reader; not byte bounds.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a", + "bytes_at_full_source": "n/a", + "measurement_basis": "code read", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing.prepare_acs_housing_population -> build_acs_pums_unit_frame (default chunksize); auth.py:747-751 -> read_acs_person_coverage_columns(chunksize=min(1000, len(keys)))", + "classification": "not-a-transport", + "notes": "Batch sizes; the selected tables still materialise whole (acs_pums.py:482 `pd.concat(pieces)`, columns.py:261)." + }, + { + "constant": "acs_sources.py:49 _DEFAULT_CHUNK_BYTES", + "value": "1024 * 1024 = 1,048,576", + "enforced_at": [ + "acs_sources.py:146 default chunk_bytes of fetch_acs_pums_sources; :151 `if chunk_bytes <= 0: raise ValueError`; :196 `_iter_chunks(response, chunk_bytes)`", + "acs_sources.py:283 `_iter_chunks(source, _DEFAULT_CHUNK_BYTES)` (_file_identity)" + ], + "refusal_codes": [ + "ValueError(\"chunk_bytes must be positive.\") (:152)" + ], + "encodes": "Download / hashing chunk; no document.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a", + "bytes_at_full_source": "n/a", + "measurement_basis": "code read", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "fetch_acs_pums_sources is called from no src module (grep); the runners reach acs_sources only through load_acs_source_manifest (housing.py:106 _pins), which reads the packaged manifest", + "classification": "not-a-transport", + "notes": "The fetch path's size checks (:198-202, :273-276, :300-304) are enforcement sites of the manifest size_bytes row, also unreachable from the runners." + }, + { + "constant": "acs_population_catalogue.py:25 _ARCHIVE_BYTES", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "acs_population_catalogue.py:484 `_require(sum(p[3] for p in pins) <= _ARCHIVE_BYTES, \"ARCHIVE_LIMIT\")`", + "survey_population_preparation.py:513-514 (_file_limits for acs/csv_hus.zip and acs/csv_pus.zip) and :546-553 `sum(lstat().st_size ...) <= acs_catalogue._ARCHIVE_BYTES` -> \"ARCHIVE_LIMIT\"", + "acs_population_catalogue.py:121-128 identity-checked against the literal 8*1024**3 (\"LIMITS\")" + ], + "refusal_codes": [ + "ARCHIVE_LIMIT (ACSSourceCatalogueError :484; SurveyPopulationPreparationError preparation.py:552)", + "LIMITS (:127)" + ], + "encodes": "Upstream: the two pinned archives' sizes (854,347,733).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "854,347,733, unchanged", + "bytes_at_full_source": "854,347,733, unchanged", + "measurement_basis": "manifest", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> prepare_authenticated_survey_population -> _source_files (:1929) -> :546-553; -> acs_catalogue.issue_acs_source_catalogue (:1939) -> :484", + "classification": "must-not-move", + "notes": "Twin of binding.MAX_ARCHIVE_BYTES; pinned by the LIMITS check (:121-128) so it cannot be raised without editing the literal tuple." + }, + { + "constant": "acs_population_catalogue.py:26 _EXPANDED_BYTES", + "value": "16 * 1024**3 = 17,179,869,184", + "enforced_at": [ + "acs_population_catalogue.py:151-152 `expanded += sum(member.file_size for member in members)`; `_require(expanded <= _EXPANDED_BYTES, \"EXPANDED_LIMIT\")` (combined)", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "EXPANDED_LIMIT (ACSSourceCatalogueError, :152)" + ], + "encodes": "Upstream: combined declared member bytes (3,383,159,949).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "3,383,159,949, unchanged", + "bytes_at_full_source": "3,383,159,949, unchanged", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_catalogue.issue_acs_source_catalogue (:1939) -> _preflight (:489) -> :151-152", + "classification": "must-not-move", + "notes": "Real upstream size." + }, + { + "constant": "acs_population_catalogue.py:27 _MEMBERS", + "value": "64 (a count)", + "enforced_at": [ + "acs_population_catalogue.py:150 `_require(len(members) <= _MEMBERS, \"MEMBER_LIMIT\")`", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "MEMBER_LIMIT (ACSSourceCatalogueError, :150)" + ], + "encodes": "Count of ZIP members per archive (3), not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a (3)", + "bytes_at_full_source": "n/a (3)", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> catalogue._preflight (:489) -> :150", + "classification": "not-a-transport", + "notes": "Count." + }, + { + "constant": "acs_population_catalogue.py:28 _SOURCE_ROWS", + "value": "6_000_000 (a row count)", + "enforced_at": [ + "acs_population_catalogue.py:157-160 `sum(member[\"rows\"] for member in inventory) <= _SOURCE_ROWS` -> \"SOURCE_ROW_LIMIT\"", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "SOURCE_ROW_LIMIT (ACSSourceCatalogueError, :159)" + ], + "encodes": "Upstream rows per role (1,631,969 / 3,422,888), not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a", + "bytes_at_full_source": "n/a", + "measurement_basis": "preparation.json /catalogues/acs/counts", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> catalogue._preflight (:489) -> :155-160 (coverage._inventory with frozenset())", + "classification": "not-a-transport", + "notes": "Row count; upstream-file-size rule." + }, + { + "constant": "acs_population_catalogue.py:29 _BATCH_PEOPLE", + "value": "100_000 (a count)", + "enforced_at": [ + "acs_population_catalogue.py:255 `_require(count <= _BATCH_PEOPLE, \"HOUSEHOLD_EXCEEDS_BATCH\")` (one household's NP)", + "acs_population_catalogue.py:315-318 `count <= min(_BATCH_PEOPLE, literal.MAX_SELECTED_ROWS)` -> \"BATCH_PERSON_LIMIT\"", + "acs_population_catalogue.py:332 batching threshold (no refusal)", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "HOUSEHOLD_EXCEEDS_BATCH (:255)", + "BATCH_PERSON_LIMIT (:318)" + ], + "encodes": "Persons per assembly batch (the pilot recorded literal_batches 35 for 3,422,888 people), not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a", + "bytes_at_full_source": "n/a", + "measurement_basis": "preparation.json /catalogues/acs/counts/literal_batches", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> issue_acs_source_catalogue -> _collect (:510) -> :255/:316/:332", + "classification": "not-a-transport", + "notes": "Count." + }, + { + "constant": "acs_population_catalogue.py:30 _CATALOGUE_BYTES", + "value": "2 * 1024**3 = 2,147,483,648", + "enforced_at": [ + "acs_population_catalogue.py:204-209 `_charge`: `raw = native.coverage._json(record, max(0, _CATALOGUE_BYTES - size - 1)) + b\"\\n\"` (per-record cap shrinking with the running total -> auth.py:93/:147 \"CANONICAL_SIZE\") and `_require(size <= _CATALOGUE_BYTES, \"CATALOGUE_BYTE_LIMIT\")`", + "acs_population_catalogue.py:230-238 `reserve` (prospective charge of every household and person before retention; same two codes)", + "acs_population_catalogue.py:339 `_require(size == prospective_size, \"CATALOGUE_BYTE_ACCOUNTING\")`", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "CATALOGUE_BYTE_LIMIT (ACSSourceCatalogueError, :207/:238)", + "CANONICAL_SIZE (ACSCoverageAuthenticationError at the _json caps, surfaced as CATALOGUE_ISSUANCE_REFUSED :607)", + "CATALOGUE_BYTE_ACCOUNTING (:339)" + ], + "encodes": "The canonical record stream of the whole source: one household tuple per record (1,631,969, incl. vacancies) with its nested person tuples (3,422,888); identical at every fraction (catalogue.py:490 passes serialnos=None). Digested, never joined.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "374,174,650, unchanged", + "bytes_at_full_source": "374,174,650 MEASURED (preparation.json /catalogues/acs/counts/canonical_record_bytes) = 0.174 x 2 GiB", + "measurement_basis": "cited from the pilot receipt; reproduced through auth._json on invented records at full-source widths (58 B household record without members, 81 B per nested person at 3-digit PWGTP / 7-digit ordinal -> 371,908,130, 0.6% under the measured total)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_catalogue.issue_acs_source_catalogue (survey_population_preparation.py:1939) -> _collect (:510) -> reserve (:256/:295) and consume -> _charge (:325)", + "classification": "stays", + "notes": "5.7x headroom; does not move with the fraction. The bytes are streamed, but the records themselves are retained whole as Python tuples in `_Owned.records`/`vacancies` (:574-577) with no byte bound, and `.households` (:408-409) rebuilds 1.63M AcsHousehold dataclasses on each access (preparation.py:1944 `acs.households`)." + }, + { + "constant": "acs_population_catalogue.py:31 _RECEIPT_BYTES", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_population_catalogue.py:516-565 `receipt = native.coverage._json({...}, _RECEIPT_BYTES)` (catalogue receipt) -> auth.py:93/:147 \"CANONICAL_SIZE\"", + "acs_population_catalogue.py:443, :452, :486, :590 `native.coverage._json(_producer(), _RECEIPT_BYTES)` (producer document; compared for PRODUCER_CHANGED)", + "acs_population_catalogue.py:474-478 `len(candidate) <= _RECEIPT_BYTES` -> \"CANDIDATE_TYPE_SIZE\" (no candidate on the runner path)", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "CANONICAL_SIZE (surfaced as CATALOGUE_ISSUANCE_REFUSED :607 / CATALOGUE_VERIFICATION_REFUSED :461)", + "CANDIDATE_TYPE_SIZE (:478)", + "PRODUCER_CHANGED (:444/:453/:591)" + ], + "encodes": "Fixed-shape: the catalogue receipt (producer closure 17,808 B, archives, six-member inventories without header_hex, counts as integers, field contract, record schema) and the producer document alone.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "receipt 21,769 (with the real six-member inventory; 20,992 invented); producer document 17,808", + "bytes_at_full_source": "same, unchanged (0.021x)", + "measurement_basis": "the real catalogue issuance on invented archives at this head (catalogue.payload) re-encoded with the pilot-shaped six-member inventories; `len(auth._json(catalogue._producer(), _RECEIPT_BYTES))` live (measure_acs_family.py part 2)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_catalogue.issue_acs_source_catalogue (:1939) -> :485-486, :516-565; verify_acs_source_catalogue (preparation.py:1787) -> _checked (:440) -> :443/:452", + "classification": "stays", + "notes": "48x headroom. The 1/1000 artifact's /producer/acs is this producer document (17,886 B then; 17,808 B at this head)." + }, + { + "constant": "acs_population_catalogue.py:121 ceilings tuple (LIMITS)", + "value": "(8 * 1024**3, 16 * 1024**3, 64, 6_000_000, 100_000, 2 * 1024**3, 1024**2): each catalogue constant must satisfy `0 < v <= cap`", + "enforced_at": [ + "acs_population_catalogue.py:112-128 `_require(all(type(v) is int and 0 < v <= cap for v, cap in zip(limits, ceilings, strict=True)), \"LIMITS\")` inside _producer (runs at issuance :485 and every verify :443/:452/:590)" + ], + "refusal_codes": [ + "LIMITS (ACSSourceCatalogueError, :127)" + ], + "encodes": "A pin on the seven catalogue constants (their literal maxima); no document.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a", + "bytes_at_full_source": "n/a", + "measurement_basis": "code read", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> catalogue._producer (:105) -> :112-128", + "classification": "not-a-transport", + "notes": "Raising _CATALOGUE_BYTES or _RECEIPT_BYTES above these literals refuses LIMITS; a transport change to either must edit the tuple in the same commit. The producer document also carries `limits` (:140), so any move changes issued receipts." + }, + { + "constant": "acs_housing_universe_source.py:81-88 _parse_json (consumer-side re-encoding)", + "value": "`_require(type(payload) is bytes and len(payload) <= cap, \"JSON_SIZE\")`; `_require(type(value) is dict and _json(value) == payload, \"CANONICAL_JSON\")`; cap supplied per call: graph_acs_housing_universe.py:177 frame_context 64 MiB, :202/:218/:308/:310 receipts ACS_HU_RECEIPT_MAX_BYTES 1 MiB, :205/:309 projection ACS_HU_PAYLOAD_MAX_BYTES 64 MiB, :269 header 65,536", + "enforced_at": [ + "acs_housing_universe_source.py:82 JSON_SIZE, :85 CANONICAL_JSON, :86-87 JSON_DEPTH", + "graph_acs_housing_universe.py:177, :202, :204-206, :218, :269, :308, :309, :310 (the only callers; grep of packages/*/src)" + ], + "refusal_codes": [ + "JSON_SIZE, CANONICAL_JSON, JSON_DEPTH (ACSHousingSourceError; wrapped as GRAPH_ACS_TRANSPORT_CONTRACT at graph_acs_housing_universe.py:380)" + ], + "encodes": "Re-encodes every parsed document whole (canonical_json of the parsed value == payload). Roster-shaped inputs: the selected projection (64 MiB cap) and the prepared receipt (1 MiB cap); fixed inputs: source receipt, header, frame_context.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "selected projection ~28.5 MB (0.43 x 64 MiB); prepared receipt 2,453,094 (2.34 x 1 MiB)", + "bytes_at_full_source": "selected projection ~285.3 MB (4.3 x 64 MiB); prepared receipt 24,508,342 (23.4 x 1 MiB)", + "measurement_basis": "the projection and prepared-receipt rows above", + "binds_at_tenth": true, + "binds_at_full_source": true, + "reachable_from": "none", + "reach_chain": "graph_acs_housing_universe.ACSHousingCreateKernel.run (:457) -> encode_acs_housing_evidence (:482 -> :196) and ACSHousingSelectKernel.run -> bind_acs_housing_evidence (:548 -> :249); the codec `us-acs-housing-universe-2024-v1` is registered at graph_sources.py:308 but the recovered 19-node graph declares only `us-survey-population-source-v1` and `raw-bytes-v1` sources, and no node uses `us.acs_housing.*` kernels", + "classification": "consumer", + "notes": "The prepared receipt would bind at 1/10 here (65,535 keys admitted) if the ACS housing graph were run; it is not on any runner path. Binds flags refer to those two documents at these caps." + }, + { + "constant": "graph_acs_housing_universe.py:52 ACS_HU_PAYLOAD_MAX_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "graph_acs_housing_universe.py:245 `_require(len(payload) <= ACS_HU_PAYLOAD_MAX_BYTES, \"PAYLOAD_SIZE\")` (encode: MAGIC + encode_acs_housing_evidence (:482); ACSHousingSelectKernel.run (:509) -> bind_acs_housing_evidence (:548). Not a node of the 19/45 graphs (graph.json sources: us-survey-population-source-v1, raw-bytes-v1)", + "classification": "segment", + "notes": "Would bind at full source if this graph were run; unreachable from the census runners. `_verify_preparation` (:141) also re-encodes `canonical_json(chosen)` at :174 (uncapped, 24.5 MB) and the select kernel encodes `canonical_json(selected_projection)` at :607." + }, + { + "constant": "graph_acs_housing_universe.py:53 ACS_HU_HEADER_MAX_BYTES", + "value": "65536", + "enforced_at": [ + "graph_acs_housing_universe.py:242 `_require(len(encoded) <= ACS_HU_HEADER_MAX_BYTES, \"HEADER_SIZE\")`", + "graph_acs_housing_universe.py:264-267 (bind) HEADER_SIZE; :269 _parse_json cap" + ], + "refusal_codes": [ + "GRAPH_ACS_HEADER_SIZE (ACSHousingSourceError)" + ], + "encodes": "Fixed-shape header (:224-240): 15 keys of digests and integers.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "under 1 KB (15 short keys, seven 64-hex digests), unchanged", + "bytes_at_full_source": "same", + "measurement_basis": "code read of :224-240 (not encoded; unreachable module)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "encode_acs_housing_evidence (:196) / bind_acs_housing_evidence (:249); not reached by the 19/45 runners", + "classification": "stays", + "notes": "Length prefix `struct.pack(\" `canonical_json(selection)` (cap: none)", + "value": "none", + "enforced_at": [ + "graph_acs_housing_universe.py:634-637 artifact `\"selection\": canonical_json(selection)`; :641 `\"selection_sha256\": source._sha(canonical_json(selection))` (encoded twice)" + ], + "refusal_codes": [ + "none" + ], + "encodes": "Per-row roster: positions (person and household int lists), native_keys (household SERIALNOs, person [SERIALNO, SPORDER] pairs), ordered_ids for all six entities (tax units, families, marital units, SPM units are SCALED ratios of the 1/1000 artifact).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "~13.2 MB ESTIMATE (86 B per household with two persons, excluding the four group-entity id lists)", + "bytes_at_full_source": "~131.7 MB ESTIMATE, plus the group-entity ids (SCALED)", + "measurement_basis": "canonical_json on invented positions/native_keys/ordered_ids at 1,000 and 2,000 households (measure_acs_family.py part 1 graph_selection_artifact); small ints used, 7-digit ints at full source add bytes", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "ACSHousingSelectKernel.run (:509) -> :593-641; not a node of the census graphs", + "classification": "segment", + "notes": "Uncapped whole copy; recorded for completeness only." + }, + { + "constant": "asec_housing_universe.py:42 PAYLOAD_MAX_BYTES (enforced by graph_housing_universe.py; definition belongs to the ASEC family)", + "value": "len(MAGIC) 9 + 4 + HEADER_MAX_BYTES 65,536 + MAX_HOUSEHOLDS 400,000 x (8 x 7 RAW_COLUMNS + 7 DERIVED_COLUMNS = 63) + 32 = 25,265,581", + "enforced_at": [ + "graph_housing_universe.py:78-82 `hu._require(type(payload) is bytes and len(hu.MAGIC) + 4 + 32 < len(payload) <= hu.PAYLOAD_MAX_BYTES, \"BOUND_HU_SIZE\")`", + "graph_housing_universe.py:151-155 `expected_size = cursor + rows * (8 * len(hu.RAW_COLUMNS) + len(hu.DERIVED_COLUMNS)) + 32`; `_require(len(payload) == expected_size, \"BOUND_HU_LENGTH\")` (the 63-byte-per-household width)", + "graph_housing_universe.py:144-150 `0 < rows <= hu.MAX_HOUSEHOLDS` -> \"BOUND_HU_ROWS\"", + "graph_housing_universe.py:98-103 `struct.unpack_from(\" \"BOUND_HU_HEADER_SIZE\"", + "asec_housing_universe.py:382 (the ASEC owner's own decode; ASEC family)" + ], + "refusal_codes": [ + "BOUND_HU_SIZE, BOUND_HU_LENGTH, BOUND_HU_ROWS, BOUND_HU_HEADER_SIZE (HousingUniverseRefusalError via hu._require)" + ], + "encodes": "A numpy body: 7 int64 + 7 uint8 columns per ASEC household (63 B), over the ASEC source's own households (55,762), identical at every fraction; plus a fixed header.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "3,513,006 body (63 x 55,762) + header, unchanged", + "bytes_at_full_source": "3,513,006 + header, unchanged (0.139x)", + "measurement_basis": "constants read from asec_housing_universe.py:19-47 and multiplied (measure_acs_family.py part 1 asec_housing_universe); ASEC household count from brief section 3", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "bind_housing_universe is called from graph_composed_asec_binding.py:592 and graph_asec_prepared.py:440/:666 (and verify_graph_housing_rows from :686 / :443 / :680); none of those modules is in the brief's 19/45 node table", + "classification": "must-not-move", + "notes": "Fixed width x an upstream roster (ASEC file rows, not the stacked survey). Listed here only for its enforcement sites in graph_housing_universe.py; the row belongs to the ASEC family's census keyed at asec_housing_universe.py:42. MAX_HOUSEHOLDS 400,000 (:41) is a count wearing no byte name: 7.2x headroom over 55,762." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/acs/measure_acs_family.py (run with .venv/bin/python, sys.path = packages/*/src; part 1 encodes invented rows through auth._json/_json_size/_json_roster/_body_row_bound, housing._json, literal._json, canonical_json at 1,000 and 2,000 rows; part 2 drives the real issue_acs_source_catalogue and issue_acs_native_coverage on a two-household invented archive pair and measures every fixed-shape document at this head)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/acs/measurements.json (receipt; head 7e04682421041c986c4599d609d591672c22743c)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/acs/measure_acs_family.log (stdout)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/acs/invented/ (the invented archives and the private captures the issuance wrote)", + "experiments/native-byte-transports/byte-census-measurements.json (cited for the coverage body, cells JSON, key list, serialno list, old charges, raw sizes and state counts)" + ], + "open_questions": [ + "HEAD moved during the census from 77bafbb3acd8db70e13ddee58d44d2592a56742a to 7e04682421041c986c4599d609d591672c22743c (commit 'Stream the budget view's comparison; drop the segment-at-most-maximum guards'). `git diff --stat 77bafbb3a HEAD` over all twelve family files, asec_housing_universe.py and the sources manifest is empty, and sha256 of acs_person_coverage_authentication.py still equals binding._ACCEPTED (b8400706...), so every line cited holds at both heads.", + "docs/us-native-byte-transports.md is referenced by the code comments at acs_person_coverage_authentication.py:46 and acs_native_coverage_binding.py:38 but does not exist in the tree at this head (docs/ has us-native-row-ceilings.md, us-native-scale-transport.md, us-native-verification-once.md, us-native-spm-role-source-enrichment.md).", + "Row keying: where one constant fronts several roster documents (MAX_ROSTER_BYTES: charge, serialno pre-check, key-pair digest, issuance receipt; MAX_BODY_BYTES: body), I emitted one row per document with the constant's definition as the key prefix and a bracketed document tag, listing every enforcement site in each, so each verifier receives one document. Merge them if the orchestrator wants strictly one row per constant.", + "Four whole-roster encodings in this family have no bound at all at head (housing.py:913 prepared receipt; housing.py:605 selected-serialno digest; columns.py:292-294 native_roster_sha256; columns.py:295-297 projection_sha256, the last ~440 MB at full source). The brief's classification list has no 'uncapped' class; I recorded them as `segment` with value 'none'. The orchestrator should decide whether they are rows or notes.", + "Upstream-shaped derived documents that never move with the fraction (the 292,484,053-byte full lexical projection under 8 GiB; the 374,174,650-byte catalogue record stream under 2 GiB) are classified `stays` with shape `upstream`, reserving `must-not-move` for real file sizes, widths and scalars as the brief defines it.", + "WGTP/PWGTP token widths are not in the section-4 inputs. The 3-digit variant reproduces the row-count lane's measured full-projection.json to 0.03% (292,401,960 vs 292,484,053) and the catalogue-record estimate lands 0.6% under the pilot receipt, so the selected-projection numbers (~28.5 MB / ~285.3 MB) are calibrated ESTIMATES, flagged as such.", + "Full-source selection semantics: the runner always passes an explicit key tuple (`serialnos=acs_keys`, survey_population_preparation.py:1946-1963, ACS keys only), so the issuance receipt, prepared receipt and serialno pre-check carry 1,531,614 entries at full source; the `serialnos is None` route (fixed-shape receipts) is never taken. The row-count lane's verifier note that used 1,587,376 (stacked households) for ACS_HU_RECEIPT_MAX_BYTES conflicts with the brief's section-3 count; I used 1,531,614.", + "acs_inputs.py, acs_housing_universe.py and source_csv_builtin.py were read in full and carry no byte bound (zero rows). acs_sources.py's only byte constant is a chunk size; its fetch path is reachable from no src module.", + "The `age` runner (survey_age_calibration) was not traced into this family: it consumes an issued preparation, and any re-verification of the ACS native coverage would go through survey_population_preparation._validate (:1785-1795), which the 19-node chain already covers.", + "graph_housing_universe.py enforces only constants defined in asec_housing_universe.py (ASEC family); that row is included here for its enforcement sites and should be merged with the ASEC family's row keyed at asec_housing_universe.py:42.", + "Per-preparation cost not captured by any byte bound: four private captures each copy both archives (854 MB) and require 17.8 GiB free; the 292 MB full projection is built twice and read back whole once; the 24.5 MB issuance receipt and the ~285 MB selected projection are json.loads-ed whole on every verify_acs_native_coverage (preparation.py:1789, current_survey_predictors.py:99); the catalogue retains 1.63M household tuples with nested persons in memory." + ] + } + ] +} \ No newline at end of file diff --git a/experiments/native-byte-transports/dependent_test_battery.py b/experiments/native-byte-transports/dependent_test_battery.py new file mode 100644 index 000000000..eda3087d5 --- /dev/null +++ b/experiments/native-byte-transports/dependent_test_battery.py @@ -0,0 +1,125 @@ +"""Run every test file that names a module this lane touched, one per process. + +CI runs the build shard's groups in separate pytest processes so that one +file's ``monkeypatch.setattr`` on a module-level function cannot trip +``survey_population_preparation._producer``'s live-code seal for every later +file; the native-scale lane's report §9a is the write-up of what happens when +they share one. This runs them the way CI does, with a bounded number of +processes at a time, and records each file's own summary line. The pattern is +the retention lane's ``dependent_test_battery.py``; the markers are this lane's. + + python experiments/native-byte-transports/dependent_test_battery.py OUT.json [JOBS] +""" + +import concurrent.futures +import json +import pathlib +import subprocess +import sys +import time + +ROOT = pathlib.Path(__file__).resolve().parents[2] # the worktree root +PYTHON = ROOT / ".venv/bin/python" +MARKERS = ( + "acs_person_coverage_authentication", + "acs_native_coverage_binding", + "acs_population_catalogue", + "survey_population_preparation", + "graph_survey_population", + "survey_origin_budget", + "graph_survey_budget", + "graph_survey_calibration", + "graph_survey_age_artifact", + "current_survey_predictors", + "graph_child_property_income", + "current_child_property_income_source", + "graph_joint_empirical", +) +FILES = sorted( + str(path.relative_to(ROOT)) + for package in ("microcosm-build", "microcosm-fit") + for path in (ROOT / "packages" / package / "tests").glob("test_*.py") + if any(marker in path.read_text(encoding="utf-8") for marker in MARKERS) +) + + +def run(relative): + started = time.monotonic() + completed = subprocess.run( + [ + str(PYTHON), + "-m", + "pytest", + relative, + "-p", + "no:randomly", + "-p", + "no:cacheprovider", + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + tail = [line for line in completed.stdout.splitlines() if line.strip()] + summary = next( + ( + line + for line in reversed(tail) + if " passed" in line or " failed" in line or " error" in line + ), + "", + ) + return { + "file": relative, + "returncode": completed.returncode, + "summary": summary.strip(), + "wall_seconds": round(time.monotonic() - started, 1), + } + + +def main(): + out = pathlib.Path(sys.argv[1]) + jobs = int(sys.argv[2]) if len(sys.argv) > 2 else 4 + head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True + ).stdout.strip() + dirty = subprocess.run( + ["git", "status", "--porcelain", "--", "packages"], + cwd=ROOT, + capture_output=True, + text=True, + ).stdout.strip() + results = [] + print(f"head={head} dirty_packages={'yes' if dirty else 'no'} files={len(FILES)}") + with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool: + for result in pool.map(run, FILES): + results.append(result) + print( + f"{'ok ' if result['returncode'] == 0 else 'FAIL'} " + f"{result['file']} {result['summary']} ({result['wall_seconds']}s)", + flush=True, + ) + out.write_text( + json.dumps( + { + "scope": "diagnostic; not a build, certification or release artifact", + "release_eligible": False, + "invocation": "one pytest process per file, as CI runs them", + "head": head, + "packages_dirty_at_start": bool(dirty), + "markers": list(MARKERS), + "files": len(results), + "failing_files": [r["file"] for r in results if r["returncode"] != 0], + "results": sorted(results, key=lambda r: r["file"]), + }, + indent=2, + ) + + "\n" + ) + print( + f"\nfiles={len(results)} failing={len([r for r in results if r['returncode']])}" + ) + + +if __name__ == "__main__": + main() diff --git a/experiments/native-byte-transports/fifteenth/harness19_diag_fifteenth.py b/experiments/native-byte-transports/fifteenth/harness19_diag_fifteenth.py new file mode 100644 index 000000000..dc375f264 --- /dev/null +++ b/experiments/native-byte-transports/fifteenth/harness19_diag_fifteenth.py @@ -0,0 +1,747 @@ +"""The retention-seal lane's 1/15 run: above the transport ceiling that was. + +A copy of the native-scale lane's committed ``harness19_tenth.py`` with exactly +three changes, each named here and nowhere else in the file: + +* the sample is 1/15 -- 105,825 of the 1,587,376 source households, where the + single bounded encode refused at 96,860 -- rather than 1/10; +* ``RSS_CEILING`` is the retention brief's 48 GiB rather than 64 GiB; +* the label and the docstring say which lane and which fraction. + +It carries two further corrections the 1/15 measurement harness did not: its +``sample.fraction`` and its ``scope`` sentence report 1/15, where that harness +left both as the tenth's 1/10 while running ``Fraction(1, 15)``. Neither field +reaches any code path; both are receipt prose. + +Finally it carries the 1/10 refusal diagnostic's committed ``sys.monitoring`` +RAISE block verbatim, inserted at the same point in the file, so the cause the +three nested catch-alls discard is observed rather than inferred. + +Everything else, including the CPU and wall ceilings, the sampler, the flush +cadence, the receipt capture and every refusal, is byte-for-byte the tenth +harness's. Max chose 1/15 now and 1/10 when the machine is quiet (2026-09-17), +so this file exists beside the tenth harness rather than replacing it. + +Original docstring follows. + +The native-scale lane's 1/10 run: above the transport ceiling that was. + +Identical to the lane's baseline harness except for the sample, the ceilings and +one addition. The sample is 1/10 -- 158,737 of the 1,587,376 source households, +where the single bounded encode refused at 96,860 -- so it runs against its own +source tree, whose ``selection-request.json`` carries ``fraction: [1,10]`` +because the staged pilot request pins 1/1000 and ``_request`` requires the +argument to match it. Every other source file in that tree is an APFS clone of +the staged original and hashes identically; the recovered originals are not +written, linked or moved. + +The addition: every flush captures the roster transport's own on-disk receipt +(``snapshots/preparation/header.json``) and the allocation receipt from the +manifest if one exists yet. The ceiling question is answered by the preparation +and allocation nodes, which run in the first minutes, so a run the CPU ceiling +truncates still carries the evidence for it. + +Original docstring follows. + +Baseline for the native-scale lane: the 19-node graph, uncapped by node. + +This is ``harness19_verify_once.py`` from the verify-once lane with four +additions and no change to what is measured -- same call, same arguments, same +sample, same seed, same sampler, same interval, same refusals: + +* the ceilings are the native-scale brief's (9,000 CPU-s, 48 GiB, 12,000 wall-s) + rather than the verify-once lane's, so no node is cut short; +* a current-resident-size series is sampled every second through ``libproc``'s + ``proc_pidinfo``, because ``ru_maxrss`` is a high-water mark and a retention + slope needs the trace, not the peak; +* the ``run_atomic_survey_financial`` call is timed on its own, so the runner + time outside the node loop is a subtraction between measured quantities + rather than an inference; +* the returned manifest's per-node ``wall_time_s`` is recorded, which is what + the subtraction subtracts. + +Nothing is wrapped: the runtime still refuses instrumented producers, so +attribution is still the stack sampler's. + +Original docstring follows. + +Nineteen-node measurement for the verify-once lane, on this branch only. + +The instrumentation is ``probe_verify_once.py`` unchanged -- the same stack +sampler at the same interval, the same ceilings, the same refusals, the same +output shape. The only difference is the call: this runs the base financial +graph with the exact arguments the accepted v4 cold pilot ran +(``~/PolicyEngine/_recovered/pilot-runs/native19-v2/run/test_native_nineteen_node_financial.py``: +fraction 1/1000, seed 20260908, geography seed 20260908, +``demographic_conditioning=True``, ``n_estimators=2``, ``resume="auto"``), so +its wall, CPU and peak RSS are comparable with that receipt's +5,362.05 s / 5,278.61 s / 8.67 GiB. + +Two differences from that receipt are known and are not this change: it ran on +source snapshot ``2ca11c85a``, and its ``person-income-attachment.h5`` was the +pre-restoration ``5996dcdd...`` rather than the staged ``9ebc0ef2...``. Both +files are exactly 666,333,922 B, so the bytes the source tree hashes are the +same volume either way. + +Original docstring follows. + +Before/after measurement for the verify-once lane: the nine-node prefix. + +A parameterised copy of the v5 pilot probe +(``~/PolicyEngine/_recovered/pilot-runs/native45-v5/probe_repeated_verification.py``). +Everything that decides *what is measured* is verbatim: the same nine-node +population prefix, the same 1/1000 sample, the same seed, the same stack +sampler at the same interval, the same ceilings, the same output shape. The +only changes are the ones a before/after comparison needs: + +* the source tree, the staged run inputs and the output directory come from + the environment, so the identical file measures both trees; +* every imported ``microcosm`` module is asserted to live under that tree, + so a stray editable install cannot silently measure the wrong source; +* a resident-memory ceiling is enforced alongside the CPU ceiling; +* the load average and the interpreter's third-party versions are recorded + with each run, because this machine carries other lanes' work. + +Sources are read by path and never written. Nothing here is a build, a +certification or a release artifact. + + MEASURE_BASE= MEASURE_RUN= \ + MEASURE_PROBE= MEASURE_LABEL=
/`` + and carries an HMAC-SHA256 made with a per-user key kept outside the memo + root, so a copied or foreign memo directory confers nothing. A lookup + accepts an entry only if the index is a bounded regular non-symlink file + in canonical form, its MAC verifies, its key document equals the requested + one byte for byte and every blob is a regular non-symlink file of the + recorded size and SHA-256. Anything else is a counted miss, never a + partial trust: the owner recomputes from source and the fresh entry + atomically replaces the rejected one. On a miss the code identity and the + file inputs are recomputed after the computation; a value is stored only + if neither moved and its encoding is proven to decode to exactly the + computed value. + +Activation + Off by default, so an ordinary call runs exactly the unmemoized path. + ``with source_memo(root):`` enables it for the current context; + ``source_memo(None)`` disables it inside that context. Otherwise the + ``MICROCOSM_US_SOURCE_MEMO`` environment variable names the root. + ``MICROCOSM_US_SOURCE_MEMO_KEY`` (or ``key_path=``) names the HMAC key + file, by default ``$XDG_CONFIG_HOME/microcosm/us-source-memo.key`` or + ``~/.config/microcosm/us-source-memo.key``, created 0600 on first use. + The memo is a cache: deleting its root is always safe. +""" + +from __future__ import annotations + +import _csv +import _json +import contextvars +import csv +import hashlib +import hmac +import importlib +import json +import json.decoder +import json.encoder +import json.scanner +import math +import os +import platform +import re +import secrets +import stat +import struct +import sys +import sysconfig +import tempfile +import threading +import time +import zipfile +import zlib +from collections import Counter +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from types import CodeType, FunctionType + +import numpy as np +import pandas as pd + +PROTOCOL = "microcosm.us.source-memo.v1" +ROOT_ENV = "MICROCOSM_US_SOURCE_MEMO" +KEY_ENV = "MICROCOSM_US_SOURCE_MEMO_KEY" +_KEY_DOMAIN = PROTOCOL.encode("ascii") + b"\0" +_MAC_DOMAIN = PROTOCOL.encode("ascii") + b"\0index\0" +_LAYOUT = "v1" +_SECRET_BYTES = 32 +_INDEX_MAX_BYTES = 1024**2 +_BLOB_MAX_BYTES = 16 * 1024**3 +_BLOBS_MAX = 64 +_CHUNK = 1024**2 +_COMPILED_MAX = 64 +_HEX = frozenset("0123456789abcdef") +_DISABLED = object() +_SKIP = object() +_ACTIVE = contextvars.ContextVar("microcosm_us_source_memo", default=None) +_LOCK = threading.RLock() +_STATISTICS = Counter() +_ENVIRONMENT_MEMOS = {} +_COMPILED = {} +_RUNTIME = None +_SELF_SHA256 = hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + + +class SourceMemoError(ValueError): + """Static refusal code; never names a path, key or source value.""" + + +def _require(condition, code): + if not condition: + raise SourceMemoError(code) + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def canonical(value) -> bytes: + """Sorted compact JSON as strict UTF-8: the only encoding of keys and indexes. + + Strict UTF-8 refuses a lone surrogate instead of escaping it, so two + distinct strings never share an encoding. + """ + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +def ordered(value) -> bytes: + """Compact strict-UTF-8 JSON preserving mapping order, for JSON-native values.""" + return json.dumps( + value, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ).encode("utf-8") + + +def _is_hex(value, size=64): + return type(value) is str and len(value) == size and set(value) <= _HEX + + +def _count(namespace, event, amount=1): + with _LOCK: + _STATISTICS[f"{namespace}:{event}"] += amount + _STATISTICS[f"*:{event}"] += amount + + +def statistics() -> dict: + """Process-wide counters and seconds, by ``namespace:event`` and ``*:event``.""" + with _LOCK: + return {key: _STATISTICS[key] for key in sorted(_STATISTICS)} + + +def reset_statistics() -> None: + with _LOCK: + _STATISTICS.clear() + + +# -- identities ------------------------------------------------------------- + + +def _binary_sha(module) -> str: + """Digest of the file providing a C module, libpython when it is built in. + + The interpreter is usually reached through a virtual environment's + symlink, so this one path is resolved before it is hashed. + """ + path = getattr(module, "__file__", None) + if path is None: + path = ( + Path(sysconfig.get_config_var("LIBDIR")) + / sysconfig.get_config_var("LDLIBRARY") + if sysconfig.get_config_var("Py_ENABLE_SHARED") + else Path(sys.executable) + ) + return file_sha256(os.path.realpath(path))[0] + + +def runtime_identity() -> dict: + """Interpreter, platform and parser-provider identity, once per process.""" + global _RUNTIME + with _LOCK: + if _RUNTIME is None: + _RUNTIME = { + "memo_sha256": _SELF_SHA256, + "python": sys.version, + "hexversion": sys.hexversion, + "cache_tag": sys.implementation.cache_tag, + "optimize": sys.flags.optimize, + "machine": platform.machine(), + "system": platform.system(), + "byteorder": sys.byteorder, + "zlib_runtime_version": zlib.ZLIB_RUNTIME_VERSION, + "c_providers_sha256": { + name: _binary_sha(module) + for name, module in ( + ("_csv", _csv), + ("_json", _json), + ("zlib", zlib), + ) + }, + "stdlib_sha256": { + name: _sha(Path(module.__file__).read_bytes()) + for name, module in ( + ("csv", csv), + ("json", json), + ("json.decoder", json.decoder), + ("json.encoder", json.encoder), + ("json.scanner", json.scanner), + ("zipfile", zipfile), + ) + }, + } + return json.loads(canonical(_RUNTIME)) + + +def _self_unchanged() -> bool: + try: + return _sha(Path(__file__).read_bytes()) == _SELF_SHA256 + except OSError: + return False + + +def _compiled(path: str, payload: bytes) -> dict: + """Code objects compiled from exact module bytes, by qualified name.""" + key = (path, _sha(payload)) + with _LOCK: + codes = _COMPILED.get(key) + if codes is None: + codes = {} + + def visit(code): + codes.setdefault(code.co_qualname, []).append(code) + for value in code.co_consts: + if isinstance(value, CodeType): + visit(value) + + visit(compile(payload, path, "exec", dont_inherit=True)) + with _LOCK: + while len(_COMPILED) >= _COMPILED_MAX: + del _COMPILED[next(iter(_COMPILED))] + _COMPILED[key] = codes + return codes + + +def _immutable(value, depth=0): + """A JSON form of an immutable constant, or _SKIP for anything else.""" + _require(depth <= 32, "MEMO_CONSTANT_DEPTH") + kind = type(value) + if value is None or kind in (bool, int, str): + return [kind.__name__, value] + if kind is float: + return ["float", value.hex()] + if kind is bytes: + return ["bytes", value.hex()] + if kind is re.Pattern: + return ["pattern", _immutable(value.pattern, depth + 1), value.flags] + if kind in (tuple, frozenset): + items = [_immutable(item, depth + 1) for item in value] + if any(item is _SKIP for item in items): + return _SKIP + if kind is frozenset: + items = sorted(items, key=canonical) + return [kind.__name__, items] + return _SKIP + + +def _function(value, module, codes: dict, seen: set) -> list: + """Identity of one bound function, checking code the module itself defines.""" + if id(value) in seen: + return ["seen", value.__qualname__] + seen.add(id(value)) + code = value.__code__ + own = value.__globals__ is vars(module) and code.co_filename != "" + if own: + candidates = codes.get(code.co_qualname, ()) + _require(any(code == item for item in candidates), "MEMO_LIVE_CODE") + return [ + value.__module__, + value.__qualname__, + code.co_qualname, + own, + [ + _function(cell.cell_contents, module, codes, seen) + for cell in value.__closure__ or () + if isinstance(cell.cell_contents, FunctionType) + ], + ] + + +def live_code(*modules) -> dict: + """Current file bytes, checked loaded code and constants of ``modules``. + + Raises ``SourceMemoError`` when a function or method a module defines no + longer runs the code compiled from its file, so the memo is bypassed. + """ + result = {} + for module in modules: + path = module.__file__ + payload = Path(path).read_bytes() + codes = _compiled(path, payload) + functions, constants, seen = {}, {}, set() + for name, value in sorted(vars(module).items()): + if name.startswith("__"): + continue + if isinstance(value, FunctionType): + functions[name] = _function(value, module, codes, seen) + elif isinstance(value, type) and value.__module__ == module.__name__: + for method_name, method in sorted(vars(value).items()): + if isinstance(method, (staticmethod, classmethod)): + method = method.__func__ + parts = ( + (method.fget, method.fset, method.fdel) + if isinstance(method, property) + else (method,) + ) + for index, part in enumerate(parts): + if isinstance(part, FunctionType): + functions[f"{name}.{method_name}/{index}"] = _function( + part, module, codes, seen + ) + else: + form = _immutable(value) + if form is not _SKIP: + constants[name] = form + result[module.__name__] = { + "sha256": _sha(payload), + "functions": functions, + "constants": constants, + } + return result + + +@dataclass(frozen=True) +class FileInput: + """A source file hashed at call time through a no-follow descriptor.""" + + role: str + path: object + + +@dataclass(frozen=True) +class DigestInput: + """Bytes the owner already holds and hashed itself (never a claimed digest).""" + + role: str + sha256: str + size: int + + +def _stat_identity(value): + return ( + value.st_dev, + value.st_ino, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + + +def file_sha256(path, *, maximum=None) -> tuple[str, int]: + """SHA-256 and size of one regular non-symlink file, unchanged while read.""" + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + before = os.fstat(descriptor) + _require(stat.S_ISREG(before.st_mode), "MEMO_INPUT_KIND") + _require(maximum is None or before.st_size <= maximum, "MEMO_INPUT_SIZE") + digest, count = hashlib.sha256(), 0 + with os.fdopen(descriptor, "rb", buffering=0, closefd=False) as stream: + while chunk := stream.read(_CHUNK): + count += len(chunk) + _require(count <= before.st_size, "MEMO_INPUT_CHANGED") + digest.update(chunk) + _require( + count == before.st_size + and _stat_identity(before) == _stat_identity(os.fstat(descriptor)), + "MEMO_INPUT_CHANGED", + ) + return digest.hexdigest(), count + finally: + os.close(descriptor) + + +def _inputs(inputs): + result = [] + for item in inputs: + if type(item) is FileInput: + digest, size = file_sha256(item.path) + else: + _require(type(item) is DigestInput, "MEMO_INPUT_TYPE") + _require( + _is_hex(item.sha256) and type(item.size) is int and item.size >= 0, + "MEMO_INPUT_DIGEST", + ) + digest, size = item.sha256, item.size + _require(type(item.role) is str and item.role, "MEMO_INPUT_ROLE") + result.append({"role": item.role, "sha256": digest, "bytes": size}) + return result + + +# -- strict value comparison -------------------------------------------------- + + +def strict_equal(left, right) -> bool: + """Type-exact deep equality, including mapping order and signed zeros.""" + stack = [(left, right)] + while stack: + a, b = stack.pop() + if type(a) is not type(b): + return False + kind = type(a) + if kind in (list, tuple): + if len(a) != len(b): + return False + stack.extend(zip(a, b, strict=True)) + elif kind is dict: + if list(a) != list(b): + return False + stack.extend(zip(a.values(), b.values(), strict=True)) + stack.extend(zip(a, b, strict=True)) + elif kind is float: + if not (a == b and math.copysign(1.0, a) == math.copysign(1.0, b)): + return False + elif kind in (str, int, bool, bytes, type(None)): + if a != b: + return False + else: + return False + return True + + +def exact_text(value) -> bool: + """A str that strict UTF-8 JSON round-trips exactly: no surrogate code points. + + Strictly decoded UTF-8 never contains one, and ``ordered`` refuses to + encode one, so this only lets a proof refuse early instead of at encoding. + """ + if value.isascii(): + return True + try: + value.encode("utf-8") + except UnicodeEncodeError: + return False + return True + + +def json_native(value) -> bool: + """Only dict(str keys)/list/str/int/bool/None: ``ordered`` round-trips exactly. + + Strings must be surrogate-free (``exact_text``). Mapping order survives + because ``ordered`` does not sort keys and the decoder preserves order. + """ + stack = [value] + while stack: + item = stack.pop() + kind = type(item) + if kind is str: + if not exact_text(item): + return False + elif kind is list: + stack.extend(item) + elif kind is dict: + if not all(type(key) is str and exact_text(key) for key in item): + return False + stack.extend(item.values()) + elif kind not in (int, bool, type(None)): + return False + return True + + +# -- the store ------------------------------------------------------------------ + + +def _default_key_path() -> Path: + base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config") + return Path(base) / "microcosm" / "us-source-memo.key" + + +def _private_directory(path: Path) -> Path: + try: + info = path.lstat() + except FileNotFoundError: + try: + path.mkdir(mode=0o700) + except FileExistsError: + pass + info = path.lstat() + _require(stat.S_ISDIR(info.st_mode), "MEMO_DIRECTORY") + _require(info.st_uid == os.getuid(), "MEMO_DIRECTORY_OWNER") + _require(not info.st_mode & 0o022, "MEMO_DIRECTORY_MODE") + return path + + +def _load_secret(path: Path) -> bytes: + path = Path(os.path.abspath(path)) + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + except FileNotFoundError: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + # Write a complete key under a private name, then link it into place: + # a concurrent first use either wins the link or reads the winner's key. + descriptor, temporary = tempfile.mkstemp(prefix=".key-", dir=path.parent) + try: + os.write(descriptor, secrets.token_bytes(_SECRET_BYTES)) + os.fsync(descriptor) + os.close(descriptor) + try: + os.link(temporary, path) + except FileExistsError: + pass + finally: + os.unlink(temporary) + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + info = os.fstat(descriptor) + _require( + stat.S_ISREG(info.st_mode) + and info.st_uid == os.getuid() + and not info.st_mode & 0o077 + and info.st_size == _SECRET_BYTES, + "MEMO_KEY_FILE", + ) + secret = os.read(descriptor, _SECRET_BYTES + 1) + _require(len(secret) == _SECRET_BYTES, "MEMO_KEY_FILE") + return secret + finally: + os.close(descriptor) + + +def _read_regular(path: Path, maximum: int, exact=None) -> bytes | None: + """Whole regular non-symlink file, or None if absent. Raises on anything odd.""" + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + except FileNotFoundError: + return None + try: + before = os.fstat(descriptor) + _require(stat.S_ISREG(before.st_mode), "MEMO_ENTRY_KIND") + _require(before.st_size <= maximum, "MEMO_ENTRY_SIZE") + _require(exact is None or before.st_size == exact, "MEMO_ENTRY_SIZE") + # One read per GiB: a blob up to 1 GiB is a single allocation, never + # copied again. One byte past the recorded size detects growth. + parts, remaining = [], before.st_size + 1 + with os.fdopen(descriptor, "rb", buffering=0, closefd=False) as stream: + while remaining > 0: + chunk = stream.read(min(remaining, 1024**3)) + if not chunk: + break + parts.append(chunk) + remaining -= len(chunk) + value = parts[0] if len(parts) == 1 else b"".join(parts) + del parts + _require( + len(value) == before.st_size + and _stat_identity(before) == _stat_identity(os.fstat(descriptor)), + "MEMO_ENTRY_CHANGED", + ) + return value + finally: + os.close(descriptor) + + +class _Memo: + def __init__(self, root, key_path=None): + root = Path(os.path.realpath(os.path.abspath(root))) + root.mkdir(mode=0o700, parents=True, exist_ok=True) + _private_directory(root) + key_path = Path( + os.path.realpath( + os.path.abspath( + key_path or os.environ.get(KEY_ENV) or _default_key_path() + ) + ) + ) + _require(not key_path.is_relative_to(root), "MEMO_KEY_INSIDE_ROOT") + self._secret = _load_secret(key_path) + self.root = _private_directory(root / _LAYOUT) + for name in ("index", "blobs", "tmp"): + _private_directory(self.root / name) + + # Paths are derived only from validated lowercase hex digests. + def _index_path(self, key: str) -> Path: + return self.root / "index" / key[:2] / f"{key}.json" + + def _blob_path(self, digest: str) -> Path: + return self.root / "blobs" / digest[:2] / digest + + def _mac(self, body: bytes) -> str: + return hmac.new(self._secret, _MAC_DOMAIN + body, hashlib.sha256).hexdigest() + + def lookup(self, namespace, key, document_bytes): + """Exact verified blobs, or None. Never raises for a bad entry.""" + try: + raw = _read_regular(self._index_path(key), _INDEX_MAX_BYTES) + if raw is None: + return None + entry = json.loads(raw) + _require(type(entry) is dict and canonical(entry) == raw, "INDEX_FORM") + _require( + set(entry) == {"protocol", "key", "document", "blobs", "mac"}, + "INDEX_FIELDS", + ) + mac = entry.pop("mac") + _require( + _is_hex(mac) and hmac.compare_digest(mac, self._mac(canonical(entry))), + "INDEX_MAC", + ) + _require( + entry["protocol"] == PROTOCOL + and entry["key"] == key + and canonical(entry["document"]) == document_bytes, + "INDEX_KEY", + ) + blobs = entry["blobs"] + _require( + type(blobs) is list + and 0 < len(blobs) <= _BLOBS_MAX + and all( + type(item) is dict + and set(item) == {"sha256", "bytes"} + and _is_hex(item["sha256"]) + and type(item["bytes"]) is int + and 0 <= item["bytes"] <= _BLOB_MAX_BYTES + for item in blobs + ), + "INDEX_BLOBS", + ) + values = [] + for item in blobs: + value = _read_regular( + self._blob_path(item["sha256"]), _BLOB_MAX_BYTES, item["bytes"] + ) + _require(value is not None, "BLOB_MISSING") + _require(_sha(value) == item["sha256"], "BLOB_DIGEST") + values.append(value) + return values + except SourceMemoError as error: + _count(namespace, f"rejected:{error}") + except (OSError, ValueError, TypeError, KeyError, RecursionError): + _count(namespace, "rejected:ENTRY_UNREADABLE") + return None + + def _publish(self, target: Path, data: bytes): + _private_directory(target.parent) + descriptor, temporary = tempfile.mkstemp(prefix=".memo-", dir=self.root / "tmp") + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(temporary, 0o400) + os.replace(temporary, target) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + def _blob_present(self, path, digest, size): + try: + return file_sha256(path, maximum=size) == (digest, size) + except (OSError, SourceMemoError): + return False + + def store(self, namespace, key, document, blobs): + # lookup() refuses any other count, so such an entry would never hit. + _require(type(blobs) is list and 0 < len(blobs) <= _BLOBS_MAX, "BLOB_COUNT") + described = [] + for value in blobs: + _require( + type(value) is bytes and len(value) <= _BLOB_MAX_BYTES, "BLOB_SIZE" + ) + digest = _sha(value) + described.append({"sha256": digest, "bytes": len(value)}) + path = self._blob_path(digest) + if not self._blob_present(path, digest, len(value)): + self._publish(path, value) + entry = { + "protocol": PROTOCOL, + "key": key, + "document": document, + "blobs": described, + } + entry["mac"] = self._mac(canonical(entry)) + body = canonical(entry) + _require(len(body) <= _INDEX_MAX_BYTES, "INDEX_SIZE") + self._publish(self._index_path(key), body) + _count(namespace, "stored") + _count(namespace, "stored_bytes", sum(len(value) for value in blobs)) + + +@contextmanager +def source_memo(root, *, key_path=None): + """Enable the memo at ``root`` for this context; ``None`` disables it.""" + memo = _DISABLED if root is None else _Memo(root, key_path) + token = _ACTIVE.set(memo) + try: + yield None if memo is _DISABLED else memo + finally: + _ACTIVE.reset(token) + + +def _current(): + memo = _ACTIVE.get() + if memo is _DISABLED: + return None + if memo is not None: + return memo + root = os.environ.get(ROOT_ENV) + if not root: + return None + selector = (root, os.environ.get(KEY_ENV)) + with _LOCK: + memo = _ENVIRONMENT_MEMOS.get(selector) + if memo is None: + memo = _ENVIRONMENT_MEMOS[selector] = _Memo(root, selector[1]) + return memo + + +def enabled() -> bool: + return _current() is not None + + +def memoized( + namespace, *, code, inputs, parameters, compute, encode, decode, proof=None +): + """``compute()``, or the value an identical earlier computation recorded. + + ``code`` returns the owner's JSON code identity and is evaluated before the + lookup and again after a miss's computation. ``inputs`` are FileInput or + DigestInput items and ``parameters`` a canonical JSON value; either may be + a zero-argument callable, evaluated only when the memo is enabled. + ``encode(value)`` returns a list of bytes blobs and ``decode(blobs)`` + rebuilds the value. ``proof(value, blobs)`` must return True only if + ``decode(blobs)`` is exactly ``value``; by default the decoded value is + compared with ``strict_equal``. Exceptions from ``compute`` always + propagate unchanged, so owner refusals keep their codes and timing, and + nothing is recorded for them. + """ + memo = _current() + if memo is None: + return compute() + started = time.perf_counter() + if not _self_unchanged(): + _count(namespace, "bypassed:MEMO_CODE_CHANGED") + return compute() + try: + # Callables keep a disabled memo free of any key-forming work. + inputs = tuple(inputs() if callable(inputs) else inputs) + parameters = parameters() if callable(parameters) else parameters + code_before = code() + document = { + "protocol": PROTOCOL, + "namespace": namespace, + "runtime": runtime_identity(), + "code": code_before, + "inputs": _inputs(inputs), + "parameters": parameters, + } + document_bytes = canonical(document) + except Exception as error: + # An identity that cannot be formed is never a key. The owner's own + # path then runs unchanged and refuses, if it must, with its own code + # at its own point. + _count(namespace, f"bypassed:{type(error).__name__}") + return compute() + key = _sha(_KEY_DOMAIN + document_bytes) + blobs = memo.lookup(namespace, key, document_bytes) + _count(namespace, "key_seconds", time.perf_counter() - started) + if blobs is not None: + decoding = time.perf_counter() + try: + value = decode(blobs) + except Exception: + _count(namespace, "rejected:DECODE") + else: + _count(namespace, "hit") + _count(namespace, "decode_seconds", time.perf_counter() - decoding) + return value + del blobs + _count(namespace, "miss") + computing = time.perf_counter() + value = compute() + _count(namespace, "compute_seconds", time.perf_counter() - computing) + storing = time.perf_counter() + try: + if code() != code_before: + _count(namespace, "not_stored:CODE_CHANGED") + return value + if _inputs(inputs) != document["inputs"] or not _self_unchanged(): + _count(namespace, "not_stored:INPUT_CHANGED") + return value + encoded = encode(value) + _require( + type(encoded) is list and all(type(item) is bytes for item in encoded), + "ENCODED_TYPE", + ) + verified = ( + proof(value, encoded) + if proof is not None + else strict_equal(decode(encoded), value) + ) + if verified is not True: + _count(namespace, "not_stored:ROUND_TRIP") + return value + memo.store(namespace, key, document, encoded) + except Exception as error: + # Storing is an optimization for later calls: a failure to record + # never changes the value this call returns. + _count(namespace, f"not_stored:{type(error).__name__}") + finally: + _count(namespace, "store_seconds", time.perf_counter() - storing) + return value + + +# -- small codecs shared by owners --------------------------------------------- + + +def encode_json(value): + """One blob of order-preserving JSON; refuses anything not JSON-native.""" + _require(json_native(value), "NOT_JSON_NATIVE") + return [ordered(value)] + + +def decode_json(blobs: list[bytes]): + _require(len(blobs) == 1, "BLOB_COUNT") + return json.loads(blobs[0]) + + +def prove_json(value, blobs: list[bytes]) -> bool: + """``encode_json`` admits only JSON-native values, which ``ordered`` round-trips + exactly; decoding again would only repeat that guarantee at the cost of a + second copy of the value.""" + return len(blobs) == 1 and json_native(value) + + +# -- exact typed values ------------------------------------------------------------ + +_TYPED_DEPTH = 64 + + +def _to_typed(value, depth: int): + _require(depth <= _TYPED_DEPTH, "TYPED_DEPTH") + kind = type(value) + if value is None or kind in (bool, int): + return value + if kind is str: + _require(exact_text(value), "TYPED_TEXT") + return value + if kind is float: + return ["F", struct.pack(">d", value).hex()] + if kind is bytes: + return ["B", value.hex()] + if kind in (list, tuple): + tag = "L" if kind is list else "T" + return [tag, *(_to_typed(item, depth + 1) for item in value)] + if kind is dict: + _require( + all(type(key) is str and exact_text(key) for key in value), "TYPED_KEY" + ) + return [ + "D", + *([key, _to_typed(item, depth + 1)] for key, item in value.items()), + ] + raise SourceMemoError("TYPED_VALUE") + + +def _from_typed(value, depth: int): + _require(depth <= _TYPED_DEPTH, "TYPED_DEPTH") + if type(value) is not list: + _require(value is None or type(value) in (bool, int, str), "TYPED_TAG") + return value + _require(bool(value), "TYPED_TAG") + tag, items = value[0], value[1:] + if tag in ("F", "B"): + _require(len(items) == 1 and type(items[0]) is str, "TYPED_TAG") + raw = bytes.fromhex(items[0]) + return struct.unpack(">d", raw)[0] if tag == "F" else raw + if tag in ("L", "T"): + decoded = [_from_typed(item, depth + 1) for item in items] + return decoded if tag == "L" else tuple(decoded) + _require(tag == "D", "TYPED_TAG") + result = {} + for pair in items: + _require(type(pair) is list and len(pair) == 2, "TYPED_KEY") + key = pair[0] + _require(type(key) is str and key not in result, "TYPED_KEY") + result[key] = _from_typed(pair[1], depth + 1) + return result + + +def encode_typed(value) -> bytes: + """Exact JSON of None/bool/int/str/float/bytes/list/tuple/dict (str keys). + + Every container is a tagged list, so tuples stay tuples, mapping order + survives and floats retain their IEEE-754 bits, including NaN payloads. + """ + return ordered(_to_typed(value, 0)) + + +def decode_typed(raw: bytes): + return _from_typed(json.loads(raw), 0) + + +# -- exact data frames ------------------------------------------------------------ + +_NUMPY_KINDS = frozenset("biuf") +_STRING_STORAGES = frozenset({"python", "pyarrow"}) + + +def _string_form(dtype) -> list | None: + """``[storage, na]`` of an exact pandas StringDtype, else None.""" + if type(dtype) is not pd.StringDtype or dtype.storage not in _STRING_STORAGES: + return None + na = dtype.na_value + if na is pd.NA: + return [dtype.storage, "NA"] + if type(na) is float and math.isnan(na): + return [dtype.storage, "nan"] + return None + + +def _string_dtype(form: list): + _require( + type(form) is list + and len(form) == 2 + and form[0] in _STRING_STORAGES + and form[1] in ("NA", "nan"), + "FRAME_DTYPE", + ) + return pd.StringDtype(form[0], na_value=pd.NA if form[1] == "NA" else np.nan) + + +def _numpy_dtype(form: str): + _require(type(form) is str, "FRAME_DTYPE") + dtype = np.dtype(form) + _require(dtype.kind in _NUMPY_KINDS and dtype.str == form, "FRAME_DTYPE") + return dtype + + +def _string_values(series) -> list: + values = series.array.to_numpy(dtype=object, na_value=None).tolist() + missing = series.isna().to_numpy(dtype=bool) + _require(len(values) == len(missing), "FRAME_VALUES") + for value, absent in zip(values, missing.tolist(), strict=True): + if absent: + _require(value is None, "FRAME_VALUES") + else: + _require(type(value) is str and exact_text(value), "FRAME_VALUES") + return values + + +def _object_values(series) -> list: + """Encode only inert scalar cells; never serialize arbitrary Python objects. + + Object columns can contain distinct missing markers. Keep None, pd.NA + and every float's bits separate rather than normalizing through isna(). + """ + values = [] + for value in series.to_numpy(copy=False): + if value is pd.NA: + values.append(["NA"]) + else: + _require( + value is None or type(value) in (bool, int, float, str, bytes), + "FRAME_OBJECT", + ) + values.append(_to_typed(value, 0)) + return values + + +def _decode_object_values(raw: bytes, rows: int) -> list: + values = json.loads(raw) + _require(type(values) is list and len(values) == rows, "FRAME_ROWS") + decoded = [] + for value in values: + if value == ["NA"]: + decoded.append(pd.NA) + else: + item = _from_typed(value, 0) + _require( + item is None or type(item) in (bool, int, float, str, bytes), + "FRAME_OBJECT", + ) + decoded.append(item) + return decoded + + +def encode_frame(frame) -> list[bytes]: + """A header blob, then one blob per column; refuses any other frame shape. + + Admits exactly: a plain DataFrame with no attrs, a RangeIndex, unique str + column labels held in an object or StringDtype index, and columns that are + numpy bool/int/uint/float (stored as their raw bytes, so NaN payloads and + signed zeros survive) or a python/pyarrow StringDtype with either missing + marker (stored as JSON text or null), or object columns of exact Python + scalar cells (None/bool/int/float/str/bytes/pd.NA, with tagged encoding). + """ + _require(type(frame) is pd.DataFrame, "FRAME_TYPE") + _require(not frame.attrs and frame.flags.allows_duplicate_labels, "FRAME_FLAGS") + index = frame.index + _require( + type(index) is pd.RangeIndex + and (index.name is None or type(index.name) is str), + "FRAME_INDEX", + ) + columns = frame.columns + names = list(columns) + _require( + type(columns) is pd.Index + and columns.name is None + and all(type(name) is str and exact_text(name) for name in names) + and len(set(names)) == len(names), + "FRAME_COLUMNS", + ) + if isinstance(columns.dtype, np.dtype) and columns.dtype.kind == "O": + _require(columns.dtype.metadata is None, "FRAME_DTYPE") + labels = "object" + else: + labels = _string_form(columns.dtype) + _require(labels is not None, "FRAME_COLUMNS") + header = { + "format": "microcosm.us.source-memo.frame.v1", + "rows": len(frame), + "index": [index.start, index.stop, index.step, index.name], + "labels": labels, + "columns": [], + } + blobs = [] + for position, name in enumerate(names): + series = frame.iloc[:, position] + dtype = series.dtype + if isinstance(dtype, np.dtype): + _require(dtype.metadata is None, "FRAME_DTYPE") + if isinstance(dtype, np.dtype) and dtype.kind == "O": + header["columns"].append([name, "object", None]) + blobs.append(ordered(_object_values(series))) + continue + if type(dtype) is not pd.StringDtype: + _require( + isinstance(dtype, np.dtype) and dtype.kind in _NUMPY_KINDS, + "FRAME_DTYPE", + ) + array = series.to_numpy(copy=False) + _require(array.dtype == dtype and array.ndim == 1, "FRAME_VALUES") + header["columns"].append([name, "numpy", dtype.str]) + blobs.append(np.ascontiguousarray(array).tobytes()) + continue + form = _string_form(dtype) + _require(form is not None, "FRAME_DTYPE") + header["columns"].append([name, "string", form]) + blobs.append(ordered(_string_values(series))) + return [ordered(header), *blobs] + + +def _frame_header(raw: bytes) -> dict: + header = json.loads(raw) + _require( + type(header) is dict + and list(header) == ["format", "rows", "index", "labels", "columns"] + and header["format"] == "microcosm.us.source-memo.frame.v1" + and type(header["rows"]) is int + and header["rows"] >= 0 + and type(header["index"]) is list + and len(header["index"]) == 4 + and all(type(v) is int for v in header["index"][:3]) + and (header["index"][3] is None or type(header["index"][3]) is str) + and type(header["columns"]) is list + and all( + type(column) is list and len(column) == 3 and type(column[0]) is str + for column in header["columns"] + ), + "FRAME_HEADER", + ) + return header + + +def decode_frame(blobs: list[bytes]): + """Rebuild exactly the frame ``encode_frame`` described, one block per column.""" + _require(type(blobs) is list and blobs, "FRAME_BLOBS") + header = _frame_header(blobs[0]) + columns = header["columns"] + _require(len(blobs) == 1 + len(columns), "FRAME_BLOBS") + start, stop, step, index_name = header["index"] + index = pd.RangeIndex(start, stop, step, name=index_name) + rows = header["rows"] + _require(len(index) == rows, "FRAME_ROWS") + names = [column[0] for column in columns] + _require(len(set(names)) == len(names), "FRAME_COLUMNS") + data = {} + for (name, kind, form), raw in zip(columns, blobs[1:], strict=True): + if kind == "numpy": + dtype = _numpy_dtype(form) + _require(len(raw) == rows * dtype.itemsize, "FRAME_ROWS") + array = np.frombuffer(raw, dtype=dtype).copy() + data[name] = pd.Series(array, index=index, name=name, copy=False) + elif kind == "object": + _require(form is None, "FRAME_DTYPE") + data[name] = pd.Series( + _decode_object_values(raw, rows), + dtype=object, + index=index, + name=name, + ) + else: + _require(kind == "string", "FRAME_DTYPE") + values = json.loads(raw) + _require( + type(values) is list + and len(values) == rows + and all(v is None or type(v) is str for v in values), + "FRAME_VALUES", + ) + data[name] = pd.Series( + pd.array(values, dtype=_string_dtype(form)), + index=index, + name=name, + copy=False, + ) + labels = header["labels"] + label_dtype = object if labels == "object" else _string_dtype(labels) + return pd.DataFrame( + data, index=index, columns=pd.Index(names, dtype=label_dtype), copy=False + ) + + +def decode_frames(blobs: list[bytes]) -> list: + """Consecutive ``encode_frame`` encodings, consumed exactly.""" + frames, position = [], 0 + while position < len(blobs): + width = 1 + len(_frame_header(blobs[position])["columns"]) + _require(position + width <= len(blobs), "FRAME_BLOBS") + frames.append(decode_frame(blobs[position : position + width])) + position += width + return frames + + +def frames_identical(left, right) -> bool: + """Type-, dtype-, label- and bit-exact equality of two data frames. + + Beyond ``encode_frame``'s shapes this is simply False, never an error. + """ + if type(left) is not pd.DataFrame or type(right) is not pd.DataFrame: + return False + if left.shape != right.shape or left.attrs or right.attrs: + return False + if left.flags.allows_duplicate_labels != right.flags.allows_duplicate_labels: + return False + a_index, b_index = left.index, right.index + if not ( + type(a_index) is pd.RangeIndex + and type(b_index) is pd.RangeIndex + and (a_index.start, a_index.stop, a_index.step, a_index.name) + == (b_index.start, b_index.stop, b_index.step, b_index.name) + ): + return False + a_columns, b_columns = left.columns, right.columns + if any( + isinstance(columns.dtype, np.dtype) and columns.dtype.metadata is not None + for columns in (a_columns, b_columns) + ): + return False + if not ( + type(a_columns) is type(b_columns) + and a_columns.name is None + and b_columns.name is None + and list(a_columns) == list(b_columns) + and all(type(name) is str for name in a_columns) + and all(type(name) is str for name in b_columns) + and type(a_columns.dtype) is type(b_columns.dtype) + and a_columns.dtype == b_columns.dtype + and repr(a_columns.dtype) == repr(b_columns.dtype) + ): + return False + for position in range(left.shape[1]): + a, b = left.iloc[:, position], right.iloc[:, position] + if any( + isinstance(series.dtype, np.dtype) and series.dtype.metadata is not None + for series in (a, b) + ): + return False + if not ( + type(a.dtype) is type(b.dtype) + and a.dtype == b.dtype + and repr(a.dtype) == repr(b.dtype) + and type(a.array) is type(b.array) + and a.name == b.name + ): + return False + if isinstance(a.dtype, np.dtype) and a.dtype.kind == "O": + try: + if ordered(_object_values(a)) != ordered(_object_values(b)): + return False + except SourceMemoError: + return False + elif type(a.dtype) is pd.StringDtype: + if _string_form(a.dtype) is None: + return False + try: + if _string_values(a) != _string_values(b): + return False + except SourceMemoError: + return False + else: + if not (isinstance(a.dtype, np.dtype) and a.dtype.kind in _NUMPY_KINDS): + return False + x, y = a.to_numpy(copy=False), b.to_numpy(copy=False) + if ( + x.dtype != y.dtype + or np.ascontiguousarray(x).tobytes() + != np.ascontiguousarray(y).tobytes() + ): + return False + return True + + +# -- third-party parser identity ---------------------------------------------------- + +# pandas options that choose the string dtypes a CSV read produces. +_PANDAS_OPTIONS = ("future.infer_string", "mode.string_storage") +# Parsing, type inference, hashing (isin/duplicated) and sorting code of the +# libraries a frame derivation runs, digested once per process as loaded. +_LIBRARY_MODULES = ( + "numpy._core._multiarray_umath", + "pandas._libs.algos", + "pandas._libs.hashtable", + "pandas._libs.lib", + "pandas._libs.parsers", + "pandas.io.parsers.base_parser", + "pandas.io.parsers.c_parser_wrapper", + "pandas.io.parsers.readers", + "pyarrow.lib", +) +_LIBRARIES = None + + +def _parser_value(value): + """Exact JSON identity of the defaults used by the CSV parser.""" + from enum import Enum + + immutable = _immutable(value) + if immutable is not _SKIP: + return immutable + if isinstance(value, Enum): + return [ + "enum", + type(value).__module__, + type(value).__qualname__, + value.name, + _parser_value(value.value), + ] + if type(value) is dict: + _require(all(type(key) is str for key in value), "MEMO_PARSER_DEFAULT") + return ["dict", [[key, _parser_value(value[key])] for key in sorted(value)]] + if type(value) in (list, tuple, set, frozenset): + items = [_parser_value(item) for item in value] + if type(value) in (set, frozenset): + items.sort(key=canonical) + return [type(value).__name__, items] + raise SourceMemoError("MEMO_PARSER_DEFAULT") + + +def _parser_module_identity(module, class_name): + """Bind source-defined parser functions/methods to their live code. + + A replacement is refused even when it forges the original function's + name. Check the defining globals as well as the code, since the same code + executed with other globals need not parse the same bytes. Overloads share + a qualified name; only the final definition is the runtime implementation. + """ + payload = Path(module.__file__).read_bytes() + codes = _compiled(module.__file__, payload) + owner = getattr(module, class_name) + _require( + isinstance(owner, type) and type(owner) is type(owner.__bases__[0]), + "MEMO_LIVE_PARSER", + ) + functions = {} + for name, candidates in codes.items(): + parts = name.split(".") + if ( + "<" in name + or parts[-1] == "__annotate__" + or not candidates[-1].co_flags & 1 # CO_OPTIMIZED: function, not class + or not (len(parts) == 1 or len(parts) == 2 and parts[0] == class_name) + ): + continue + value = vars(module if len(parts) == 1 else owner).get(parts[-1]) + if isinstance(value, (staticmethod, classmethod)): + value = value.__func__ + _require( + type(value) is FunctionType + and value.__globals__ is vars(module) + and value.__code__ == candidates[-1] + and ( + not value.__closure__ + or value.__code__.co_freevars == ("__class__",) + and len(value.__closure__) == 1 + and value.__closure__[0].cell_contents is owner + ), + "MEMO_LIVE_PARSER", + ) + functions[name] = { + "defaults": _parser_value(value.__defaults__), + "kwdefaults": _parser_value(value.__kwdefaults__), + } + # An added special method (for example __getattribute__) can alter the + # dispatch of all the checked methods without replacing any of them. + for name, value in vars(owner).items(): + if isinstance(value, FunctionType): + _require(f"{class_name}.{name}" in functions, "MEMO_LIVE_PARSER") + return {"sha256": _sha(payload), "functions": functions} + + +def _live_parser_identity(): + """The pandas C-engine dispatch chain used by ACS archive reads. + + Provider file digests alone cannot authenticate live Python bindings. + Verify these on every lookup, including calls made before a first memo + entry exists, rather than trusting a lazily captured reference snapshot. + """ + from collections.abc import Iterator + + import pandas.io.parsers as exports + import pandas.io.parsers.base_parser as base + import pandas.io.parsers.c_parser_wrapper as cparser + import pandas.io.parsers.readers as readers + from pandas._libs import parsers + + _require( + pd.read_csv is readers.read_csv + and exports.read_csv is readers.read_csv + and exports.TextFileReader is readers.TextFileReader + and readers.CParserWrapper is cparser.CParserWrapper + and readers.ParserBase is base.ParserBase + and cparser.ParserBase is base.ParserBase + and cparser.parsers is parsers + and readers.TextFileReader.__bases__ == (Iterator,) + and cparser.CParserWrapper.__bases__ == (base.ParserBase,) + and base.ParserBase.__bases__ == (object,), + "MEMO_LIVE_PARSER", + ) + # Native constructors must still belong to this exact extension type, + # rejecting a Python substitute or subclass. Cython may expose a mutable + # extension type, so check its read methods as well as its module binding. + native = parsers.TextReader + _require( + type(native) is type + and native.__module__ == "pandas._libs.parsers" + and native.__name__ == "TextReader" + and getattr(native.__new__, "__self__", None) is native + and getattr(native.__init__, "__objclass__", None) is native, + "MEMO_LIVE_PARSER", + ) + for name in ("read", "read_low_memory", "close"): + method = getattr(native, name) + _require( + getattr(method, "__objclass__", None) is native + or ( + type(method).__name__ == "cython_function_or_method" + and type(method).__module__.startswith("_cython_") + and method.__module__ == "pandas._libs.parsers" + and method.__qualname__ == f"TextReader.{name}" + ), + "MEMO_LIVE_PARSER", + ) + return { + "modules": { + module.__name__: _parser_module_identity(module, class_name) + for module, class_name in ( + (readers, "TextFileReader"), + (cparser, "CParserWrapper"), + (base, "ParserBase"), + ) + }, + "defaults": { + "readers.parser_defaults": _parser_value(readers.parser_defaults), + "readers._c_parser_defaults": _parser_value(readers._c_parser_defaults), + "base.parser_defaults": _parser_value(base.parser_defaults), + "readers.STR_NA_VALUES": _parser_value(readers.STR_NA_VALUES), + }, + } + + +def library_identity() -> dict: + """numpy/pandas/pyarrow versions and file digests plus the current pandas + string options and source-verified live CSV parser bindings/defaults. + Mutable parser state is checked on every call, including the first one.""" + global _LIBRARIES + with _LOCK: + if _LIBRARIES is None: + import pyarrow # an absent pyarrow leaves the identity unformable + + _LIBRARIES = { + "versions": { + "numpy": np.__version__, + "pandas": pd.__version__, + "pyarrow": pyarrow.__version__, + }, + "files_sha256": { + name: file_sha256( + os.path.realpath(importlib.import_module(name).__file__) + )[0] + for name in _LIBRARY_MODULES + }, + } + static = json.loads(canonical(_LIBRARIES)) + return { + **static, + "pandas_options": {name: pd.get_option(name) for name in _PANDAS_OPTIONS}, + "live_csv_parser": _live_parser_identity(), + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py b/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py index ce6b6e296..4f39da34b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py @@ -8,14 +8,17 @@ from __future__ import annotations +import hashlib import re from collections.abc import Mapping +from dataclasses import asdict, dataclass +from types import MappingProxyType from typing import Any import numpy as np import pandas as pd -from microcosm.build.us_runtime.puf_support import ( +from microcosm.build.us_runtime.operator_column_contracts import ( PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID, ) from microcosm.build.us_runtime.support_provenance import ( @@ -36,7 +39,14 @@ Weights, ) -__all__ = ["assemble_spines"] +__all__ = [ + "SpinePreparation", + "SpineHarmonization", + "assemble_spines", + "prepare_spines", + "stack_survey_spines", + "harmonize_spine_weights", +] _CHANNEL_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$") _SUPPORT_CLONE_INDEX = 0 @@ -80,11 +90,146 @@ def assemble_spines( provenance, columns, or ID spaces violate the assembly contract. """ + prepared = _stack_spine_tables( + spines, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + harmonized = _harmonize_spine_values( + prepared.tables["household"], prepared.values, prepared.context + ) + # These are newly composed per-source records, not a carried Frame log. + # Table provenance remains owned by the independent stacking result. + assembled_mass_log = harmonized.mass_log + result = Frame( + prepared.tables, + US_SCHEMA, + {"household": harmonized.weights}, + prepared.strata, + mass_log=assembled_mass_log, + metadata=prepared.metadata, + ) + validate_assembly_provenance(result, boundary="spine assembly output") + anchor_mass = prepared.context["incoming_masses"][mass_anchor_channel] + if not np.isclose( + result.weights_for("household").total, anchor_mass, rtol=_SHARE_RTOL, atol=0.0 + ): + raise RuntimeError("Spine assembly failed to conserve anchor household mass.") + return result + + +@dataclass(frozen=True) +class SpinePreparation: + """Actual DESIGN-weight tables plus an immutable assembly context. + + The context schema determines which subsequent weight operation can use it. + This numerical container does not authenticate source issuance or membership. + """ + + frame: Frame + context: Mapping[str, object] + + def __post_init__(self) -> None: + if not isinstance(self.frame, Frame): + raise TypeError("SpinePreparation.frame must be a Frame.") + if self.frame.weights_for("household").kind is not WeightKind.DESIGN: + raise ValueError("Spine preparation requires actual DESIGN weights.") + if not isinstance(self.context, Mapping): + raise ValueError("Spine preparation context must be a mapping.") + object.__setattr__(self, "context", _freeze_context(self.context)) + + +@dataclass(frozen=True) +class SpineHarmonization: + """Computed importance weights and exact ordered legacy operator history.""" + + weights: Weights + mass_log: tuple[MassChangeRecord, ...] + + +@dataclass(frozen=True) +class _StackedTables: + tables: Mapping[str, pd.DataFrame] + strata: pd.Series + values: np.ndarray + context: Mapping[str, object] + metadata: Mapping[str, object] + mass_log: tuple[MassChangeRecord, ...] + + +def _freeze_context(value: object) -> object: + if isinstance(value, Mapping): + return MappingProxyType( + {key: _freeze_context(item) for key, item in value.items()} + ) + if isinstance(value, (list, tuple)): + return tuple(_freeze_context(item) for item in value) + return value + + +def _numeric_digest(values: np.ndarray, dtype: str) -> str: + array = np.ascontiguousarray(values, dtype=dtype) + return hashlib.sha256(memoryview(array).cast("B")).hexdigest() + + +def _channel_digest(values: np.ndarray) -> str: + digest = hashlib.sha256() + for value in values: + if not isinstance(value, str): + raise ValueError("Prepared household channels must be strings.") + encoded = value.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "little")) + digest.update(encoded) + return digest.hexdigest() + + +def _stack_spine_tables( + spines: Mapping[str, Frame], + *, + household_mass_shares: Mapping[str, float], + mass_anchor_channel: str, + require_design: bool = False, +) -> _StackedTables: ordered_channels = _validated_channels(spines, mass_anchor_channel) shares = _validated_shares(household_mass_shares, ordered_channels) frames = {channel: spines[channel] for channel in ordered_channels} + prepared = _stack_source_tables(frames, ordered_channels, require_design) + context = dict(prepared.context) + context.update( + schema="microcosm.us.spine-preparation.v1", + household_mass_shares=shares, + mass_anchor_channel=mass_anchor_channel, + incoming_masses={ + channel: float(frames[channel].weights_for("household").total) + for channel in ordered_channels + }, + ) + return _StackedTables( + prepared.tables, + prepared.strata, + prepared.values, + _freeze_context(context), + prepared.metadata, + prepared.mass_log, + ) + + +def _stack_source_tables( + spines: Mapping[str, Frame], + ordered_channels: tuple[str, ...], + require_design: bool, +) -> _StackedTables: + """Preserve source values while remapping only structural collisions.""" + frames = {channel: spines[channel] for channel in ordered_channels} for channel, frame in frames.items(): _validate_source_frame(frame, channel=channel) + if ( + require_design + and frame.weights_for("household").kind is not WeightKind.DESIGN + ): + raise ValueError( + f"Spine {channel!r} preparation requires actual DESIGN weights." + ) _validate_shared_column_dtypes(frames) offsets = _id_offsets(frames, ordered_channels) @@ -106,51 +251,286 @@ def assemble_spines( column_orders=column_orders, ) - anchor_mass = frames[mass_anchor_channel].weights_for("household").total - weights, mass_log = _assembled_household_weights( - frames, - ordered_channels=ordered_channels, - shares=shares, - anchor_mass=anchor_mass, + values = np.concatenate( + [ + frames[channel].weights_for("household").values + for channel in ordered_channels + ] ) - household_order = group_orders.get("household") - if household_order is not None: - weights = Weights(weights.values[household_order], weights.kind) - weights = _with_exact_total(weights, anchor_mass) - + order = group_orders.get("household") + if order is not None: + values = values[order] + household = tables["household"] + source_ids_digest = hashlib.sha256() + for channel in ordered_channels: + source_ids = np.ascontiguousarray( + prepared[channel]["household"]["household_id"].to_numpy(), dtype=" SpinePreparation: + """Stack actual DESIGN sources without performing importance allocation. + + Legacy ``assemble_spines`` continues to accept its existing mixed-kind + inputs. This new graph-facing seam never relabels such inputs as DESIGN. + """ + prepared = _stack_spine_tables( + spines, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + require_design=True, + ) + return _design_preparation(prepared) + + +def stack_survey_spines(spines: Mapping[str, Frame]) -> SpinePreparation: + """Stack original DESIGN survey weights before domain allocation. + + Channels are ordered lexically, independent of mapping insertion order. + Every household weight, source age and measured value is carried unchanged; + only structural IDs are remapped and source provenance is added. No survey + total is an anchor, no shares are assigned, and no mass is normalized. + + This is the numerical stacking seam. A graph source owner must separately + authenticate the original issuances, complete membership and publisher + anchors. Sampling and population-domain allocation are subsequent declared + operations. The returned v1 survey-stack context is deliberately refused + by the legacy anchor-total harmonizer. + """ + channels = _validated_source_channels(spines) + prepared = _stack_source_tables(spines, channels, require_design=True) + return _design_preparation(prepared) + + +def _design_preparation(prepared: _StackedTables) -> SpinePreparation: + frame = Frame( + prepared.tables, + US_SCHEMA, + {"household": Weights(prepared.values, WeightKind.DESIGN)}, + prepared.strata, + mass_log=prepared.mass_log, + metadata=prepared.metadata, ) - if not np.isclose( - result.weights_for("household").total, - anchor_mass, - rtol=_SHARE_RTOL, - atol=0.0, + validate_assembly_provenance(frame, boundary="spine preparation output") + return SpinePreparation(frame, prepared.context) + + +def harmonize_spine_weights( + *, + household: pd.DataFrame, + weights: Weights, + context: Mapping[str, object], +) -> SpineHarmonization: + """Compute importance allocation from declared DESIGN views and context. + + Input digests bind exact normalized values and ordered IDs/channels. + The explicit permutation restores source-order sums and correction rows. + The result includes legacy per-source logs, not an executor graph ledger. + """ + if not isinstance(weights, Weights) or weights.kind is not WeightKind.DESIGN: + raise ValueError("Spine harmonization requires actual DESIGN weights.") + if ( + not isinstance(context, Mapping) + or context.get("schema") != "microcosm.us.spine-preparation.v1" ): - raise RuntimeError("Spine assembly failed to conserve anchor household mass.") - return result + raise ValueError("Unsupported spine preparation context.") + kinds = context.get("source_weight_kinds") + ordered = context.get("ordered_channels") + if ( + not isinstance(kinds, Mapping) + or not isinstance(ordered, (tuple, list)) + or set(kinds) != set(ordered) + or any(kind != WeightKind.DESIGN.value for kind in kinds.values()) + ): + raise ValueError("Spine preparation context does not declare DESIGN sources.") + return _harmonize_spine_values(household, weights.values, context) + + +def _validated_spine_inputs( + household: pd.DataFrame, values: np.ndarray, context: Mapping[str, object] +) -> tuple: + if ( + not isinstance(context, Mapping) + or context.get("schema") != "microcosm.us.spine-preparation.v1" + ): + raise ValueError("Unsupported spine preparation context.") + if not isinstance(household, pd.DataFrame) or not { + "household_id", + support_channel_column("household"), + }.issubset(household): + raise ValueError("Harmonization requires household IDs and source channels.") + ids = household["household_id"].to_numpy() + channels = household[support_channel_column("household")].to_numpy() + if not np.issubdtype(ids.dtype, np.integer) or len(values) != len(ids): + raise ValueError("Prepared household IDs/weights are malformed.") + bindings = { + "household_ids_sha256": _numeric_digest(ids, " SpineHarmonization: + ordered, shares, counts, source_values, source_channels, order, anchor_mass = ( + _validated_spine_inputs(household, values, context) + ) + result, logs, allocated, start = [], [], 0.0, 0 + for index, channel in enumerate(ordered): + count = counts[channel] + existing_values = source_values[start : start + count] + if not np.all(source_channels[start : start + count] == channel): + raise ValueError( + "Prepared permutation does not restore source channel order." + ) + start += count + existing_mass = float(existing_values.sum()) + if existing_mass != float(context["incoming_masses"][channel]): + raise ValueError("Prepared source-order mass changed.") + target = ( + anchor_mass - allocated + if index == len(ordered) - 1 + else anchor_mass * shares[channel] + ) + scaled = _values_to_total(existing_values, target) + result.append(scaled) + allocated += float(scaled.sum()) + logs.extend( + MassChangeRecord(**dict(record)) + for record in context["source_mass_logs"][channel] + ) + logs.append( + MassChangeRecord( + entity="household", + old_total=existing_mass, + new_total=float(scaled.sum()), + declared_factor=target / existing_mass, + reason=f"allocated {channel!r} source mass in pre-operator spine assembly", + ) + ) + combined = np.concatenate(result) + if order is not None: + combined = combined[order] + return SpineHarmonization( + _with_exact_total(Weights(combined, WeightKind.IMPORTANCE), anchor_mass), + tuple(logs), + ) def _validated_channels( spines: Mapping[str, Frame], mass_anchor_channel: str, ) -> tuple[str, ...]: + channels = _validated_source_channels(spines) + if ( + not isinstance(mass_anchor_channel, str) + or _CHANNEL_PATTERN.fullmatch(mass_anchor_channel) is None + ): + raise ValueError( + "mass_anchor_channel must be a stable lower-snake-case identifier." + ) + if mass_anchor_channel not in channels: + raise ValueError( + f"mass_anchor_channel {mass_anchor_channel!r} is absent from spines." + ) + return ( + mass_anchor_channel, + *(channel for channel in channels if channel != mass_anchor_channel), + ) + + +def _validated_source_channels(spines: Mapping[str, Frame]) -> tuple[str, ...]: if not isinstance(spines, Mapping): raise TypeError(f"spines must be a mapping, got {type(spines).__name__}.") if len(spines) < 2: @@ -171,21 +551,7 @@ def _validated_channels( f"{PUF_TAX_DETAIL_SUPPORT_CHANNEL!r} is a clone operator channel, " "not a peer household spine." ) - if ( - not isinstance(mass_anchor_channel, str) - or _CHANNEL_PATTERN.fullmatch(mass_anchor_channel) is None - ): - raise ValueError( - "mass_anchor_channel must be a stable lower-snake-case identifier." - ) - if mass_anchor_channel not in spines: - raise ValueError( - f"mass_anchor_channel {mass_anchor_channel!r} is absent from spines." - ) - return ( - mass_anchor_channel, - *sorted(channel for channel in channels if channel != mass_anchor_channel), - ) + return tuple(sorted(channels)) def _validated_shares( @@ -394,6 +760,11 @@ def _id_offsets( ) offsets[channel][entity] = offset remapped = ids if offset == 0 else ids + offset + if int(remapped.max()) > PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID: + raise ValueError( + f"Spine {channel!r} {entity!r} collision remapping exceeds " + f"the clone-safe bound {PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID}." + ) accumulated[entity] = np.concatenate([used, remapped]) return offsets @@ -493,45 +864,6 @@ def _combined_tables( return tables, group_orders -def _assembled_household_weights( - frames: Mapping[str, Frame], - *, - ordered_channels: tuple[str, ...], - shares: Mapping[str, float], - anchor_mass: float, -) -> tuple[Weights, tuple[MassChangeRecord, ...]]: - values: list[np.ndarray] = [] - mass_log: list[MassChangeRecord] = [] - allocated = 0.0 - for index, channel in enumerate(ordered_channels): - existing = frames[channel].weights_for("household") - target = ( - anchor_mass - allocated - if index == len(ordered_channels) - 1 - else anchor_mass * shares[channel] - ) - scaled = _values_to_total(existing.values, target) - values.append(scaled) - allocated += float(scaled.sum()) - factor = target / existing.total - mass_log.extend(frames[channel].mass_log) - mass_log.append( - MassChangeRecord( - entity="household", - old_total=existing.total, - new_total=float(scaled.sum()), - declared_factor=factor, - reason=( - f"allocated {channel!r} source mass in pre-operator spine assembly" - ), - ) - ) - return ( - Weights(np.concatenate(values), WeightKind.IMPORTANCE), - tuple(mass_log), - ) - - def _values_to_total(values: np.ndarray, target: float) -> np.ndarray: result = np.asarray(values, dtype=np.float64) * ( target / float(np.asarray(values, dtype=np.float64).sum()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index d71320311..1e62b8c2d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -26,10 +26,11 @@ ``w_i' = w_i * share_s * M_anchor / M_s`` where ``M_s`` is arm ``s``'s incoming household mass and ``M_anchor`` is the -mass-anchor arm's incoming mass. For the seeded ACS sample, -``M_acs_sample ~= fraction * M_acs_full``, so the allocation factor contains -the inverse-sampling upweighting ``1 / fraction`` automatically; the realized -per-arm scale factors are receipted rather than assumed. +mass-anchor arm's incoming mass. Production sampling first normalizes each +source by its full-source mass divided by its realized sampled mass. For +unequal source weights, that ratio is not generally the inverse realized +household inclusion count ``N / n``. Legacy pilot controls allocate directly +from the sampled ACS mass. Both paths receipt their actual scale factors. """ from __future__ import annotations @@ -40,7 +41,6 @@ import math import os import pickle -import struct import sys from collections import Counter from collections.abc import Callable, Mapping, Sequence @@ -80,6 +80,12 @@ ) from microcosm.build.serialization_dtypes import canonicalize_table_string_dtypes from microcosm.build.source_manifest import load_source_manifest +from microcosm.build.table_identity import ( + TABLE_VALUES_DIGEST_CODEC as _LATE_TABLE_DIGEST_CODEC, +) +from microcosm.build.table_identity import ( + table_values_sha256 as _late_table_values_sha256, +) from microcosm.build.us_runtime import ( post_transfer_calibration as post_transfer_calibration_runtime, ) @@ -183,7 +189,13 @@ puf_tax_detail_tail_bound_quantiles_identity, validate_puf_clone_attachment, ) -from microcosm.build.us_runtime.spine_assembly import assemble_spines +from microcosm.build.us_runtime.spine_assembly import ( + _freeze_context, + _validated_spine_inputs, + assemble_spines, + harmonize_spine_weights, + prepare_spines, +) from microcosm.build.us_runtime.support_provenance import ( BASE_ASEC_SUPPORT_CHANNEL, PUF_TAX_DETAIL_CLONE_INDEX, @@ -220,7 +232,7 @@ us_late_producer_schedule_receipt, ) from microcosm.fit import Regime -from microcosm.frame import US_SCHEMA, Frame, WeightKind +from microcosm.frame import US_SCHEMA, Frame, MassChangeRecord, WeightKind, Weights __all__ = [ "ACS_STACKED_SUPPORT_CHANNEL", @@ -249,6 +261,11 @@ "StackedLateProducerResult", "StackedPostPufTransferResult", "StackedSpineResult", + "StackedSpinePreparation", + "StackedSpineHarmonization", + "prepare_stacked_spine", + "harmonize_stacked_spine_weights", + "finish_stacked_spine", "assemble_stacked_spine", "assert_stacked_tail_cells_preserved", "by_origin_battery", @@ -553,7 +570,47 @@ def _acs_native_group_quarters_receipt( } -def assemble_stacked_spine( +@dataclass(frozen=True) +class _SampledStack: + spines: Mapping[str, Frame] + shares: Mapping[str, float] + incoming_masses: Mapping[str, float] + sampling: Mapping[str, object] + manifest_version: int + mass_anchor_channel: str + + +@dataclass(frozen=True) +class StackedSpinePreparation: + """DESIGN preparation and a distinct pre-harmonization receipt.""" + + frame: Frame + receipt: Mapping[str, object] + + def __post_init__(self) -> None: + if not isinstance(self.frame, Frame) or not isinstance(self.receipt, Mapping): + raise TypeError( + "StackedSpinePreparation requires a Frame and receipt mapping." + ) + object.__setattr__(self, "receipt", _freeze_context(self.receipt)) + _validate_stacked_preparation(self.frame, self.receipt) + + +@dataclass(frozen=True) +class StackedSpineHarmonization: + """Actual weights and completed context, with ordered legacy history.""" + + weights: Weights + metadata: Mapping[str, object] + receipt: Mapping[str, object] + legacy_mass_log: tuple[MassChangeRecord, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "metadata", _freeze_context(self.metadata)) + object.__setattr__(self, "receipt", _freeze_context(self.receipt)) + + +def _sample_stacked_inputs( asec: Frame, acs: Frame, *, @@ -563,27 +620,8 @@ def assemble_stacked_spine( sample_seed: int | None = None, household_mass_shares: Mapping[str, float] | None = None, mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, -) -> StackedSpineResult: - """Assemble uniformly sampled ASEC and ACS survey arms into one spine. - - Production callers provide the single ``sample_fraction`` and - ``sample_seed`` scale-ladder controls. The exact same fraction is applied - independently to both survey arms at whole-household grain; each sampled - arm is then normalized back to its full-source household mass before the - reviewed :func:`assemble_spines` harmonization. This preserves each arm's - composition and prevents the anchor population from shrinking with the - rung. PUF donors are not accepted here and therefore remain unsampled. - - ``acs_sample_fraction``/``acs_sample_seed`` retain the reviewed version-1 - pilot contract for reproducibility only: ASEC remains full and ACS alone - is sampled. Supplying pilot and production controls together fails - closed. - - Returns: - A validated :class:`StackedSpineResult` whose receipt mirrors the - frozen manifest as a JSON-ready mapping. - """ - +) -> _SampledStack: + """Run the unchanged pilot or production sampling/normalization controls.""" shares = ( dict(DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES) if household_mass_shares is None @@ -688,6 +726,287 @@ def assemble_stacked_spine( }, } + return _SampledStack( + { + BASE_ASEC_SUPPORT_CHANNEL: sampled_asec, + ACS_STACKED_SUPPORT_CHANNEL: sampled_acs, + }, + shares, + incoming_masses, + sampling_manifest, + manifest_version, + mass_anchor_channel, + ) + + +def assemble_stacked_spine( + asec: Frame, + acs: Frame, + *, + acs_sample_fraction: float | None = None, + acs_sample_seed: int | None = None, + sample_fraction: float | None = None, + sample_seed: int | None = None, + household_mass_shares: Mapping[str, float] | None = None, + mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, +) -> StackedSpineResult: + """Compose actual sampling, stacking and importance harmonization. + + Production controls normalize each sampled source by M_full/M_sample. + This ratio is not generally inverse realized household inclusion N/n. + Legacy pilot controls and accepted source weight kinds remain unchanged. + """ + sampled = _sample_stacked_inputs( + asec, + acs, + acs_sample_fraction=acs_sample_fraction, + acs_sample_seed=acs_sample_seed, + sample_fraction=sample_fraction, + sample_seed=sample_seed, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + if all( + frame.weights_for("household").kind is WeightKind.DESIGN + for frame in sampled.spines.values() + ): + return finish_stacked_spine(_prepare_sampled_stack(sampled)) + # Existing direct callers may supply calibrated or mixed source kinds. + # They keep their historical path, never relabeled as a DESIGN anchor. + return _finish_legacy_sampled_stack(sampled) + + +def prepare_stacked_spine( + asec: Frame, + acs: Frame, + *, + acs_sample_fraction: float | None = None, + acs_sample_seed: int | None = None, + sample_fraction: float | None = None, + sample_seed: int | None = None, + household_mass_shares: Mapping[str, float] | None = None, + mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, +) -> StackedSpinePreparation: + """Sample/normalize actual DESIGN sources and stack them before allocation. + + Returns a dedicated preparation receipt, never an incomplete completed + stacked-spine manifest. No harmonized weights are precomputed. + """ + sampled = _sample_stacked_inputs( + asec, + acs, + acs_sample_fraction=acs_sample_fraction, + acs_sample_seed=acs_sample_seed, + sample_fraction=sample_fraction, + sample_seed=sample_seed, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + return _prepare_sampled_stack(sampled) + + +def _prepare_sampled_stack(sampled: _SampledStack) -> StackedSpinePreparation: + prepared = prepare_spines( + sampled.spines, + household_mass_shares=sampled.shares, + mass_anchor_channel=sampled.mass_anchor_channel, + ) + samples = sampled.sampling.get( + "survey_samples", + {ACS_STACKED_SUPPORT_CHANNEL: sampled.sampling.get("acs_sample")}, + ) + full_masses = { + channel: float(samples[channel]["incoming_household_mass"]) + if channel in samples + else float(sampled.incoming_masses[channel]) + for channel in sampled.spines + } + receipt = { + "schema": "microcosm.us.stacked-spine-preparation.v1", + "stacked_manifest_version": sampled.manifest_version, + "sampling": sampled.sampling, + "full_source_masses": full_masses, + "household_mass_shares": sampled.shares, + "mass_anchor_channel": sampled.mass_anchor_channel, + "assembly_context": prepared.context, + "assembly_metadata": prepared.frame.metadata, + "acs_native_group_quarters": _acs_native_group_quarters_receipt( + sampled.spines[ACS_STACKED_SUPPORT_CHANNEL], prepared.frame + ), + } + return StackedSpinePreparation(prepared.frame, receipt) + + +def _validate_stacked_preparation(frame: Frame, receipt: Mapping[str, object]) -> None: + boundary = "stacked spine preparation" + if frame.weights_for("household").kind is not WeightKind.DESIGN: + raise ValueError("Stacked spine preparation requires actual DESIGN weights.") + if STACKED_SPINE_MANIFEST_KEY in frame.metadata: + raise ValueError( + "A preparation must not carry a completed stacked spine manifest." + ) + validate_assembly_provenance(frame, boundary=boundary) + _validate_preparation_receipt(receipt) + if _json_ready(receipt["assembly_metadata"]) != _json_ready(frame.metadata): + raise ValueError("Prepared assembly metadata differs from the actual frame.") + _validated_spine_inputs( + frame.table("household"), + frame.weights_for("household").values, + receipt["assembly_context"], + ) + version, sampling = receipt["stacked_manifest_version"], receipt["sampling"] + production = version == _STACKED_SPINE_MANIFEST_VERSION + fraction = sampling["sample_fraction" if production else "acs_sample_fraction"] + seed = sampling["sample_seed" if production else "acs_sample_seed"] + _validate_fraction(fraction) + _validate_seed(seed) + samples = ( + sampling["survey_samples"] + if production + else {ACS_STACKED_SUPPORT_CHANNEL: sampling["acs_sample"]} + ) + expected = ( + {BASE_ASEC_SUPPORT_CHANNEL, ACS_STACKED_SUPPORT_CHANNEL} + if production + else {ACS_STACKED_SUPPORT_CHANNEL} + ) + if set(samples) != expected: + raise ValueError( + "Prepared sampling receipts do not cover their declared sources." + ) + for channel, sample in samples.items(): + _validate_survey_sample_receipt( + frame, + channel=channel, + fraction=fraction, + seed=seed, + sample=sample, + boundary=boundary, + require_normalization=production, + ) + if float(receipt["full_source_masses"][channel]) != float( + sample["incoming_household_mass"] + ): + raise ValueError( + "Prepared full-source mass differs from its sampling receipt." + ) + _validate_stacked_clone_role_lifecycle(frame, boundary=boundary) + _validated_acs_native_group_quarters_masks(frame, receipt, boundary=boundary) + + +def _validate_preparation_receipt(preparation: Mapping[str, object]) -> None: + if ( + not isinstance(preparation, Mapping) + or preparation.get("schema") != "microcosm.us.stacked-spine-preparation.v1" + ): + raise ValueError("Unsupported stacked-spine preparation receipt.") + if ( + preparation.get("stacked_manifest_version") + not in _SUPPORTED_STACKED_SPINE_MANIFEST_VERSIONS + ): + raise ValueError("Unsupported prepared stacked-spine manifest version.") + for field_name in ( + "sampling", + "full_source_masses", + "household_mass_shares", + "assembly_context", + "assembly_metadata", + "acs_native_group_quarters", + ): + if not isinstance(preparation.get(field_name), Mapping): + raise ValueError( + f"Prepared stacked-spine field {field_name!r} must be a mapping." + ) + context = preparation["assembly_context"] + if preparation.get("mass_anchor_channel") != context.get( + "mass_anchor_channel" + ) or preparation["household_mass_shares"] != context.get("household_mass_shares"): + raise ValueError("Prepared stack controls disagree with the assembly context.") + if set(preparation["full_source_masses"]) != { + BASE_ASEC_SUPPORT_CHANNEL, + ACS_STACKED_SUPPORT_CHANNEL, + }: + raise ValueError("Prepared full-source masses must name ASEC and ACS.") + + +def harmonize_stacked_spine_weights( + *, + household: pd.DataFrame, + weights: Weights, + preparation: Mapping[str, object], +) -> StackedSpineHarmonization: + """Compute importance weights and completed metadata from declared views. + + The preparation producer validates full source/person lineage. This helper + binds its household views and performs allocation without restacking or + reading a precomputed final-weight vector. The returned legacy mass log is + distinct from a graph executor's structural transition history. + """ + _validate_preparation_receipt(preparation) + result = harmonize_spine_weights( + household=household, weights=weights, context=preparation["assembly_context"] + ) + shares = preparation["household_mass_shares"] + incoming = preparation["assembly_context"]["incoming_masses"] + anchor = preparation["mass_anchor_channel"] + harmonization = _harmonization_receipt_from_views( + household, + result.weights, + shares=shares, + anchor_mass=incoming[anchor], + incoming_masses=incoming, + ) + manifest = { + "version": preparation["stacked_manifest_version"], + **preparation["sampling"], + "household_mass_shares": { + channel: float(share) for channel, share in shares.items() + }, + "mass_anchor_channel": anchor, + "weight_harmonization": harmonization, + "acs_native_group_quarters": preparation["acs_native_group_quarters"], + } + metadata = { + **preparation["assembly_metadata"], + STACKED_SPINE_MANIFEST_KEY: manifest, + } + return StackedSpineHarmonization( + result.weights, metadata, _json_ready(manifest), result.mass_log + ) + + +def finish_stacked_spine(prepared: StackedSpinePreparation) -> StackedSpineResult: + """Complete a validated preparation, preserving exact legacy Frame history.""" + if not isinstance(prepared, StackedSpinePreparation): + raise TypeError("finish_stacked_spine requires a StackedSpinePreparation.") + _validate_stacked_preparation(prepared.frame, prepared.receipt) + result = harmonize_stacked_spine_weights( + household=prepared.frame.table("household"), + weights=prepared.frame.weights_for("household"), + preparation=prepared.receipt, + ) + frame = Frame( + {entity: prepared.frame.table(entity) for entity in prepared.frame.entities}, + prepared.frame.schema, + {"household": result.weights}, + prepared.frame.strata, + mass_log=result.legacy_mass_log, + metadata=result.metadata, + ) + validated = validate_stacked_spine_frame( + frame, boundary="stacked spine assembly output" + ) + return StackedSpineResult(frame, _json_ready(validated)) + + +def _finish_legacy_sampled_stack(sampled: _SampledStack) -> StackedSpineResult: + sampled_asec, sampled_acs = ( + sampled.spines[BASE_ASEC_SUPPORT_CHANNEL], + sampled.spines[ACS_STACKED_SUPPORT_CHANNEL], + ) + shares, incoming_masses = sampled.shares, sampled.incoming_masses + sampling_manifest, manifest_version = sampled.sampling, sampled.manifest_version + mass_anchor_channel = sampled.mass_anchor_channel assembled = assemble_spines( { BASE_ASEC_SUPPORT_CHANNEL: sampled_asec, @@ -1632,9 +1951,25 @@ def _harmonization_receipt( anchor_mass: float, incoming_masses: Mapping[str, float], ) -> dict[str, dict[str, float]]: - household = assembled.table("household") + return _harmonization_receipt_from_views( + assembled.table("household"), + assembled.weights_for("household"), + shares=shares, + anchor_mass=anchor_mass, + incoming_masses=incoming_masses, + ) + + +def _harmonization_receipt_from_views( + household: pd.DataFrame, + household_weights: Weights, + *, + shares: Mapping[str, float], + anchor_mass: float, + incoming_masses: Mapping[str, float], +) -> dict[str, dict[str, float]]: channel_values = household[support_channel_column("household")].astype(str) - weights = np.asarray(assembled.weights_for("household").values, dtype=np.float64) + weights = np.asarray(household_weights.values, dtype=np.float64) receipt: dict[str, dict[str, float]] = {} for channel, share in shares.items(): incoming = float(incoming_masses[channel]) @@ -5104,7 +5439,6 @@ def validate_stacked_post_puf_transfer_receipt( ) -_LATE_TABLE_DIGEST_CODEC = "canonical_scalar_v1" _LATE_PRIMARY_QRF_INPUT_BINDING_ARTIFACT_KIND = ( "populace_us_stacked_late_primary_qrf_input_binding" ) @@ -5112,259 +5446,6 @@ def validate_stacked_post_puf_transfer_receipt( _LATE_RESOURCE_SEMANTICS_ARTIFACT_KIND = ( "populace_us_stacked_late_producer_resource_semantics" ) -_LATE_TABLE_DIGEST_CHUNK_ROWS = 65_536 - - -def _late_digest_part( - digest, - *, - domain: str, - payload: bytes | bytearray | memoryview | np.ndarray, -) -> None: - """Append one length-framed, domain-separated byte field to a digest.""" - - domain_bytes = domain.encode("utf-8") - payload_view = memoryview(payload) - if payload_view.format != "B" or payload_view.ndim != 1: - payload_view = payload_view.cast("B") - digest.update(struct.pack(" np.ndarray: - """Return a contiguous, explicitly little-endian numeric byte source.""" - - array = np.asarray(values) - array = array.astype(array.dtype.newbyteorder("<"), copy=False) - return np.ascontiguousarray(array) - - -def _late_scalar_bytes(value: object) -> bytes: - """Encode one supported object scalar without lossy intermediary hashes.""" - - missing = pd.isna(value) - if isinstance(missing, (bool, np.bool_)) and bool(missing): - return b"null" - if isinstance(value, (bool, np.bool_)): - return b"bool\x01" if bool(value) else b"bool\x00" - if isinstance(value, (int, np.integer)): - return b"integer\x00" + str(int(value)).encode("ascii") - if isinstance(value, (float, np.floating)): - if isinstance(value, np.floating) and value.dtype.itemsize > 8: - raise TypeError( - "US late-producer content digest does not support object " - f"floating scalar {value.dtype!s}." - ) - return b"float64\x00" + struct.pack(" 16: - raise TypeError( - "US late-producer content digest does not support object " - f"complex scalar {value.dtype!s}." - ) - numeric = complex(value) - return b"complex128\x00" + struct.pack(" None: - """Stream framed string or object scalars in bounded-memory chunks.""" - - for chunk_index, start in enumerate( - range(0, len(values), _LATE_TABLE_DIGEST_CHUNK_ROWS) - ): - stop = min(start + _LATE_TABLE_DIGEST_CHUNK_ROWS, len(values)) - lengths = np.zeros(stop - start, dtype=" None: - """Hash one ordered logical Series with explicit dtype and null domains.""" - - dtype = series.dtype - missing = series.isna().to_numpy(dtype=bool) - _late_digest_part( - digest, - domain=f"{domain}/dtype", - payload=str(dtype).encode("utf-8"), - ) - _late_digest_part( - digest, - domain=f"{domain}/row_count", - payload=struct.pack(" str: - """Hash ordered table scalars directly with typed, null-aware framing.""" - - values = ( - canonicalize_table_string_dtypes( - table, - boundary="late-producer content digest", - table_name="declared_surface", - ) - if normalize_strings - else table - ) - if isinstance(values.index, pd.MultiIndex): - index_levels = [ - pd.Series(values.index.get_level_values(level), copy=False) - for level in range(values.index.nlevels) - ] - else: - index_levels = [pd.Series(values.index, copy=False)] - header = { - "codec": _LATE_TABLE_DIGEST_CODEC, - "columns": [str(column) for column in values.columns], - "dtypes": [ - str(values.iloc[:, index].dtype) for index in range(values.shape[1]) - ], - "index_type": type(values.index).__name__, - "index_dtype": str(values.index.dtype), - "index_level_dtypes": [str(level.dtype) for level in index_levels], - "index_names": [ - None if name is None else str(name) for name in values.index.names - ], - } - digest = hashlib.sha256() - _late_digest_part( - digest, - domain="late_table_digest_codec", - payload=_LATE_TABLE_DIGEST_CODEC.encode("ascii"), - ) - _late_digest_part( - digest, - domain="late_table_header_json", - payload=json.dumps( - header, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8"), - ) - for level_index, level in enumerate(index_levels): - _late_digest_series_values( - digest, - level, - domain=f"index_level/{level_index}", - ) - for column_index in range(values.shape[1]): - _late_digest_series_values( - digest, - values.iloc[:, column_index], - domain=f"column/{column_index}", - ) - return digest.hexdigest() def _late_virtual_resource_kind(column: str) -> str: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py new file mode 100644 index 000000000..deffa57ee --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py @@ -0,0 +1,220 @@ +"""Explicit S0101-only activation for the selected ACS/ASEC development path. + +A reviewed declaration pins two source responses. The caller's review and +guarded acquisition supply trust; this decoder is not independent attestation. +There is deliberately no default genuine declaration or mixed-inventory reader. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass, replace +from pathlib import Path + +from microcosm.calibrate.registry import TargetRegistry, TargetSpec +from microcosm.graph.canonical import canonical_json + +from . import demographic_calibration_graph as demographic +from . import national_age_activation as age +from .survey_age_sources import _digest, read_survey_age_response, survey_age_requests + +PROFILE = "selected_survey_s0101_development_v1" +_CONVENTION = ( + "ACS 2024 observed interview ages and ASEC 2025 interview ages with income " + "year 2024; completed years used verbatim without temporal aging; calibration " + "to the published 2024 all-resident distribution does not establish that " + "the selected survey population covers that complete universe" +) + + +def _require(condition: object, reason: str) -> None: + if not condition: + raise ValueError("SURVEY_AGE_ACTIVATION_" + reason) + + +@dataclass(frozen=True, slots=True) +class SurveyAgeActivation: + """Exact source pins whose approval is a separate caller responsibility.""" + + metadata_sha256: str + data_sha256: str + + def __post_init__(self) -> None: + _digest(self.metadata_sha256) + _digest(self.data_sha256) + + +def activation_binding(declaration: SurveyAgeActivation) -> dict: + """Return the complete, detached semantic declaration, without source I/O.""" + _require(type(declaration) is SurveyAgeActivation, "DECLARATION") + declaration.__post_init__() + return { + "profile": PROFILE, + "requests": list(survey_age_requests()), + "metadata_sha256": declaration.metadata_sha256, + "data_sha256": declaration.data_sha256, + "period": "2024", + "entity": "household", + "universe": age.NATIONAL_AGE_ACTIVATION.universe, + "age_convention_id": "observed_interview_age_completed_years", + "age_convention": _CONVENTION, + "bands": [ + { + "variable": b.variable, + "label": b.label, + "low": b.low, + "high": b.high, + "column": b.column, + } + for b in age.NATIONAL_AGE_ACTIVATION.bands + ], + "published_total": "S0101_C01_001_consistency_only", + "consumes_se": False, + "release_eligible": False, + } + + +def declaration_from_binding(binding: object) -> SurveyAgeActivation: + """Validate value-level profile semantics; grant no source authorization.""" + _require(type(binding) is dict, "DECLARATION") + _require("metadata_sha256" in binding and "data_sha256" in binding, "DECLARATION") + declaration = SurveyAgeActivation( + binding["metadata_sha256"], binding["data_sha256"] + ) + _require( + canonical_json(binding) == canonical_json(activation_binding(declaration)), + "DECLARATION", + ) + return declaration + + +def activation_digest(declaration: SurveyAgeActivation) -> str: + """Bind source pins and all selected-survey conventions in one identifier.""" + return hashlib.sha256(canonical_json(activation_binding(declaration))).hexdigest() + + +def validate_survey_age_registry( + registry: TargetRegistry, binding: dict +) -> TargetRegistry: + """Check numerical profile metadata, not the truth of caller-supplied values.""" + declaration = declaration_from_binding(binding) + frozen = demographic._registry_from_json(demographic._registry_json(registry)) + bands = age.NATIONAL_AGE_ACTIVATION.bands + _require(len(frozen) == len(bands), "REGISTRY") + digest = activation_digest(declaration) + for spec, band in zip(frozen, bands, strict=True): + _require( + spec.name == band.variable + and spec.measure == band.column + and spec.metadata["table"] == "S0101" + and spec.metadata["reference_sha256"] == digest + and spec.metadata["evidence_scope"] == "source_documented", + "REGISTRY", + ) + return frozen + + +def activate_survey_age_targets( + source_dir: str | Path, *, declaration: SurveyAgeActivation +) -> TargetRegistry: + """Derive exactly 18 targets from the two explicitly pinned local responses.""" + binding = activation_binding(declaration) + _, metadata = read_survey_age_response( + source_dir, "metadata", declaration.metadata_sha256 + ) + _, data = read_survey_age_response(source_dir, "data", declaration.data_sha256) + # Reuse the established publisher/count/MOE interpretation without invoking + # its old inventory loader. This object supplies only pure field semantics. + semantics = replace(age.NATIONAL_AGE_ACTIVATION, age_convention=_CONVENTION) + row = age._national_row(data, semantics) + _require( + all( + name in {"GEO_ID", "NAME", "us"} or name.startswith("S0101_") + for name in row + ), + "RESPONSE_SCOPE", + ) + _require( + type(metadata) is dict and type(metadata.get("variables")) is dict, "METADATA" + ) + _require( + all( + type(name) is str + and (name in {"GEO_ID", "NAME", "us"} or name.startswith("S0101_")) + for name in metadata["variables"] + ), + "METADATA_SCOPE", + ) + + def cell(variable): + _require( + all(variable + suffix in row for suffix in ("E", "EA", "M", "MA")), "CELL" + ) + return { + "estimate": age.classify_acs_value( + row[variable + "E"], row[variable + "EA"] + ), + "moe": age.classify_acs_value(row[variable + "M"], row[variable + "MA"]), + } + + digest = activation_digest(declaration) + specs = [] + data_url = survey_age_requests()[1]["url"] + for band in semantics.bands: + published = age._published_variable(metadata, band.variable + "E", semantics) + _require( + published.get("label") == band.label + and published.get("predicateType") == "int", + "PUBLISHED_DEFINITION", + ) + derived = cell(band.variable) + specs.append( + TargetSpec( + name=band.variable, + entity="household", + measure=band.column, + value=float(age._count_estimate(derived, band.variable)), + period="2024", + se=age._standard_error(derived, band.variable, confidence_level=0.9), + source=f"{data_url} (S0101 {band.variable}E)", + family="acs.S0101", + hierarchy=age.demographic_target_hierarchy( + "S0101", band, geography="0100000US" + ), + notes=f"activation={digest}; label={band.label}; age_convention={_CONVENTION}; published 90% MOE divided by {age._MOE_90_TO_SE}, not consumed by current loss", + metadata={ + "table": "S0101", + "reference_sha256": digest, + "geography": "0100000US", + "universe": "population", + "role": "calibration", + "evidence_scope": "source_documented", + }, + ) + ) + total = age._published_variable(metadata, "S0101_C01_001E", semantics) + _require( + total.get("label") == "Estimate!!Total!!Total population" + and total.get("predicateType") == "int", + "PUBLISHED_TOTAL_DEFINITION", + ) + _require( + sum(int(s.value) for s in specs) + == age._count_estimate(cell("S0101_C01_001"), "S0101_C01_001"), + "PARTITION", + ) + return validate_survey_age_registry(TargetRegistry(specs, country="us"), binding) + + +def verify_survey_age_targets( + source_dir: str | Path, + *, + declaration: SurveyAgeActivation, + registry: TargetRegistry, +) -> None: + """Rederive the pinned target values; no stored registry substitutes for this.""" + actual = activate_survey_age_targets(source_dir, declaration=declaration) + _require( + demographic._registry_json(actual) == demographic._registry_json(registry), + "REGISTRY", + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py new file mode 100644 index 000000000..c2051861b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py @@ -0,0 +1,952 @@ +"""Country admission for survey age profiles with an explicit source prefix. + +Every call reconstructs the accepted source preparation. The numerical kernels +are values-only; source, full-population, group/row-cap and cache checks live +here. The original entry point remains invented-only; the named development +entry point derives its targets from an explicitly pinned S0101-only capture. +Both permit only the first IMPORTANCE-to-CALIBRATED transition and no release. +An optional atomic-geography recipe enriches the accepted allocation before +cloning; the sampling budget continues to retain the original raw allocation. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from dataclasses import dataclass, replace +from types import SimpleNamespace + +import numpy as np + +from microcosm.frame import Frame +from microcosm.graph import ( + CompiledGraph, + KernelRegistry, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.artifact_edges import typed_contracts +from microcosm.graph.canonical import canonical_json +from microcosm.graph.executor import ( + _all_node_keys, + _input_writers, + _project_context, + _source_paths_and_keys, + _tolerance_writer_payload, +) +from microcosm.graph.keys import ( + _capabilities_projection, + artifact_key, + frame_key, + opaque_artifact_key, + seed, + weights_key, +) +from microcosm.graph.manifest import _freeze_json +from microcosm.graph.population import ( + Population, + mass_record_receipt, + weight_cap_receipt, +) +from microcosm.graph.store import _verified_meta + +from . import graph_combined_clone as clone +from . import graph_survey_age_artifact as ages +from . import graph_survey_budget as transport +from . import graph_survey_calibration as numerical +from . import graph_survey_population as source_graph +from . import survey_age_activation as age_activation +from . import survey_calibration_diagnostics as diagnostic_check +from . import survey_origin_budget as budgets +from . import survey_population_replay as replay + +COUNT_NODE = "survey.age_count_matrix" + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_AGE_RUN_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _node_identity(node): + """Immutable snapshot excluding execution time and cache-hit bookkeeping.""" + return canonical_json( + { + "key": node.key, + "ref": node.kernel_ref, + "implementation": node.kernel_impl_hash, + "capabilities": _capabilities_projection(node.capabilities), + "typed_artifacts": node.typed_artifacts, + "seed": node.seed, + "frame_key": node.frame_key, + "weight_key": node.weight_key, + "artifacts": [ + [entity, column, key] + for (entity, column), key in sorted(node.artifacts.items()) + ], + "opaque_artifacts": node.opaque_artifacts, + "receipt": node.receipt, + "legacy_capabilities": node.legacy_capabilities, + } + ) + + +def _measurement(population): + node = ages.survey_age_count_artifact_node( + population=population.version, node_id=COUNT_NODE + ) + context = _project_context( + node, + population, + key="0" * 64, + sources={}, + tolerances={}, + numerics={}, + ) + measured = ages.SurveyAgeCountArtifactKernel().run(context) + payload = measured.artifacts["counts"] + values = ages.decode_survey_age_counts(payload) + _require( + tuple(values.household_ids) + == tuple(population.frame.table("household").household_id), + "MEASUREMENT_HOUSEHOLD_ORDER", + ) + return payload, json.loads(canonical_json(measured.receipt)) + + +def _measure(population): + return _measurement(population)[0] + + +def _load_diagnostics(store, key): + """Bound the diagnostic payload buffer before reading it. + + The existing store verifier still authenticates metadata and streams its + payload hashes. This does not claim a bound on that verifier's metadata + decoding or total I/O. Only the selected diagnostic payload is buffered here. + """ + path = store.object_path(key) + payload_path = path / "payload.bin" + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + fd = os.open(payload_path, flags) + try: + before = os.fstat(fd) + limit = diagnostic_check.MAX_DIAGNOSTIC_BYTES + _require( + stat.S_ISREG(before.st_mode) and 0 < before.st_size <= limit, + "DIAGNOSTIC_STORE_LIMIT", + ) + metadata = _verified_meta(path, expected_kind="bytes") + _require( + set(metadata["payloads"]) == {"payload.bin"}, "DIAGNOSTIC_STORE_ROSTER" + ) + record = metadata["payloads"]["payload.bin"] + _require(record["size"] == before.st_size, "DIAGNOSTIC_STORE_SIZE") + with os.fdopen(fd, "rb", closefd=False) as stream: + payload = stream.read(limit + 1) + after = os.fstat(fd) + current = os.stat(payload_path, follow_symlinks=False) + + def seal(s): + return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns, s.st_ctime_ns) + + _require( + seal(before) == seal(after) == seal(current) + and stat.S_ISREG(current.st_mode) + and len(payload) == record["size"] <= limit + and _sha(payload) == record["sha256"], + "DIAGNOSTIC_STORE_CHANGED", + ) + return payload + finally: + os.close(fd) + + +def _expected_nodes( + compiled: CompiledGraph, + kernels: KernelRegistry, + keys: dict[str, str], + implementations: dict[str, str], + prefix_identities: dict[str, bytes], + frame: Frame, +): + """Detach declaration expectations before any graph kernel/store execution.""" + states = {} + cells = tuple((e, str(c)) for e in frame.entities for c in frame.table(e)) + for node in compiled.graph.nodes: + if node.id in prefix_identities: + state = json.loads(prefix_identities[node.id]) + state["artifacts"] = {(e, c): k for e, c, k in state["artifacts"]} + # The accepted prefix runner already independently sealed this + # receipt. No receipt from the receiving run is trusted here. + states[node.id] = state + continue + key = keys[node.id] + structural = node.structural is not StructuralDelta.NONE + outputs = ( + cells if structural else tuple((o.entity, o.column) for o in node.outputs) + ) + states[node.id] = { + "key": key, + "ref": node.kernel, + "implementation": implementations[node.id], + "capabilities": json.loads( + canonical_json( + _capabilities_projection(kernels.get(node.kernel).capabilities) + ) + ), + "typed_artifacts": json.loads( + canonical_json(typed_contracts(compiled, node, keys, kernels)) + ), + "seed": seed(key), + "frame_key": frame_key(key) if structural else None, + "weight_key": weights_key(key, node.weights.entity) + if node.weights + else None, + "artifacts": {(e, c): artifact_key(key, e, c) for e, c in outputs}, + "opaque_artifacts": { + o.name: opaque_artifact_key(key, o.name) for o in node.artifact_outputs + }, + } + return states + + +def _complete_receipts( + compiled: CompiledGraph, + states: dict[str, dict[str, object]], + authored: dict[str, dict[str, object]], + observed: dict[str, Population], +): + """Normalize independently authored receipts through the executor's rules.""" + previous = {} + for node_id in compiled.order: + state = states[node_id] + if node_id in authored: + node = compiled.graph.node(node_id) + receipt = json.loads(canonical_json(authored[node_id])) + receipt["capabilities"] = dict(state["capabilities"]) + writers = _tolerance_writer_payload( + _input_writers(compiled, node_id, receipts=previous) + ) + if writers: + receipt["capabilities"]["tolerance_writers"] = writers + if node.structural not in {StructuralDelta.NONE, StructuralDelta.CREATE}: + receipt["mass"] = mass_record_receipt(observed[node_id].mass_ledger[-1]) + receipt.update(weight_cap_receipt(observed[node_id], node)) + state["receipt"] = json.loads(canonical_json(receipt)) + previous[node_id] = SimpleNamespace(receipt=state["receipt"]) + + +def _check_manifest(manifest, compiled, states): + _require( + manifest.country == compiled.graph.country and manifest.decisions == (), + "MANIFEST_COUNTRY_DECISIONS", + ) + _require( + set(manifest.populations) == set(compiled.versions.values()) + and set(manifest.mass_ledgers) == set(compiled.versions.values()), + "MANIFEST_POPULATION_VERSIONS", + ) + # Detached expectations use JSON arrays. NodeReceipt freezes those arrays + # as tuples, including nested capabilities and typed artifact descriptors. + # Match its representation without weakening the canonical value check. + frozen_states = { + node_id: { + **state, + "receipt": _freeze_json(state["receipt"]), + "typed_artifacts": _freeze_json(state["typed_artifacts"]), + } + for node_id, state in states.items() + } + source_graph._check_node_states(manifest, frozen_states) + for node_id, state in states.items(): + expected = {**state, "legacy_capabilities": False} + expected["artifacts"] = [ + [e, c, key] for (e, c), key in sorted(state["artifacts"].items()) + ] + _require( + _node_identity(manifest.node(node_id)) == canonical_json(expected), + "MANIFEST_CANONICAL_VALUES", + ) + + +@dataclass(frozen=True) +class SurveyAgeCalibrationRun: + """Values validated at return; retained source handles still require rechecks.""" + + manifest: object + compiled: object + budget: object + successor: object + diagnostics: dict + counts_sha256: str + numeric_bounds_sha256: str + release_eligible: bool = False + + +def run_survey_age_calibration( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed_value, + target_registry, + epochs, + learning_rate, + resume="auto", + geography_config=None, +): + """Reconstruct sources and admit a complete invented calibration graph. + + The source and store paths are the caller's explicitly approved local + inputs. Cache reuse never bypasses fresh source preparation or the complete + receiving-population and sampling-reference checks. This v1 does not admit + genuine targets, donor-detail descendants, pruning or repeated calibration. + """ + # Refuse unsupported targets/options before any source I/O. + calibration = numerical.survey_age_calibration_node( + target_registry, + base=clone.COMBINED_CLONE_NODE, + budget_node=transport.BUDGET_NODE, + count_node=COUNT_NODE, + epochs=epochs, + learning_rate=learning_rate, + ) + return _run_survey_age_calibration( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed_value=seed_value, + epochs=epochs, + learning_rate=learning_rate, + resume=resume, + calibration=calibration, + geography_config=geography_config, + ) + + +def run_survey_age_development( + source_dir, + *, + age_source_dir, + activation, + snapshot_root, + store_root, + fraction, + seed_value, + epochs, + learning_rate, + resume="auto", + geography_config=None, +): + """Run development calibration after deriving targets from exact source pins. + + The caller separately reviews the declaration and explicitly admits these + local source paths. There is no default capture, caller-supplied registry, + mixed-inventory fallback, production attestation or release permission. + """ + _require(type(resume) is str and resume in {"auto", "require"}, "RESUME") + _require(type(epochs) is int and 1 <= epochs <= 1000, "EPOCHS") + _require( + type(learning_rate) in (int, float) + and np.isfinite(learning_rate) + and 0 < learning_rate <= 1, + "LEARNING_RATE", + ) + binding = age_activation.activation_binding(activation) + registry = age_activation.activate_survey_age_targets( + age_source_dir, declaration=activation + ) + calibration = numerical.survey_age_development_node( + registry, + activation_binding=binding, + base=clone.COMBINED_CLONE_NODE, + budget_node=transport.BUDGET_NODE, + count_node=COUNT_NODE, + epochs=epochs, + learning_rate=learning_rate, + ) + return _run_survey_age_calibration( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed_value=seed_value, + epochs=epochs, + learning_rate=learning_rate, + resume=resume, + calibration=calibration, + age_source_dir=age_source_dir, + activation=activation, + geography_config=geography_config, + ) + + +def _run_survey_age_calibration( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed_value, + epochs, + learning_rate, + resume, + calibration, + age_source_dir=None, + activation=None, + geography_config=None, +): + """Shared population, budget, replay, diagnostic and final-return checks.""" + _require(type(resume) is str and resume in {"auto", "require"}, "RESUME") + activation_identity = None + calibration_binding = numerical.calibration_activation_binding(calibration.params) + _require( + (calibration_binding is None) == (activation is None), "ACTIVATION_REQUIRED" + ) + if activation is not None: + activation_identity = canonical_json( + age_activation.activation_binding(activation) + ) + _require( + canonical_json(calibration_binding) == activation_identity, + "ACTIVATION_BINDING", + ) + geography = None + geography_identity = None + geography_config_payload = None + prefix_populations = {} + if geography_config is None: + prefix = source_graph.run_authenticated_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed_value, + resume=resume, + clones=True, + return_values=True, + ) + else: + from . import graph_atomic_survey_population as atomic_source + + geography_config_payload = ( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + ) + prefix = atomic_source.run_atomic_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed_value, + resume=resume, + geography_config=geography_config, + return_values=True, + ) + geography = atomic_source.reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + replay.same_replayed_population( + geography.population, prefix.geography_population + ) + geography_identity = budgets._population_identity(prefix.geography_population) + prefix_populations = { + stage.node.id: stage.population for stage in geography.stages + } + source_owner, source_view = source_graph._checked_preparation(prefix.preparation) + preparation_entry = source_owner._ISSUED.get(id(prefix.preparation)) + instructions = source_graph.allocation_instructions( + source_view.selection_plan, source_view.receipt["origins"]["households"] + ) + _, allocated_context, allocation_payload, _, _ = source_graph._allocation_output( + source_view.frame, source_view.context, instructions, _sha(source_view.payload) + ) + prefix_artifacts = ( + ( + source_graph.CREATE_NODE, + "preparation", + source_graph.PREPARATION_TYPE, + source_view.payload, + source_graph.SurveyPopulationCreateKernel.capabilities, + ), + ( + source_graph.CREATE_NODE, + "frame_context", + source_graph.US_FRAME_CONTEXT_TYPE, + source_view.context, + source_graph.SurveyPopulationCreateKernel.capabilities, + ), + ( + source_graph.ALLOCATION_NODE, + "allocation", + source_graph.ALLOCATION_TYPE, + allocation_payload, + source_graph.SurveyPopulationAllocationKernel.capabilities, + ), + ( + source_graph.ALLOCATION_NODE, + "frame_context", + source_graph.US_FRAME_CONTEXT_TYPE, + allocated_context, + source_graph.SurveyPopulationAllocationKernel.capabilities, + ), + ) + if geography is not None: + prefix_artifacts += tuple( + ( + node.id, + "support", + atomic_source.ATOMIC_SUPPORT_TYPE, + geography.support_payload, + atomic_source.AtomicSupportImportKernel.capabilities, + ) + for node in geography.nodes + if node.kernel == atomic_source.AtomicSupportImportKernel.ref + ) + gate = geography.stages[-1] + prefix_artifacts += ( + ( + gate.node.id, + "validation", + atomic_source.reconstruction.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE, + gate.receipt, + atomic_source.reconstruction.atomic_graph.AtomicGeographyGateKernel.capabilities, + ), + ) + initial = prefix.clone_population + _require(initial is not None, "COMPLETE_CLONE_REQUIRED") + budget = budgets.freeze_survey_origin_budget( + prefix.preparation, + allocated_population=prefix.allocated_population, + clone_population=initial, + geography_config=geography_config, + ) + budget_view = budget.checked_view() + budget_payload = budget_view.payload + numeric_payload = transport.numeric_survey_budget_payload(budget_payload) + numeric_bounds = numerical.decode_numeric_survey_bounds(numeric_payload) + count_payload, count_receipt = _measurement(initial) + _require( + numeric_bounds.grouped.household_ids + == tuple(initial.frame.table("household").household_id), + "BUDGET_HOUSEHOLD_ORDER", + ) + prefix_identities = { + name: _node_identity(row) for name, row in prefix.manifest.nodes.items() + } + initial_identity = budgets._population_identity(initial) + allocation_identity = budgets._population_identity(prefix.allocated_population) + prefix_population_identities = { + name: budgets._population_identity(population) + for name, population in prefix_populations.items() + } + prefix_budget = budget + prefix_budget_entry = budgets._entry(prefix_budget, budgets.SamplingOriginBudget) + kernels = prefix.kernels + kernels.register(transport.SurveySamplingBudgetKernel(budget_payload)) + kernels.register(ages.SurveyAgeCountArtifactKernel()) + kernels.register(numerical.SurveyAgeCalibrationKernel()) + age_nodes = ( + transport.survey_sampling_budget_node(budget_sha256=_sha(budget_payload)), + ages.survey_age_count_artifact_node( + population=initial.version, node_id=COUNT_NODE + ), + calibration, + ) + graph = replace( + prefix.compiled.graph, + nodes=(*prefix.compiled.graph.nodes, *age_nodes), + ) + compiled = compile_graph(graph) + prefix_roster = set(prefix.compiled.order) + age_roster = {transport.BUDGET_NODE, COUNT_NODE, numerical.CALIBRATION_NODE} + _require( + set(prefix_identities) == prefix_roster + and not prefix_roster & age_roster + and tuple(node.id for node in age_nodes) + == (transport.BUDGET_NODE, COUNT_NODE, numerical.CALIBRATION_NODE) + and set(compiled.order) == prefix_roster | age_roster + and len(compiled.order) == len(prefix.compiled.order) + 3 + and tuple(name for name in compiled.order if name in prefix_roster) + == prefix.compiled.order, + "EXACT_GRAPH", + ) + _require( + clone.COMBINED_CLONE_CLAIM_NODE in compiled.predecessors[transport.BUDGET_NODE], + "OWNERSHIP_CLAIM_EDGE", + ) + _paths, source_keys = _source_paths_and_keys(compiled, prefix.sources, prefix.store) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + expected_nodes = _expected_nodes( + compiled, kernels, keys, implementations, prefix_identities, initial.frame + ) + observed, snapshots = {}, {} + admitted = None + successor_entry = None + successor_payload = None + receiving_initial = None + receiving_budget_entry = None + receiving_node = ( + clone.COMBINED_CLONE_CLAIM_NODE + if geography is None + else geography.stages[-1].node.id + ) + + def observe(node_id, population): + nonlocal admitted, budget, receiving_initial, receiving_budget_entry + nonlocal successor_entry, successor_payload + _require(node_id not in observed, "DUPLICATE_OBSERVATION") + if node_id == numerical.CALIBRATION_NODE: + _require(receiving_initial is not None, "RECEIVING_BUDGET_REQUIRED") + # Independent of the numerical kernel, on both execution and load. + numerical.check_numeric_survey_weights( + numeric_bounds, population.frame.weights_for("household").values + ) + admitted = budgets.admit_survey_weight_only_population( + budget, + previous=receiving_initial, + current=population, + ) + successor_entry = budgets._entry(admitted, budgets.SamplingOriginSuccessor) + successor_payload = admitted.payload + elif node_id in {transport.BUDGET_NODE, COUNT_NODE}: + # Read-only count/transport may precede the geography gate. Compare + # their complete current version, including whichever prefix writes + # have actually occurred. Calibration waits for every base member. + if geography is None: + replay.same_replayed_frame(initial.frame, population.frame) + else: + preceding = next( + name + for name in reversed(observed) + if name in prefix_populations + and compiled.versions[name] == initial.version + ) + replay.same_replayed_population( + prefix_populations[preceding], population + ) + _require( + population.frame.weights_for("household").values.tobytes() + == numeric_bounds.incoming.tobytes(), + "RECEIVING_INCOMING_WEIGHTS", + ) + if node_id == transport.BUDGET_NODE and geography is None: + _require(receiving_initial is not None, "RECEIVING_BUDGET_REQUIRED") + _require( + budgets._population_identity(population) + == snapshots[clone.COMBINED_CLONE_CLAIM_NODE], + "COMPLETE_CLAIMED_CLONE", + ) + elif node_id == COUNT_NODE: + _require(_measure(population) == count_payload, "REMEASURED_COUNTS") + elif node_id == source_graph.ALLOCATION_NODE: + replay.same_replayed_population(prefix.allocated_population, population) + elif node_id == receiving_node: + replay.same_replayed_population(initial, population) + _require(source_graph.ALLOCATION_NODE in observed, "ALLOCATION_REQUIRED") + # Actual source reconstruction rebinds an issued budget to this run's + # receiving objects. Candidate bytes grant no authority by themselves. + budget = budgets.freeze_survey_origin_budget( + prefix.preparation, + allocated_population=observed[source_graph.ALLOCATION_NODE], + clone_population=population, + candidate=budget_payload, + geography_config=geography_config, + ) + receiving_budget_entry = budgets._entry( + budget, budgets.SamplingOriginBudget + ) + receiving_initial = population + elif node_id == source_graph.CREATE_NODE: + replay.same_replayed_frame( + prefix.manifest.population(node_id), population.frame + ) + elif node_id == clone.COMBINED_CLONE_NODE: + actual_allocation = observed[ + source_graph.ALLOCATION_NODE + if geography is None + else atomic_source.projection.NODE + ] + source_graph._verify_cloned_frame( + actual_allocation.frame, + population.frame, + actual_allocation.design_weights["household"], + ) + if geography is None: + replay.same_replayed_frame(initial.frame, population.frame) + else: + replay.same_replayed_population(prefix_populations[node_id], population) + elif node_id in prefix_populations: + replay.same_replayed_population(prefix_populations[node_id], population) + else: + _require(False, "UNEXPECTED_NODE") + observed[node_id] = population + snapshots[node_id] = budgets._population_identity(population) + + manifest = run_graph( + compiled, + sources=prefix.sources, + store=prefix.store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + _require( + tuple(observed) == compiled.order and admitted is not None, + "OBSERVATION_COVERAGE", + ) + _require(set(manifest.nodes) == set(compiled.order), "MANIFEST_NODES") + for node in graph.nodes: + actual = manifest.node(node.id) + _require( + actual.key == keys[node.id] + and actual.kernel_ref == node.kernel + and actual.kernel_impl_hash == implementations[node.id] + and actual.typed_artifacts == expected_nodes[node.id]["typed_artifacts"] + and actual.seed == seed(keys[node.id]) + and actual.legacy_capabilities is False, + "MANIFEST_NODE_IDENTITY", + ) + if node.id in prefix_identities: + _require( + _node_identity(actual) == prefix_identities[node.id], + "PREFIX_RECEIPT_REPLAY", + ) + for node_id, name, type_, payload, capabilities in ( + *prefix_artifacts, + ( + transport.BUDGET_NODE, + "budget", + budgets.BUDGET_TYPE, + budget_payload, + transport.SurveySamplingBudgetKernel.capabilities, + ), + ( + transport.BUDGET_NODE, + "numeric_bounds", + numerical.BOUNDS_TYPE, + numeric_payload, + transport.SurveySamplingBudgetKernel.capabilities, + ), + ( + COUNT_NODE, + "counts", + ages.COUNTS_TYPE, + count_payload, + ages.SurveyAgeCountArtifactKernel.capabilities, + ), + ): + source_graph._final_artifact( + manifest, + prefix.store, + node_id=node_id, + name=name, + type_=type_, + payload=payload, + capabilities=capabilities, + ) + receipt = manifest.node(numerical.CALIBRATION_NODE) + diagnostic_payload = _load_diagnostics( + prefix.store, + expected_nodes[numerical.CALIBRATION_NODE]["opaque_artifacts"]["diagnostics"], + ) + current = observed[numerical.CALIBRATION_NODE] + anchors = { + "budget_sha256": _sha(budget_payload), + "numeric_bounds_sha256": _sha(numeric_payload), + "counts_sha256": _sha(count_payload), + "accepted_weight_sha256": _sha( + current.frame.weights_for("household").values.tobytes() + ), + "constraint_digest": numeric_bounds.grouped.digest, + "weight_anchor": calibration.params["weight_anchor"], + "cap_enforcement": calibration.params["cap_enforcement"], + "fixed_zero_rows": int(np.count_nonzero(numeric_bounds.incoming == 0)), + } + _require( + _sha(diagnostic_payload) == receipt.receipt.get("diagnostics_sha256"), + "DIAGNOSTICS_DIGEST", + ) + _require( + all(receipt.receipt.get(k) == v for k, v in anchors.items()), "RECEIPT_ANCHORS" + ) + registry = numerical.demographic._registry_from_json(calibration.params["registry"]) + diagnostics = diagnostic_check.validate_survey_calibration_diagnostics( + diagnostic_payload, + counts_payload=count_payload, + bounds_payload=numeric_payload, + weights=current.frame.weights_for("household").values, + registry=registry, + epochs=epochs, + learning_rate=learning_rate, + anchors=anchors, + activation_binding=calibration_binding, + ) + diagnostic_identity = canonical_json(diagnostics) + _complete_receipts( + compiled, + expected_nodes, + { + transport.BUDGET_NODE: { + "budget_sha256": _sha(budget_payload), + "numeric_bounds_sha256": _sha(numeric_payload), + "constraint_digest": numeric_bounds.grouped.digest, + "group_count": numeric_bounds.grouped.group_count, + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + COUNT_NODE: count_receipt, + numerical.CALIBRATION_NODE: { + **anchors, + "diagnostics_sha256": _sha(diagnostic_payload), + "scope": numerical.calibration_receipt_scope(calibration.params), + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + }, + observed, + ) + _check_manifest(manifest, compiled, expected_nodes) + result = SurveyAgeCalibrationRun( + manifest, + compiled, + budget, + admitted, + diagnostics, + _sha(count_payload), + _sha(numeric_payload), + ) + final_numbers = numerical.decode_numeric_survey_bounds(numeric_payload) + # A population version can be shared by several nonstructural nodes. Check + # the actual last receiving state for every version, including the prefix. + terminal_nodes = tuple({compiled.versions[n]: n for n in compiled.order}.values()) + terminal_frames = { + node_id: manifest.population(compiled.versions[node_id]) + for node_id in terminal_nodes + } + terminal_ledgers = { + node_id: manifest.mass_ledger(compiled.versions[node_id]) + for node_id in terminal_nodes + } + # Finish storage and key/implementation I/O before final source/target + # revalidation and complete-population checks. No store read follows. + final_keys, final_implementations = _all_node_keys(compiled, kernels, source_keys) + _require( + final_keys == keys and final_implementations == implementations, + "FINAL_IMPLEMENTATIONS", + ) + _require(_measure(initial) == count_payload, "FINAL_REMEASUREMENT") + _require( + _measure(receiving_initial) == count_payload, "FINAL_RECEIVING_MEASUREMENT" + ) + # Retain both authorities; the original source/planned budget is not waived + # when a candidate-identical handle is issued for the actual receiving run. + budgets.verify_survey_origin_budget(prefix_budget) + budgets.verify_survey_weight_only_successor(admitted) + if activation is not None: + # Recheck target evidence before the optional final support read; + # complete population, configuration and owner seals follow both. + age_activation.verify_survey_age_targets( + age_source_dir, declaration=activation, registry=registry + ) + if geography is not None: + _require( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + == geography_config_payload, + "FINAL_GEOGRAPHY_CONFIG", + ) + final_support, _support_identity = atomic_source.reconstruction._read_support( + geography_config + ) + _require(final_support == geography.support_payload, "FINAL_GEOGRAPHY_SUPPORT") + _require( + source_owner._ISSUED.get(id(prefix.preparation)) is preparation_entry + and preparation_entry is not None + and prefix.preparation.payload == preparation_entry[1], + "FINAL_SOURCE_ISSUANCE", + ) + source_owner._pure_final(preparation_entry[2]) + if geography is not None: + _require( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + == geography_config_payload + and budgets._population_identity(prefix.geography_population) + == geography_identity + and all( + budgets._population_identity(population) + == prefix_population_identities[name] + for name, population in prefix_populations.items() + ), + "FINAL_GEOGRAPHY_PREFIX", + ) + _require(budget.payload == budget_payload, "FINAL_BUDGET_BYTES") + _require( + budgets._population_identity(initial) == initial_identity + and budgets._population_identity(prefix.allocated_population) + == allocation_identity, + "FINAL_PREFIX_POPULATIONS", + ) + for node_id, population in observed.items(): + _require( + budgets._population_identity(population) == snapshots[node_id], + "FINAL_POPULATION_STATE", + ) + # A nonstructural node can precede the claim within the same version; + # its own snapshot is checked above. Terminal version state is checked + # on the declared weight transition and complete ownership claim. + if node_id in terminal_nodes: + replay.same_replayed_frame(population.frame, terminal_frames[node_id]) + _require( + terminal_ledgers[node_id] == population.mass_ledger, + "FINAL_MASS_LEDGER", + ) + numerical.check_numeric_survey_weights( + final_numbers, + current.frame.weights_for("household").values, + ) + if activation is not None: + _require( + canonical_json(age_activation.activation_binding(activation)) + == activation_identity + and calibration.params.get("activation") == activation_identity.decode(), + "FINAL_ACTIVATION_BINDING", + ) + # No decoding, store access, source borrow or implementation helper follows + # this seal. It covers the complete selected prefix and age trio, including + # scope and release flags, using expectations detached before execution. + _check_manifest(manifest, compiled, expected_nodes) + _require( + canonical_json(result.diagnostics) == diagnostic_identity, + "FINAL_DIAGNOSTICS_VALUES", + ) + # The verifications above perform I/O. Finish with the retained issuance + # entries, so the second verification cannot invalidate the first handle. + _require( + budgets._entry(prefix_budget, budgets.SamplingOriginBudget) + is prefix_budget_entry + and budgets._entry(budget, budgets.SamplingOriginBudget) + is receiving_budget_entry + and prefix_budget.payload == budget.payload == budget_payload, + "FINAL_BUDGET_ISSUANCE", + ) + _require( + budgets._entry(admitted, budgets.SamplingOriginSuccessor) is successor_entry + and admitted.payload == successor_payload, + "FINAL_SUCCESSOR_ISSUANCE", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py new file mode 100644 index 000000000..d4314af93 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py @@ -0,0 +1,181 @@ +"""Two explicit S0101-only source captures; no network or target activation. + +Callers supply already acquired response bytes. Reading an explicit capture +never discovers files or falls back to a mixed national reference inventory. +""" + +from __future__ import annotations + +import hashlib +import os +import stat +from pathlib import Path +from urllib.parse import urlencode + +from microcosm.graph.canonical import canonical_json + +from .cd_reference_sources import strict_json + +MAX_RESPONSE_BYTES = 8 * 1024**2 +MAX_DESCRIPTOR_BYTES = 4096 +_DESCRIPTOR_FIELDS = frozenset( + { + "table", + "year", + "dataset", + "geography", + "kind", + "url", + "sha256", + "size_bytes", + "path", + } +) + + +def _require(condition: object, reason: str) -> None: + if not condition: + raise ValueError("SURVEY_AGE_SOURCE_" + reason) + + +def survey_age_requests() -> tuple[dict[str, object], dict[str, object]]: + """Return detached descriptors of exactly two supported national requests.""" + base = "https://api.census.gov/data/2024/acs/acs1/subject" + common = { + "table": "S0101", + "year": 2024, + "dataset": "acs/acs1/subject", + "geography": "us:*", + } + return ( + {**common, "kind": "metadata", "url": f"{base}/groups/S0101.json"}, + { + **common, + "kind": "data", + "url": base + "?" + urlencode({"get": "group(S0101)", "for": "us:*"}), + }, + ) + + +def _request(value: object) -> dict[str, object]: + _require(type(value) is dict, "REQUEST") + for request in survey_age_requests(): + if set(value) == set(request) and all( + type(value[k]) is type(v) and value[k] == v for k, v in request.items() + ): + return request + _require(False, "REQUEST") + + +def _digest(value: object) -> str: + _require( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value), + "PIN", + ) + return value + + +def _path(value: str | Path) -> Path: + path = Path(value) + _require(".." not in path.parts, "PATH") + path = path.absolute() + for member in (*reversed(path.parents), path): + _require(not member.is_symlink(), "SYMLINK") + return path + + +def _read(path: Path, limit: int) -> bytes: + path = _path(path) + try: + # A FIFO must reach the regular-file check instead of waiting for a + # writer before fstat can reject it. Nonblocking leaves regular reads unchanged. + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + _require(stat.S_ISREG(before.st_mode), "REGULAR_FILE") + _require(0 < before.st_size <= limit, "SIZE") + payload = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + _require( + len(payload) == before.st_size + and (before.st_size, before.st_mtime_ns, before.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns), + "CHANGED_FILE", + ) + return payload + except OSError: + raise ValueError("SURVEY_AGE_SOURCE_FILE") from None + + +def _write(path: Path, payload: bytes, limit: int) -> None: + path = _path(path) + if path.exists(): + _require(_read(path, limit) == payload, "IMMUTABLE") + return + path.parent.mkdir(parents=True, exist_ok=True) + _path(path) + with path.open("xb") as stream: + stream.write(payload) + + +def capture_survey_age_response( + root: str | Path, request: dict, payload: bytes +) -> dict: + """Store one caller-acquired response immutably; grant no target permission.""" + request = _request(request) + _require(type(payload) is bytes and 0 < len(payload) <= MAX_RESPONSE_BYTES, "SIZE") + strict_json(payload) + root = _path(root) + digest = hashlib.sha256(payload).hexdigest() + descriptor = { + **request, + "sha256": digest, + "size_bytes": len(payload), + "path": f"raw/{digest}.json", + } + key = hashlib.sha256(request["url"].encode()).hexdigest() + descriptor_path = root / "requests" / f"{key}.json" + encoded = canonical_json(descriptor) + # Refuse replacement before writing even an unreferenced new raw file. + if _path(descriptor_path).exists(): + _require(_read(descriptor_path, MAX_DESCRIPTOR_BYTES) == encoded, "IMMUTABLE") + _write(root / descriptor["path"], payload, MAX_RESPONSE_BYTES) + _write(descriptor_path, encoded, MAX_DESCRIPTOR_BYTES) + return descriptor + + +def read_survey_age_response( + root: str | Path, kind: str, expected_sha256: str +) -> tuple[bytes, object]: + """Read the fixed named response against a separately reviewed content pin.""" + _require(type(kind) is str and kind in {"metadata", "data"}, "KIND") + expected_sha256 = _digest(expected_sha256) + request = next(row for row in survey_age_requests() if row["kind"] == kind) + root = _path(root) + key = hashlib.sha256(request["url"].encode()).hexdigest() + descriptor = strict_json( + _read(root / "requests" / f"{key}.json", MAX_DESCRIPTOR_BYTES) + ) + _require( + type(descriptor) is dict and set(descriptor) == _DESCRIPTOR_FIELDS, "DESCRIPTOR" + ) + _request({key: descriptor[key] for key in request}) + _require( + descriptor["sha256"] == expected_sha256 + and descriptor["path"] == f"raw/{expected_sha256}.json", + "PIN", + ) + _require( + type(descriptor["size_bytes"]) is int + and 0 < descriptor["size_bytes"] <= MAX_RESPONSE_BYTES, + "SIZE", + ) + payload = _read(root / f"raw/{expected_sha256}.json", MAX_RESPONSE_BYTES) + _require( + len(payload) == descriptor["size_bytes"] + and hashlib.sha256(payload).hexdigest() == expected_sha256, + "RESPONSE_PIN", + ) + return payload, strict_json(payload) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py new file mode 100644 index 000000000..2beb174dc --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py @@ -0,0 +1,639 @@ +"""Reconstruct observed constraints, initial clones and geography from live sources. + +The immutable recipe names normalized support bytes; their digest proves byte +integrity, not publisher provenance. A native source adapter must establish the +latter separately. This helper executes the maintained pure geography operators, +not graph kernels, and issues no source, Population, or release authority. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +from dataclasses import asdict, dataclass, replace +from io import BytesIO +from pathlib import Path +from zipfile import ZipFile + +import numpy as np +import pandas as pd + +from microcosm.build import atomic_geography as atomic +from microcosm.build import graph_atomic_geography as atomic_graph +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ( + Graph, + KernelResult, + Node, + SourceRef, + StructuralDelta, + compile_graph, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES +from microcosm.graph.executor import ( + _expand_declared_payload, + _expand_rewrite_coordinates, +) +from microcosm.graph.population import ( + Population, + _mass_record, + _storage_parts, + expand_lineage_receipt, + expand_writes_receipt, + patch, +) + +from . import atomic_block_support as blocks +from . import current_survey_geography as observed +from . import graph_atomic_survey_clone as composition +from . import graph_combined_clone as clone +from . import graph_current_survey_geography as observed_graph +from . import graph_survey_population as graph +from . import puf_support +from . import survey_population_preparation as source +from . import survey_population_replay as replay +from .graph_sources import frame_column_declarations + +PROTOCOL = "microcosm.us.atomic-survey-reconstruction.v2" +MAX_SUPPORT_BYTES = RAW_BYTES_MAX_BYTES +MAX_SUPPORT_EXPANDED_BYTES = 2 * 1024**3 +MAX_SUPPORT_MEMBERS = 32 + + +def _require(condition, reason): + if not condition: + raise ValueError("ATOMIC_SURVEY_RECONSTRUCTION_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyReconstruction: + """Exact immutable inputs; construction itself authenticates no source.""" + + support_path: str + support_sha256: str + source_ids: tuple[tuple[str, str], ...] + seed: int + + def __post_init__(self): + _config_bytes(self) + + def to_bytes(self): + """Revalidate and detach the recipe, including the literal source path.""" + return _config_bytes(self) + + +def _config_bytes(config): + _require(type(config) is AtomicSurveyReconstruction, "CONFIG_TYPE") + _require( + type(config.support_path) is str + and 0 < len(config.support_path) <= 4096 + and "\0" not in config.support_path + and Path(config.support_path).is_absolute() + and ".." not in Path(config.support_path).parts, + "SUPPORT_PATH", + ) + _require( + type(config.support_sha256) is str + and re.fullmatch(r"[0-9a-f]{64}", config.support_sha256) is not None, + "SUPPORT_DIGEST", + ) + _require( + type(config.source_ids) is tuple + and len(config.source_ids) == 3 + and all( + type(pair) is tuple + and len(pair) == 2 + and all(type(value) is str for value in pair) + and 0 < len(pair[1]) <= 4096 + and pair[1].strip() == pair[1] + for pair in config.source_ids + ) + and tuple(pair[0] for pair in config.source_ids) + == ("district", "population", "puma"), + "SOURCE_IDENTITIES", + ) + _require(type(config.seed) is int and 0 <= config.seed < 2**63, "SEED") + return canonical_json( + { + "protocol": PROTOCOL, + "support_path": config.support_path, + "support_sha256": config.support_sha256, + "source_ids": dict(config.source_ids), + "seed": config.seed, + } + ) + + +def _file_identity(value): + return tuple( + getattr(value, name) + for name in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + ) + + +def _read_support(config): + """Read one bounded regular file and verify its exact pinned bytes.""" + descriptor = os.open( + config.support_path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0), + ) + with os.fdopen(descriptor, "rb") as stream: + before = os.fstat(stream.fileno()) + _require( + stat.S_ISREG(before.st_mode) and 0 < before.st_size <= MAX_SUPPORT_BYTES, + "SUPPORT_SIZE", + ) + payload = stream.read(before.st_size + 1) + after = os.fstat(stream.fileno()) + current = os.stat(config.support_path, follow_symlinks=False) + _require( + stat.S_ISREG(current.st_mode) + and _file_identity(before) == _file_identity(after) == _file_identity(current) + and len(payload) == before.st_size + and _sha(payload) == config.support_sha256, + "SUPPORT_CHANGED", + ) + return payload, _file_identity(current) + + +def _decode_support(payload): + # Bound declared decompressed bytes before numpy opens any array. The shared + # decoder still owns every schema, value, mapping and sampling-weight check. + with ZipFile(BytesIO(payload)) as archive: + members = archive.infolist() + _require( + 0 < len(members) <= MAX_SUPPORT_MEMBERS + and sum(member.file_size for member in members) + <= MAX_SUPPORT_EXPANDED_BYTES, + "SUPPORT_EXPANDED_SIZE", + ) + for member in members: + with archive.open(member) as stream: + version = np.lib.format.read_magic(stream) + if version == (1, 0): + shape, _fortran, dtype = np.lib.format.read_array_header_1_0(stream) + elif version == (2, 0): + shape, _fortran, dtype = np.lib.format.read_array_header_2_0(stream) + else: + raise ValueError("ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_NPY_VERSION") + # A small ZIP member must not claim a huge allocation in its + # NPY header. Validate the physical payload length before load. + _require( + not dtype.hasobject + and all(type(size) is int and size >= 0 for size in shape) + and math.prod(shape) * dtype.itemsize + == member.file_size - stream.tell(), + "SUPPORT_ARRAY_SIZE", + ) + return atomic.decode_atomic_support(payload) + + +def _copy_population(population): + frame = population.frame + return replace( + population, + frame=Frame( + {entity: frame.table(entity).copy(deep=True) for entity in frame.entities}, + frame.schema, + { + entity: Weights( + frame.weights_for(entity).values.copy(), + frame.weights_for(entity).kind, + ) + for entity in frame.weighted_entities + }, + frame.strata.copy(deep=True), + # Frame recursively freezes fresh metadata containers. The admitted + # mass records contain scalars and immutable tuples; replacing each + # record avoids deepcopy's reflective dataclass class-cache writes. + metadata=frame.metadata, + mass_log=tuple(replace(record) for record in frame.mass_log), + ), + owners=dict(population.owners), + weight_kind=dict(population.weight_kind), + design_weights={ + name: values.copy() for name, values in population.design_weights.items() + }, + mass_ledger=tuple(replace(record) for record in population.mass_ledger), + ) + + +def _population_stamp(population): + """Pure in-process seal, including storage beneath nullable masks. + + Never persist it or compare it across reconstructions: it folds physical + storage parts (masked ``_data`` under nulls, and before microcosm#907 + object-dtype pointer bytes), so equal content rebuilt elsewhere need not + match. A cross-call pin carries ``source._frame_identity`` plus the + version, owners, weight kinds, mass ledger and design weights instead. + """ + _require(type(population) is Population, "POPULATION_TYPE") + digest = hashlib.sha256() + frame = population.frame + digest.update( + canonical_json( + { + "frame": source._frame_identity(frame), + "version": population.version, + "owners": sorted(population.owners.items()), + "weight_kind": list(population.weight_kind.items()), + "mass_ledger": [asdict(record) for record in population.mass_ledger], + } + ) + ) + for series in ( + *( + frame.table(entity)[column] + for entity in frame.entities + for column in frame.table(entity) + ), + frame.strata, + ): + # Whole series: the slice selects every row in order, byte for byte. + for part in _storage_parts(series, slice(None)): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + for entity, values in population.design_weights.items(): + digest.update(canonical_json((entity, str(values.dtype), values.shape))) + digest.update(values.tobytes()) + return digest.hexdigest() + + +def _raw_allocation(view, population): + """Reconstruct the original allocation, retaining all existing checks.""" + _require(type(population) is Population, "RAW_POPULATION_TYPE") + instructions = graph.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + _weights, _context, _allocation, receipt, expected = graph._allocation_output( + view.frame, view.context, instructions, _sha(view.payload) + ) + columns = frame_column_declarations(view.frame) + nodes = graph.survey_population_nodes( + columns, + preparation_sha256=_sha(view.payload), + fraction=view.selection_plan.fraction, + seed=view.selection_plan.seed, + ) + graph._same_frame(expected, population.frame) + replay.same_replayed_frame(expected, population.frame) + design = view.frame.weights_for("household").values + graph._check_design_anchors(population, design) + cells = ( + (entity, str(column)) + for entity in view.frame.entities + for column in view.frame.table(entity) + ) + ledger = ( + _mass_record( + view.frame, expected, nodes[1], KernelResult(receipt=receipt), "declared" + ), + ) + graph._check_population_state( + population, + version=graph.ALLOCATION_NODE, + owners=dict.fromkeys(cells, graph.ALLOCATION_NODE), + kind=WeightKind.IMPORTANCE, + ledger=ledger, + ) + return columns, nodes + + +def _clone_expectations(before, nodes, compiled): + """Independently reconstruct the complete clone, including inherited cells.""" + expanded = puf_support.clone_us_frame_for_puf_support( + before.frame, + clone_attachment_fraction=1.0, + clone_attachment_seed=0, + ) + design = graph._verify_cloned_frame( + before.frame, expanded, before.design_weights["household"] + ) + expand = next(n for n in nodes if n.structural is StructuralDelta.EXPAND) + claim = next(n for n in nodes if n.id == clone.COMBINED_CLONE_CLAIM_NODE) + ledger = ( + *before.mass_ledger, + _mass_record(before.frame, expanded, expand, KernelResult(), "conserve"), + ) + owners = { + (e, str(c)): expand.id for e in expanded.entities for c in expanded.table(e) + } + expected, receipts = {}, {} + lineage, facts = {}, {} + for entity in US_SCHEMA.entities: + lineage[entity], facts[entity] = clone._entity_lineage( + before.frame, expanded, entity + ) + authority = puf_support.validate_puf_clone_attachment( + expanded, boundary=graph.PHASE, expected_fraction=1.0, expected_seed=0 + ) + receipt = clone.USCombinedSurveyCloneExpandKernel._receipt( + before.frame, expanded, ("acs", "asec"), authority, facts + ) + receipt["expand"] = expand_lineage_receipt(lineage) + receipt["expand_declared"] = _expand_declared_payload(expand) + receipt["expand_writes"] = expand_writes_receipt( + before.frame, + expanded, + expand, + receipt, + rewrite_coordinates=_expand_rewrite_coordinates(compiled, expand), + ) + receipts[expand.id] = receipt + receipts[claim.id] = { + "phase": clone.COMBINED_CLONE_PHASE, + "claimed_cells": sorted(f"{o.entity}.{o.column}" for o in claim.outputs), + } + # Preserve exact structural and claim ownership before assignment. + for node_id in compiled.order: + if node_id not in receipts: + continue + if node_id == claim.id: + owners.update({(o.entity, o.column): claim.id for o in claim.outputs}) + expected[node_id] = Population.from_frame( + expanded, + expand.id, + owners=owners, + mass_ledger=ledger, + design_weights={"household": np.array(design, copy=True)}, + ) + return expected, receipts + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyGeographyStage: + """Expected complete Population and pure receipt, never executor evidence.""" + + node: Node + population: Population + receipt: bytes + artifacts: tuple[tuple[str, bytes], ...] = () + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyGeographyReconstruction: + """Detached reconstruction values; consumers must independently requalify.""" + + population: Population + observed_population: Population + expanded_population: Population + stages: tuple[AtomicSurveyGeographyStage, ...] + nodes: tuple[Node, ...] + definition: bytes + projection_receipt: bytes + support_payload: bytes + sources: tuple[tuple[str, str], ...] + config_sha256: str + receipt: bytes + + +def reconstruct_atomic_survey_geography(preparation, allocated_population, config): + """Qualify sources, complete initial clones, then assign on the clone version. + + Stage values follow the actual compiled order, including the structural + expansion. Support import and clone ownership may have either relative order. + Callers compare these complete expected Populations to executor observations + on cold and required-warm runs; these values do not replace those checks. + """ + config_bytes = _config_bytes(config) + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + payload, state = entry[1], entry[2] + view = source.CheckedSurveyPopulationView( + payload, state.context, state.frame, state.plan, json.loads(payload) + ) + columns, source_nodes = _raw_allocation(view, allocated_population) + original_stamp = _population_stamp(allocated_population) + current = _copy_population(allocated_population) + support_payload, support_identity = _read_support(config) + support = _decode_support(support_payload) + supports = {blocks.SYSTEM: support} + definition = blocks.assignment_definition( + identity=composition.ASSIGNMENT_IDENTITY, + state_column=observed.COLUMNS[1], + puma_column=observed.COLUMNS[2], + source_ids=dict(config.source_ids), + seed=config.seed, + ) + definition_bytes = canonical_json(definition) + projection = observed.qualify_current_survey_geography(preparation) + projection_receipt = projection.receipt + observed_graph._check_projection( + observed, projection, payload=payload, receipt_sha256=_sha(projection_receipt) + ) + _require( + projection.household.index.tolist() + == current.frame.table("household").household_id.tolist(), + "PROJECTION_ORDER", + ) + observation = observed_graph.current_survey_geography_node( + preparation_sha256=_sha(payload), + projection_receipt_sha256=_sha(projection_receipt), + population=graph.ALLOCATION_NODE, + ) + additions = composition.atomic_survey_clone_nodes( + definition, (*columns, *observation.outputs), base=graph.ALLOCATION_NODE + ) + nodes = (observation, *additions) + compiled = compile_graph( + Graph( + "us", + ( + SourceRef(graph.SOURCE_NAME, graph.SOURCE_CODEC), + SourceRef(blocks.SOURCE, "raw-bytes-v1"), + ), + (*source_nodes, *nodes), + ) + ) + wanted = {node.id for node in nodes} + ordered = tuple( + compiled.graph.node(name) for name in compiled.order if name in wanted + ) + stages = [] + cloned, clone_receipts = {}, {} + for node in ordered: + values, artifacts = {}, () + structural_or_claim = False + if node.id == observation.id: + values = { + ("household", name): projection.household[name].copy(deep=True) + for name in observed.COLUMNS + } + receipt = { + "phase": observed_graph.PHASE, + "preparation_sha256": _sha(payload), + "projection_receipt_sha256": _sha(projection_receipt), + "source_projection": json.loads(projection_receipt), + "population_admission_issued": False, + "release_eligible": False, + } + elif node.structural is StructuralDelta.EXPAND: + cloned, clone_receipts = _clone_expectations(current, nodes, compiled) + current = _copy_population(cloned[node.id]) + receipt = clone_receipts[node.id] + structural_or_claim = True + elif node.id == clone.COMBINED_CLONE_CLAIM_NODE: + _require(node.id in cloned, "CLONE_BEFORE_CLAIM") + current = _copy_population(cloned[node.id]) + receipt = clone_receipts[node.id] + structural_or_claim = True + elif node.kernel == atomic_graph.AtomicSupportImportKernel.ref: + _require( + support.metadata["system"] == node.params["system"], "SUPPORT_SYSTEM" + ) + artifacts = (("support", support_payload),) + receipt = { + "support_sha256": support.sha256, + "areas": len(support.arrays["area"]), + "metadata": support.metadata, + } + else: + households = current.frame.table("household") + if node.kernel == atomic_graph.AtomicAssignKernel.ref: + output = atomic.assign_atomic(households, definition, supports) + receipt = { + "scope": "atomic_area_assignment", + "households": len(output), + "support_sha256": { + name: value.sha256 for name, value in supports.items() + }, + } + elif node.kernel == atomic_graph.AtomicDeriveKernel.ref: + output = atomic.derive_geography(households, definition, supports) + receipt = { + "scope": "atomic_area_functional_lookup", + "layers": { + system["id"]: system["layers"] + for system in definition["systems"] + }, + } + else: + _require( + node.kernel == atomic_graph.AtomicGeographyGateKernel.ref, + "STAGE_ROSTER", + ) + output = None + receipt = atomic.validate_geography(households, definition, supports) + artifacts = (("validation", canonical_json(receipt)),) + if output is not None: + ids = pd.Index(households.household_id.to_numpy(), name="household_id") + values = { + ("household", name): pd.Series(output[name].array.copy(), index=ids) + for name in output + } + if not structural_or_claim: + current = _copy_population( + patch(current, node, KernelResult(columns=values)) + ) + stages.append( + AtomicSurveyGeographyStage( + node, current, canonical_json(receipt), artifacts + ) + ) + stage_tuple = tuple(stages) + by_id = {stage.node.id: stage for stage in stage_tuple} + observed_population = by_id[observation.id].population + expanded_population = by_id[clone.COMBINED_CLONE_CLAIM_NODE].population + validation_payload = by_id["geography.gate"].artifacts[0][1] + receipt = canonical_json( + { + "protocol": PROTOCOL, + "config_sha256": _sha(config_bytes), + "preparation_sha256": _sha(payload), + "projection_receipt_sha256": _sha(projection_receipt), + "support_sha256": _sha(support_payload), + "definition_sha256": _sha(definition_bytes), + "validation_receipt_sha256": _sha(validation_payload), + "assignment_identity": list(composition.ASSIGNMENT_IDENTITY), + "stages": [stage.node.id for stage in stage_tuple], + "frame_sha256": source._frame_identity(current.frame), + "population_sha256": _population_stamp(current), + "observed_population_sha256": _population_stamp(observed_population), + "expanded_population_sha256": _population_stamp(expanded_population), + "publisher_provenance_established": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + result = AtomicSurveyGeographyReconstruction( + population=_copy_population(current), + observed_population=observed_population, + expanded_population=expanded_population, + stages=stage_tuple, + nodes=nodes, + definition=definition_bytes, + projection_receipt=projection_receipt, + support_payload=support_payload, + sources=((blocks.SOURCE, config.support_path),), + config_sha256=_sha(config_bytes), + receipt=receipt, + ) + result_stamps = tuple( + _population_stamp(stage.population) for stage in result.stages + ) + stage_values = tuple( + (stage.node, stage.receipt, stage.artifacts) for stage in result.stages + ) + final_stamp = _population_stamp(result.population) + # Finish support and source-owner I/O before pure rechecks of every retained + # input and every detached expected Population. No callback is an authority. + _require( + _read_support(config) == (support_payload, support_identity), "FINAL_SUPPORT" + ) + final_entry = preparation._checked() + support_after = os.stat(config.support_path, follow_symlinks=False) + _require( + final_entry is entry + and source._ISSUED.get(id(preparation)) is entry + and preparation.payload == payload + and _config_bytes(config) == config_bytes, + "FINAL_ISSUANCE", + ) + _require( + stat.S_ISREG(support_after.st_mode) + and _file_identity(support_after) == support_identity, + "FINAL_SUPPORT_STAT", + ) + source._pure_final(state) + _raw_allocation(view, allocated_population) + _require( + _population_stamp(allocated_population) == original_stamp + and tuple(_population_stamp(stage.population) for stage in result.stages) + == result_stamps + and _population_stamp(result.population) == final_stamp, + "FINAL_POPULATIONS", + ) + observed_graph._check_projection( + observed, projection, payload=payload, receipt_sha256=_sha(projection_receipt) + ) + _require( + result.nodes == nodes + and result.observed_population is observed_population + and result.expanded_population is expanded_population + and tuple(stage.node for stage in result.stages) == ordered + and tuple( + (stage.node, stage.receipt, stage.artifacts) for stage in result.stages + ) + == stage_values + and result.definition == definition_bytes + and result.projection_receipt == projection_receipt + and result.support_payload == support_payload + and result.sources == ((blocks.SOURCE, config.support_path),) + and result.config_sha256 == _sha(config_bytes) + and result.receipt == receipt, + "FINAL_RESULT", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py new file mode 100644 index 000000000..033322cc3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py @@ -0,0 +1,207 @@ +"""Recompute supported age diagnostics without rerunning an optimizer. + +This is a value checker, not source authority. The caller supplies independently +verified ordered counts, sampling bounds and receiving weights. Optimizer history +has a bounded schema but cannot be reconstructed from those final values. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np + +from microcosm.calibrate import diagnostics_payload +from microcosm.calibrate.score import score_targets +from microcosm.frame import WeightKind, Weights +from microcosm.graph.canonical import canonical_json + +from . import graph_survey_calibration as numeric + +MAX_DIAGNOSTIC_BYTES = 8 * 1024**2 +HISTORY_FIELDS = ( + "initial_loss", + "loss_trajectory", + "options.matrix_format", + "options.grouped_upper_bounds.last_corrected_group_count", +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_DIAGNOSTICS_" + reason) + + +def _history(document, epochs, group_count): + options = document.get("options") + _require(type(options) is dict, "OPTIONS") + grouped = options.get("grouped_upper_bounds") + _require(type(grouped) is dict, "GROUPED_OPTIONS") + corrected = grouped.get("last_corrected_group_count") + _require(type(corrected) is int and 0 <= corrected <= group_count, "HISTORY_COUNT") + layout = options.get("matrix_format") + _require( + type(layout) is str and layout in {"dense", "sparse_csr"}, "HISTORY_LAYOUT" + ) + trajectory = document.get("loss_trajectory") + _require(type(trajectory) is list and len(trajectory) == epochs, "HISTORY_LENGTH") + _require( + all(type(v) is float and np.isfinite(v) and 0 <= v <= 10 for v in trajectory), + "HISTORY_LOSS", + ) + _require( + type(document.get("initial_loss")) is float + and document["initial_loss"] == trajectory[0], + "HISTORY_INITIAL", + ) + return corrected, layout, trajectory + + +def validate_survey_calibration_diagnostics( + payload, + *, + counts_payload, + bounds_payload, + weights, + registry, + epochs, + learning_rate, + anchors, + activation_binding=None, +): + """Return defensive schema-8 values plus an explicit verification annotation. + + Every field except the four named history fields is reconstructed using + score_targets and the package's diagnostics encoder. Only warning-free + schema 8 (every registry-backed target carrying its calibration hierarchy) + with target-loss attribution is supported in this first profile. + No convergence, optimality, target eligibility or release claim is granted. + """ + _require( + type(payload) is bytes and 0 < len(payload) <= MAX_DIAGNOSTIC_BYTES, + "PAYLOAD_LIMIT", + ) + try: + document = json.loads(payload) + except (ValueError, UnicodeError, RecursionError): + raise ValueError("SURVEY_DIAGNOSTICS_JSON") from None + _require(type(document) is dict, "DOCUMENT") + _require(canonical_json(document) == payload, "CANONICAL") + # Reuse the actual declaration's option/registry restrictions. This is a + # declaration and numeric work Frame, never a population/source substitute. + factory = numeric.survey_age_calibration_node + profile_options = {} + if activation_binding is not None: + factory = numeric.survey_age_development_node + profile_options["activation_binding"] = activation_binding + factory( + registry, + **profile_options, + base="diagnostic-values", + budget_node="bounds", + count_node="counts", + epochs=epochs, + learning_rate=learning_rate, + ) + bounds = numeric.decode_numeric_survey_bounds(bounds_payload) + counts = numeric.ages.decode_survey_age_counts(counts_payload) + _require( + counts.population == bounds.population + and tuple(counts.household_ids) == bounds.grouped.household_ids, + "ORDERED_POPULATION", + ) + numeric.check_numeric_survey_weights(bounds, weights) + accepted_weights = weights.tobytes() + corrected, layout, trajectory = _history( + document, epochs, bounds.grouped.group_count + ) + table = counts.counts.copy(deep=True) + table.insert(0, "household_id", counts.household_ids.copy()) + work = numeric.demographic.kernels_module._frame_from_context( + SimpleNamespace( + tables={"household": table}, + weights={ + "household": Weights(bounds.incoming.copy(), WeightKind.IMPORTANCE) + }, + ), + "household", + ) + targets = registry.to_target_set() + initial = score_targets( + work, targets, weights=bounds.incoming, target_loss_cap=10.0 + ) + final = score_targets(work, targets, weights=weights, target_loss_cap=10.0) + _require(not initial.skipped and not final.skipped, "SKIPPED_TARGETS") + options = { + # The closed grouped profile supplies no gates or record-budget search. + # Reconstruct these solver fields; never inherit them from the payload. + "gate_initialization_supplied": False, + "budget_basis": "nonzero_count", + "feasible_draw_pi_hi": None, + "budget_search": None, + "grouped_preserve_zeros": { + "enabled": True, + "fixed_zero_count": int(np.count_nonzero(bounds.incoming == 0)), + "ordered_zero_mask_sha256": hashlib.sha256( + np.asarray(bounds.incoming == 0, dtype=np.uint8).tobytes() + ).hexdigest(), + }, + "grouped_upper_bounds": bounds.grouped.diagnostics(weights, corrected), + "method": "adam", + "epochs": epochs, + "learning_rate": learning_rate, + # This profile always uses grouped bounds, whose accepted weights are + # the closing state. Ungrouped Adam's best-iterate history cannot apply. + "iterate_selection": "closing_state", + "iterate_selection_receipt": {}, + "mass": "free", + "mass_reason": None, + "max_weight_ratio": None, + "target_records": None, + "l1_lambda": 0.0, + "l1_penalty": "mean_initial_weight_ratio_abs", + "l2_lambda": 0.0, + "l2_anchor": "initial", + "l2_anchor_weights_supplied": False, + "l2_penalty": "mean_initial_pre_gate_weight_ratio_squared", + "seed": 0, + "target_loss_weights": final.options["target_loss_weights"], + "target_loss_scales": final.options["target_loss_scales"], + "warm_start_weights": {"enabled": False, "kind": None}, + "matrix_format": layout, + } + combined = replace( + final, + diagnostics=tuple( + replace(after, initial_estimate=before.final_estimate) + for before, after in zip( + initial.diagnostics, final.diagnostics, strict=True + ) + ), + options=options, + loss_trajectory=np.asarray(trajectory, dtype=np.float64), + ) + expected = diagnostics_payload(combined, target_registry=registry, build=anchors) + _require( + expected.get("schema_version") == 8 + and expected.get("diagnostic_warnings") == [] + and "target_loss_basis" in expected, + "UNSUPPORTED_DIAGNOSTIC_SCHEMA", + ) + _require(canonical_json(expected) == payload, "RECOMPUTED_VALUES") + # Make the distinction visible to downstream callers without altering the + # stored schema-8 evidence or its digest. + document["verification"] = { + "protocol": "microcosm.us.survey-age-diagnostics-verification.v1", + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "recomputed": "all artifact fields except the listed optimizer history", + "optimizer_history_schema_checked_only": list(HISTORY_FIELDS), + "optimizer_rerun": False, + "release_eligible": False, + } + numeric.check_numeric_survey_weights(bounds, weights) + _require(weights.tobytes() == accepted_weights, "FINAL_WEIGHT_BYTES") + return document diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py new file mode 100644 index 000000000..933e86851 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py @@ -0,0 +1,259 @@ +"""Plan one draw over complete supplied survey records, before native Frames. + +This is declaration/arithmetic only. Closed source owners must authenticate the +complete inputs and bind the result to the selected population. This module +reads no source, creates no Frame and never totals publisher weights. Original +anchors remain separate from the one-time share/inclusion multiplier. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from fractions import Fraction + +from microcosm.build.survey_domain_sample import select_domain_households + +from . import survey_population_domains as domains + +PROTOCOL = "microcosm.us.survey-catalogue-selection.v1" +_BATCH_HOUSEHOLDS = 10_000 +_BATCH_PEOPLE = 100_000 + + +class CatalogueSelectionError(ValueError): + """Static refusal, without raw source identities or observations.""" + + +def _require(condition, code): + if not condition: + raise CatalogueSelectionError(code) + + +@dataclass(frozen=True, slots=True) +class SelectedHousehold: + key: domains.HouseholdKey + domain: domains.Domain + statistical_unit: str + original_design_weight: Fraction + share: Fraction + inclusion_probability: Fraction + + @property + def importance_multiplier(self) -> Fraction: + return self.share / self.inclusion_probability + + +@dataclass(frozen=True, slots=True) +class ExcludedHousehold: + key: domains.HouseholdKey + reason: str + + +@dataclass(frozen=True, slots=True) +class SelectionCell: + source: domains.Source + domain: domains.Domain + statistical_unit: str + share: Fraction + eligible_households: int + selected_households: int + inclusion_probability: Fraction | None + + +@dataclass(frozen=True, slots=True) +class CatalogueSelectionPlan: + selected: tuple[SelectedHousehold, ...] + excluded: tuple[ExcludedHousehold, ...] + cells: tuple[SelectionCell, ...] + supplied_households: int + fraction: Fraction + seed: int + + @property + def source_authenticated(self) -> bool: + return False + + @property + def population_binding_authenticated(self) -> bool: + return False + + @property + def release_eligible(self) -> bool: + return False + + +def _sampling_key(key): + # Keep the original H_SEQ token in the result, but order ASEC keys + # numerically. All validated values fit five digits; padded strings allow + # both survey channels to use the shared selector's one literal-ID axis. + return ( + key.native_id + if key.source is domains.Source.ACS + else f"{int(key.native_id):05d}" + ) + + +def _ordered_key(key): + return key.source.value, _sampling_key(key) + + +def _original_anchor(decision): + row = decision.original + if row.key.source is domains.Source.ASEC: + return Fraction(int(row.hsup_wgt), 100) + if decision.statistical_unit == "person": + return Fraction(int(row.persons[0].pwgtp)) + return Fraction(int(row.wgtp)) + + +def plan_catalogue_selection(*, acs_households, asec_households, fraction, seed): + """Classify all supplied records and draw once in each positive-share cell. + + ``N`` counts complete eligible households in the supplied catalogue, never + selected Frame rows. Unknown classifications refuse before any draw, even + if a tentative share is zero. Known structural exclusions have a separate + ledger. Original zero ASEC anchors remain eligible; an all-zero selected + cell refuses without redrawing. No common-total normalization is performed. + + Full source completeness, immutable source authority, and original files + are obligations of the calling source orchestration, not this value API. + """ + _require( + type(acs_households) is tuple and type(asec_households) is tuple, + "CATALOGUE_TUPLES", + ) + _require(type(fraction) is Fraction and 0 < fraction <= 1, "FRACTION") + _require(type(seed) is int and 0 <= seed < 2**64, "SEED") + _require( + type(_BATCH_HOUSEHOLDS) is int + and 0 < _BATCH_HOUSEHOLDS <= min(10_000, domains.MAX_HOUSEHOLDS) + and type(_BATCH_PEOPLE) is int + and 0 < _BATCH_PEOPLE <= min(100_000, domains.MAX_TOTAL_MEMBERS), + "BATCH_LIMITS", + ) + rows, exclusions = [], [] + household_keys, person_keys = set(), set() + batch, batch_people = [], 0 + + def consume(): + for decision in domains.classify_households(tuple(batch)): + _require( + decision.status is not domains.Status.REVIEW_REQUIRED, + "SOURCE_CLASSIFICATION_REVIEW_REQUIRED", + ) + if decision.status is domains.Status.EXCLUDED: + _require(decision.share == 0, "EXCLUSION_SHARE") + exclusions.append( + ExcludedHousehold(decision.original.key, decision.reason) + ) + else: + _require( + decision.status is domains.Status.ELIGIBLE + and decision.share is not None + and decision.share > 0 + and decision.domain is not None, + "ELIGIBLE_CLASSIFICATION", + ) + # Retain only the compact household decision, not another full + # population of per-person decisions and observation objects. + rows.append( + ( + decision.original.key, + decision.domain, + decision.statistical_unit, + _original_anchor(decision), + decision.share, + ) + ) + + for source, expected, catalogue in ( + (domains.Source.ACS, domains.AcsHousehold, acs_households), + (domains.Source.ASEC, domains.AsecHousehold, asec_households), + ): + for row in catalogue: + _require(type(row) is expected, "CATALOGUE_SOURCE_TYPE") + canonical = domains._key(row.key) + _require( + canonical[0] is source + and row.key.source_year == 2024 + and row.key.survey_year + == (2024 if source is domains.Source.ACS else 2025), + "CATALOGUE_PERIOD_SOURCE", + ) + _require(canonical not in household_keys, "GLOBAL_HOUSEHOLD_COLLISION") + household_keys.add(canonical) + _require(type(row.persons) is tuple, "MEMBER_TUPLE") + _require( + len(row.persons) <= min(domains.MAX_MEMBERS, _BATCH_PEOPLE), + "HOUSEHOLD_MEMBER_BOUND", + ) + if source is domains.Source.ASEC: + for person in row.persons: + _require(type(person) is domains.AsecPerson, "PERSON_TYPE") + identity = (2024, person.peridnum) + _require(identity not in person_keys, "GLOBAL_PERSON_COLLISION") + person_keys.add(identity) + if batch and ( + len(batch) == _BATCH_HOUSEHOLDS + or batch_people + len(row.persons) > _BATCH_PEOPLE + ): + consume() + batch, batch_people = [], 0 + batch.append(row) + batch_people += len(row.persons) + if batch: + consume() + + declarations = { + (item.domain.value, source.value): (item.statistical_unit, share) + for item in domains.declaration() + for source, share in ( + (domains.Source.ACS, item.acs_share), + (domains.Source.ASEC, item.asec_share), + ) + if share > 0 + } + selection = select_domain_households( + row_ids=tuple(_sampling_key(row[0]) for row in rows), + source_channels=tuple(row[0].source.value for row in rows), + domain_keys=tuple(row[1].value for row in rows), + cells=tuple(declarations), + fraction=fraction, + seed=seed, + ) + chosen, cells = [], [] + for cell in selection: + unit, share = declarations[cell.domain, cell.source] + selected = [rows[position] for position in cell.positions] + _require( + not selected or any(row[3] > 0 for row in selected), + "SELECTED_CELL_HAS_NO_POSITIVE_ANCHOR", + ) + for key, domain, observed_unit, anchor, observed_share in selected: + _require( + (observed_unit, observed_share) == (unit, share), "DECLARATION_DRIFT" + ) + chosen.append( + SelectedHousehold( + key, domain, unit, anchor, share, cell.inclusion_probability + ) + ) + cells.append( + SelectionCell( + domains.Source(cell.source), + domains.Domain(cell.domain), + unit, + share, + cell.eligible_households, + len(selected), + cell.inclusion_probability, + ) + ) + return CatalogueSelectionPlan( + tuple(sorted(chosen, key=lambda row: _ordered_key(row.key))), + tuple(sorted(exclusions, key=lambda row: _ordered_key(row.key))), + tuple(cells), + len(household_keys), + fraction, + seed, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_demographic_contract.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_demographic_contract.py new file mode 100644 index 000000000..d98e67eed --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_demographic_contract.py @@ -0,0 +1,99 @@ +"""Immutable shared declarations for currently unbound survey demographics. + +These are the historical composed ASEC binding's exact recorded requirements. +That stage still refuses their canonical writes; a later source-qualified +producer must meet its own reviewed contract. Importing these declarations +loads no source reader, population operator, model, graph or country engine. +""" + +#: Demographic columns a later calibration wants on both arms and which this +#: stage deliberately does not bind: each names the exact source column the +#: reviewed ASEC mapping needs, the mapping itself, and what the native ACS arm +#: uses instead. Nothing is imputed, aged or defaulted. +#: +#: Whether the prepared arm carries the source column is an **observed property +#: of the population in hand**, recorded per run rather than asserted: the +#: invented fixture parent has neither column, and the genuine prepared arm has +#: both. Presence is not a licence — binding either column still needs a +#: reviewed cross-arm mapping decision that this stage may not make — so what +#: fails closed here is an attempted **write** of one of these columns +#: (``ASEC_DEMOGRAPHIC_COLUMN_BOUND``), not the source column's presence. Keying +#: the refusal on presence would have refused every genuine run while proving +#: nothing about what this stage binds. +#: Each entry is (column, required source columns, diagnostic-only source +#: columns, the explicit source contract that would supply it, the mappings +#: deliberately not adopted as proof, the caveats on reading presence as +#: observation, the ACS arm's own observed mapping). +#: The required columns are the ones an *explicit* binding needs, which is not +#: the same as the columns some existing mapping happens to read: the root +#: adjudication of 2026-09-06 refused ``P_SEQ == 1`` as semantic proof of +#: headship and required sex to carry its allocation provenance, so those +#: mappings are recorded here as not adopted rather than as the requirement. +_UNBOUND_DEMOGRAPHICS = ( + ( + "is_female", + ("A_SEX", "AXSEX"), + (), + "asec_demographic_source: A_SEX printed codes 1 = Male and 2 = Female, " + "admitted only with AXSEX printed codes 0 = No change or 4 = Allocated; " + "any other token leaves the person unbound", + ( + ( + "cps_carried.derive_us_cps_carried_inputs (is_female = A_SEX == 2)", + "maps every token other than 2 onto male, so an unprinted code " + "would be admitted as male, and it reads no allocation flag", + ), + ), + ( + "A_SEX reaches the prepared person roster as a carried source " + "column, but its presence attests the column, not that every " + "delivered token is one of the two printed codes", + ), + "SEX == 2 on the native ACS arm, whose pinned dictionary prints the " + "same 1 = Male / 2 = Female convention", + ), + ( + "is_household_head", + ("A_EXPRRP",), + ("P_SEQ",), + "asec_demographic_source: A_EXPRRP printed codes 1 = Reference person " + "with relatives and 2 = Reference person without relatives, with " + "exactly one such person per household and every household relationship " + "token inside the printed named codes", + ( + ( + "relationship_inputs (is_household_head = P_SEQ == 1)", + "P_SEQ's dictionary entry labels no code as reference person, " + "and the ordering statement behind it is expressly limited to " + "the ASCII file while this arm is restored from the CSV member", + ), + ( + "A_FAMREL == 1", + "a family relationship scoped to primary-family membership, " + "which a subfamily reference person does not carry; household " + "and family roles stay separate", + ), + ( + "asec_pool relationship recode (A_LINENO == 1)", + "a separately assigned Basic-CPS roster line number with no " + "source guarantee of agreement with any ASEC sequence", + ), + ), + ( + "asec_pool._with_relationship_recode derives A_EXPRRP from " + "A_LINENO == 1 whenever the locked input omits it, so the column's " + "presence on a prepared arm does not certify an observed source " + "value for that cohort", + "the native ACS arm's A_EXPRRP is likewise derived by acs_pums " + "from RELSHIPP, and its group-quarters households carry the " + "nonrelative code 14 rather than any reference-person code", + ), + "RELSHIPP == 20 observed on the native ACS arm's housing units only; " + "RELSHIPP 37 and 38 are the group-quarters populations and carry no " + "observed reference person", + ), +) +#: The population columns this stage records as unbound, and therefore may not +#: own on any node. Public because the measurement module's own declarations are +#: checked against it. +UNBOUND_DEMOGRAPHIC_COLUMNS = tuple(column for column, *_rest in _UNBOUND_DEMOGRAPHICS) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py new file mode 100644 index 000000000..c689b746f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py @@ -0,0 +1,254 @@ +"""Admit a checked financial graph result under its original sampling budget. + +This narrow IMPORTANCE-to-IMPORTANCE binding changes only the seven current +financial leaves authenticated by the actual twenty-node runner. It neither +changes the original budget's initial population nor admits arbitrary Frames. + +Age composition still needs an explicit dependency on financial ATTACH and a +checked binding to the actual receiving graph's Population before calibration +admission. This module does not supply those graph edges or a new age runner. +""" + +from __future__ import annotations + +import json +import weakref +from dataclasses import dataclass + +import numpy as np + +from microcosm.graph import ArtifactType + +PROTOCOL = "microcosm.us.sampling-origin-financial-successor.v1" +FINANCIAL_SUCCESSOR_TYPE = ArtifactType( + "microcosm.us.sampling_origin_financial_successor", 1 +) +MAX_PAYLOAD_BYTES = 256 * 1024 +_ISSUED = {} + + +class SurveyFinancialSuccessorError(ValueError): + """A financial result lacks its exact original budget/run ancestry.""" + + +def _require(condition, reason): + if not condition: + raise SurveyFinancialSuccessorError("SURVEY_FINANCIAL_SUCCESSOR_" + reason) + + +def _entry(value): + entry = _ISSUED.get(id(value)) + _require( + type(value) is SamplingOriginFinancialSuccessor + and entry is not None + and entry[0]() is value + and type(value.payload) is bytes + and value.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + + +@dataclass(frozen=True) +class _State: + budget: object + budget_entry: tuple + financial_run: object + run_entry: tuple + previous: object + current: object + previous_identity: tuple + current_identity: tuple + + +def _document(state, budget_view, run_view): + # Local imports avoid a budget -> financial owner -> runner -> host cycle. + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budget + + run, prefix = state.financial_run, state.financial_run.prefix + original = state.budget_entry[2] + _require( + budget._entry(state.budget, budget.SamplingOriginBudget) is state.budget_entry + and runner._run_entry(run) is state.run_entry + and original.preparation is prefix.preparation + and original.allocated is prefix.allocated_population + and original.expanded is prefix.clone_population + and original.geography_config is prefix.geography_config + and budget_view.preparation is prefix.preparation + and budget_view.allocated_population is prefix.allocated_population + and budget_view.initial_population is state.previous is prefix.clone_population + and budget_view.payload == state.budget_entry[1] + and budget_view.digest == budget._sha(state.budget_entry[1]) + and run_view.population is state.current is run.financial_population + and run_view.payload == state.run_entry[1] + and run_view.digest == budget._sha(state.run_entry[1]), + "ORIGINAL_BUDGET_RUN_ANCESTRY", + ) + previous, current = state.previous, state.current + _require( + previous.version == current.version == runner.atomic.clone.COMBINED_CLONE_NODE + and current.mass_ledger == previous.mass_ledger + and current.weight_kind == previous.weight_kind + and np.array_equal( + current.frame.weights_for("household").values, + previous.frame.weights_for("household").values, + ), + "UNCHANGED_SAMPLING_MASS", + ) + budget.graph._check_design_anchors(current, previous.design_weights["household"]) + owners = { + **previous.owners, + **{ + ("person", column): runner.financial.ATTACH_NODE + for column in runner.values.OUTPUTS + }, + } + budget.graph._check_population_state( + current, + version=previous.version, + owners=owners, + kind=budget.WeightKind.IMPORTANCE, + ledger=previous.mass_ledger, + ) + # The checked run freshly compares the complete result to source-derived + # current values and its pinned raw draws; no caller column table is used. + return budget._json( + { + "protocol": PROTOCOL, + "budget_sha256": budget._sha(state.budget_entry[1]), + "financial_run_sha256": budget._sha(state.run_entry[1]), + "financial_run": json.loads(state.run_entry[1]), + "previous_version": previous.version, + "current_version": current.version, + "previous_frame_sha256": budget.source._frame_identity(previous.frame), + "current_frame_sha256": budget.source._frame_identity(current.frame), + "owned_columns": list(runner.values.OUTPUTS), + "sampling_bounds_changed": False, + "source_admission_issued": False, + "release_eligible": False, + } + ) + + +def _pure_state(state): + """Final retained-owner checks; never opens an artifact or source file.""" + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budget + + _require( + budget._entry(state.budget, budget.SamplingOriginBudget) is state.budget_entry, + "FINAL_BUDGET_ISSUANCE", + ) + runner._pure_run(state.financial_run, state.run_entry) + _require( + budget._population_identity(state.previous) == state.previous_identity + and budget._population_identity(state.current) == state.current_identity, + "FINAL_POPULATION_SEAL", + ) + _require(budget._live() == budget._LIVE, "FINAL_PRODUCER_SEAL") + + +def _final_state(state): + from . import survey_origin_budget as budget + + # Complete the last budget/support I/O before the financial owner's pure + # seal, so that borrow cannot invalidate the already-checked financial run. + budget._final_budget_state(state.budget_entry[2]) + _pure_state(state) + + +@dataclass(frozen=True) +class SamplingOriginFinancialSuccessor: + payload: bytes + + def __post_init__(self): + raise SurveyFinancialSuccessorError("NO_PUBLIC_FINANCIAL_SUCCESSOR_CONSTRUCTOR") + + def checked_view(self): + from . import graph_atomic_survey_financial as runner + + entry = _entry(self) + state = entry[2] + _pure_state(state) + budget_view = state.budget.checked_view() + run_view = runner.check_atomic_survey_financial_run(state.financial_run) + _require(_document(state, budget_view, run_view) == entry[1], "RECONSTRUCTION") + _final_state(state) + _require(_entry(self) is entry, "FINAL_ISSUANCE") + # Do not expose a detached view to the final support-borrow callback. + # All owner/file I/O and retained-state seals are complete above. + return CheckedSamplingOriginFinancialSuccessor( + entry[1], + runner.codec.sha(entry[1]), + state.budget, + state.financial_run, + state.previous, + state.current, + ) + + def to_bytes(self): + return self.checked_view().payload + + +@dataclass(frozen=True) +class CheckedSamplingOriginFinancialSuccessor: + """Values borrowed from a checked handle, never decoded authority.""" + + payload: bytes + digest: str + budget: object + financial_run: object + previous: object + current: object + + +def admit_survey_financial_population(budget, *, financial_run, candidate=None): + """Admit only a live completed graph run under its own original budget.""" + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budgets + + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= MAX_PAYLOAD_BYTES), + "CANDIDATE_TYPE_OR_BOUND", + ) + # An unissued descriptive dataclass must fail before any owner/file borrow. + run_entry = runner._run_entry(financial_run) + budget_entry = budgets._entry(budget, budgets.SamplingOriginBudget) + budget_view = budget.checked_view() + run_view = runner.check_atomic_survey_financial_run(financial_run) + state = _State( + budget, + budget_entry, + financial_run, + run_entry, + budget_entry[2].expanded, + financial_run.financial_population, + budgets._population_identity(budget_entry[2].expanded), + budgets._population_identity(financial_run.financial_population), + ) + payload = _document(state, budget_view, run_view) + _require(len(payload) <= MAX_PAYLOAD_BYTES, "PAYLOAD_BOUND") + _require(candidate is None or candidate == payload, "CANDIDATE_RECONSTRUCTION") + result = object.__new__(SamplingOriginFinancialSuccessor) + object.__setattr__(result, "payload", payload) + identifier = id(result) + + def forget(reference): + entry = _ISSUED.get(identifier) + if entry is not None and entry[0] is reference: + _ISSUED.pop(identifier, None) + + reference = weakref.ref(result, forget) + _ISSUED[identifier] = (reference, payload, state) + entry = _entry(result) + _final_state(state) + _require(_entry(result) is entry, "FINAL_ISSUANCE") + return result + + +def verify_survey_financial_successor(binding): + _entry(binding) + binding.checked_view() + return binding diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py new file mode 100644 index 000000000..804f2f6ab --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py @@ -0,0 +1,101 @@ +"""Pure observed-age alias; no source, population or release authority. + +The caller authenticates the integer A_AGE observations. This rule changes no +observation date, top code or raw value and reads no source/weight/domain label. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +RULE = "microcosm.us.observed-age-normalization.v1" +AGE_CONVENTION = "observed_interview_age_completed_years" +# One channel's person rows at a time: _normalized_source_copy normalizes the +# ACS and ASEC native frames separately, so the ceiling is met by the larger +# channel. A full-source ACS channel carries 3,422,888 persons, and this is +# four times that, rounded up to the next whole million. +# See docs/us-native-row-ceilings.md. +MAX_ROWS = 14_000_000 +MAX_EXACT_FLOAT64_INTEGER = 2**53 + + +def rule_document() -> dict: + return { + "protocol": RULE, + "input": "A_AGE", + "output": "age", + "dtype": "float64", + "relation": "numeric_identity", + "age_convention": AGE_CONVENTION, + "existing_nonmissing": "must_equal_observed_age", + "temporal_adjustment": False, + "top_code_replacement": False, + } + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_OBSERVED_AGE_" + reason) + + +def normalize_observed_age( + raw_age: pd.Series, incumbent_age: pd.Series | None = None +) -> pd.Series: + """Return a defensive common age, accepting only verified numeric identity. + + Missing common cells may acquire the already observed value; missing or + malformed raw ages refuse. The float bound is a representation limit, not + a scientifically permitted maximum age. Measurement keeps its own range. + """ + _require(isinstance(raw_age, pd.Series), "RAW_SERIES") + _require(0 < len(raw_age) <= MAX_ROWS, "ROW_BOUND") + _require( + pd.api.types.is_integer_dtype(raw_age.dtype) + and not pd.api.types.is_bool_dtype(raw_age.dtype) + and not raw_age.isna().any(), + "RAW_INTEGER_REQUIRED", + ) + _require( + ((raw_age >= 0) & (raw_age <= MAX_EXACT_FLOAT64_INTEGER)).all(), + "RAW_REPRESENTATION", + ) + values = raw_age.to_numpy(dtype=np.float64, copy=True) + if incumbent_age is not None: + _require(isinstance(incumbent_age, pd.Series), "INCUMBENT_SERIES") + _require( + incumbent_age.index.identical(raw_age.index) + and len(incumbent_age) == len(raw_age), + "ORDERED_INDEX", + ) + present = incumbent_age.notna().to_numpy(dtype=bool) + if present.any(): + _require( + pd.api.types.is_numeric_dtype(incumbent_age.dtype) + and not pd.api.types.is_bool_dtype(incumbent_age.dtype) + and not pd.api.types.is_complex_dtype(incumbent_age.dtype), + "INCUMBENT_NUMERIC_REQUIRED", + ) + known = incumbent_age[present] + _require( + ((known >= 0) & (known <= MAX_EXACT_FLOAT64_INTEGER)).all(), + "INCUMBENT_REPRESENTATION", + ) + _require((known == raw_age[present]).all(), "INCUMBENT_CONFLICT") + if pd.api.types.is_integer_dtype(incumbent_age.dtype): + _require( + np.array_equal( + known.to_numpy(dtype=np.int64), + raw_age[present].to_numpy(dtype=np.int64), + ), + "INCUMBENT_CONFLICT", + ) + incumbent = incumbent_age.to_numpy(dtype=np.float64, na_value=np.nan) + _require( + np.isfinite(incumbent[present]).all() + and np.array_equal(incumbent[present], values[present]), + "INCUMBENT_CONFLICT", + ) + # Preserve the original float bits of every known incumbent cell. + values[present] = incumbent[present] + return pd.Series(values, index=raw_age.index.copy(deep=True), name="age") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py new file mode 100644 index 000000000..20f364fb1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py @@ -0,0 +1,1394 @@ +"""Source-qualified, sampling-aware budgets for the complete first survey clone. + +Original publisher d is never replaced. The sampling reference b=d/p and +incoming a=s*b are different quantities. These development coefficients are +provisional; this module grants neither graph execution nor release authority. +Decoded bytes cannot issue a budget or a weight-only successor. +An optional immutable geography recipe independently reconstructs the complete +postclone geography Frame; the retained allocation remains the raw source allocation. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import sys +import weakref +from dataclasses import asdict, dataclass +from fractions import Fraction +from pathlib import Path +from types import FunctionType + +import numpy as np +import pandas as pd + +from microcosm.calibrate import group_bounds +from microcosm.frame import Frame, WeightKind +from microcosm.graph import ( + ArtifactType, + KernelResult, + Node, + StructuralDelta, + WeightTransition, +) +from microcosm.graph.population import ( + Population, + _mass_record, + _storage_parts, + storage_equal, +) + +from . import graph_combined_clone as clone +from . import graph_survey_population as graph +from . import survey_atomic_geography as geography +from . import survey_financial_successor as financial_successor +from . import survey_population_preparation as source +from .graph_sources import frame_column_declarations +from .support_provenance import support_clone_index_column, support_source_id_column + +BUDGET_PROTOCOL = "microcosm.us.sampling-origin-budget.v1" +SUCCESSOR_PROTOCOL = "microcosm.us.sampling-origin-weight-only-successor.v1" +BUDGET_TYPE = ArtifactType("microcosm.us.sampling_origin_budget", 1) +SUCCESSOR_TYPE = ArtifactType("microcosm.us.sampling_origin_weight_only_successor", 1) +MAX_PAYLOAD_BYTES = 64 * 1024**2 +# The budget document carries one origin record per selected household plus +# two header entries per clone row, measured at 742 bytes per household at +# full-source widths, so a full-source document is 1,177,832,992 bytes and a +# single 64 MiB accumulation admitted 90,443 households (5.7% of source). +# MAX_PAYLOAD_BYTES stays what one accumulation may hold and what one piece +# may be; the total a process will materialise for the document is 64 +# accumulations, the same multiple survey_population_preparation uses. See +# docs/us-native-byte-transports.md. +MAX_ROSTER_BYTES = 64 * MAX_PAYLOAD_BYTES +# One group per allocation instruction, and allocation_instructions requires +# one instruction per selected household, so a full-source budget has the +# 1,587,376 households the catalogues supply. Four times that, rounded up to +# the next whole million. See docs/us-native-row-ceilings.md. +MAX_GROUPS = 7_000_000 +MAX_SCALAR_CHARS = 4096 +PRESCRIPTION = ( + "development-provisional-8b-4a-v1", + "allocation=float(exact d*s/p)", + "reference=float(exact d/p)", + "bounds=min(8.0*reference,4.0*actual_incoming)", +) +_ISSUED = {} + + +class SurveyOriginBudgetError(ValueError): + """Static refusal without source observations or identifiers.""" + + +def _require(condition, reason): + if not condition: + raise SurveyOriginBudgetError(reason) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _json(value): + return graph._bounded_json(value, MAX_PAYLOAD_BYTES) + + +def _pair(value): + _require(type(value) is Fraction, "EXACT_FRACTION_REQUIRED") + _require( + value.numerator.bit_length() <= 128 and value.denominator.bit_length() <= 128, + "FRACTION_BOUND", + ) + return [value.numerator, value.denominator] + + +def _reference(d, p, s, actual_incoming): + """Pure exact arithmetic, not source authority; no float-product shortcut.""" + for value in (d, p, s): + _pair(value) + _require( + d >= 0 and 0 < p <= 1 and s in (Fraction(1), Fraction(1, 2)), "REFERENCE_DOMAIN" + ) + _require(type(actual_incoming) is float, "INCOMING_FLOAT64_REQUIRED") + b, a = d / p, d * s / p + expected_a, reference = float(a), float(b) + _require( + math.isfinite(actual_incoming) + and actual_incoming.hex() == expected_a.hex() + and math.isfinite(reference), + "ALLOCATION_OPERATION_ORDER", + ) + design_bound, incoming_bound = 8.0 * reference, 4.0 * actual_incoming + _require( + all(math.isfinite(v) and v >= 0 for v in (design_bound, incoming_bound)), + "REFERENCE_OVERFLOW", + ) + upper = min(design_bound, incoming_bound) + _require(upper == incoming_bound, "CURRENT_SHARE_BOUND_RELATION") + return { + "d": _pair(d), + "p": _pair(p), + "s": _pair(s), + "b": _pair(b), + "a": _pair(a), + "original_float64_hex": float(d).hex(), + "reference_float64_hex": reference.hex(), + "actual_incoming_float64_hex": actual_incoming.hex(), + "design_bound_float64_hex": design_bound.hex(), + "incoming_bound_float64_hex": incoming_bound.hex(), + "upper_float64_hex": upper.hex(), + } + + +def _modules(): + modules = ( + sys.modules[__name__], + graph, + clone, + group_bounds, + sys.modules[Population.__module__], + sys.modules[Frame.__module__], + sys.modules[frame_column_declarations.__module__], + sys.modules[support_source_id_column.__module__], + sys.modules[clone.clone_us_frame_for_puf_support.__module__], + geography, + geography.replay, + financial_successor, + geography.blocks, + geography.atomic, + geography.atomic_graph, + geography.composition, + geography.observed, + geography.observed_graph, + geography.observed.demographics, + geography.observed.demographics.demographic, + geography.observed.demographics.household, + geography.observed.demographics.source_csv_builtin, + sys.modules[geography.observed.demographics.demographic._snapshot.__module__], + sys.modules[geography.atomic.keyed_uniform.__module__], + sys.modules[geography.canonical_json.__module__], + sys.modules[geography.compile_graph.__module__], + sys.modules[geography._expand_declared_payload.__module__], + ) + return tuple({module.__name__: module for module in modules}.values()) + + +def _live(): + result = {} + for module in _modules(): + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[module.__name__, name] = source._function_seal(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + result[module.__name__, name] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[module.__name__, name, method] = source._function_seal( + function + ) + result["contract"] = ( + BUDGET_PROTOCOL, + SUCCESSOR_PROTOCOL, + (BUDGET_TYPE.name, BUDGET_TYPE.schema_version), + (SUCCESSOR_TYPE.name, SUCCESSOR_TYPE.schema_version), + MAX_PAYLOAD_BYTES, + MAX_ROSTER_BYTES, + MAX_GROUPS, + MAX_SCALAR_CHARS, + PRESCRIPTION, + np.__version__, + np.ndarray, + np.float64, + ) + result["geography_contract"] = source._runtime_marker( + ( + geography.PROTOCOL, + geography.MAX_SUPPORT_BYTES, + geography.MAX_SUPPORT_EXPANDED_BYTES, + geography.MAX_SUPPORT_MEMBERS, + geography.RAW_BYTES_MAX_BYTES, + geography.blocks.SYSTEM, + geography.blocks.SOURCE, + tuple(sorted(geography.blocks._STATES)), + tuple(sorted(geography.atomic.RELATIONS)), + geography.atomic._U53, + geography.observed.PROTOCOL, + geography.observed.COLUMNS, + geography.composition.ASSIGNMENT_IDENTITY, + geography.observed.MAX_HOUSEHOLDS, + geography.observed.MAX_RECEIPT_BYTES, + geography.observed_graph.NODE, + geography.observed_graph.REF, + geography.observed_graph.PHASE, + geography.observed_graph.OUTPUT_COLUMNS, + geography.observed.demographics.PROTOCOL, + geography.observed.demographics.STATE_COLUMNS, + geography.observed.demographics.STATE_CONTRACT, + ( + geography.atomic_graph.ATOMIC_SUPPORT_TYPE.name, + geography.atomic_graph.ATOMIC_SUPPORT_TYPE.schema_version, + ), + ( + geography.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE.name, + geography.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE.schema_version, + ), + geography.atomic_graph._DEPENDENCIES, + tuple( + (kernel.ref, asdict(kernel.capabilities)) + for kernel in ( + geography.atomic_graph.AtomicSupportImportKernel, + geography.atomic_graph.AtomicAssignKernel, + geography.atomic_graph.AtomicDeriveKernel, + geography.atomic_graph.AtomicGeographyGateKernel, + ) + ), + ) + ) + result["financial_successor_contract"] = ( + financial_successor.PROTOCOL, + financial_successor.FINANCIAL_SUCCESSOR_TYPE.name, + financial_successor.FINANCIAL_SUCCESSOR_TYPE.schema_version, + financial_successor.MAX_PAYLOAD_BYTES, + ) + return result + + +def _code_bytes(): + return {m.__name__: _sha(Path(m.__file__).read_bytes()) for m in _modules()} + + +def _producer(): + _require(_live() == _LIVE and _code_bytes() == _BYTES, "PRODUCER_CHANGED") + return _BYTES + + +def _population_identity(population): + _require(type(population) is Population, "LIVE_POPULATION_REQUIRED") + return ( + source._frame_identity(population.frame), + _physical_nonweight_identity(population.frame), + population.version, + tuple(sorted(population.owners.items())), + tuple(population.weight_kind.items()), + _json([asdict(record) for record in population.mass_ledger]), + tuple( + (e, str(v.dtype), v.shape, v.tobytes()) + for e, v in population.design_weights.items() + ), + ) + + +def _physical_nonweight_identity(frame): + """Detach actual storage, including masked numeric slots, one column at a time. + + Axis/schema/dtype identity is bound separately by the full Frame identity. + This is an in-process population seal, not a cache canonicalization boundary. + """ + digest = hashlib.sha256() + for series in ( + *( + frame.table(entity)[column] + for entity in frame.entities + for column in frame.table(entity) + ), + frame.strata, + ): + # Whole series: the slice selects every row in order, byte for byte. + for part in _storage_parts(series, slice(None)): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + return digest.hexdigest() + + +def _config_payload(config): + return ( + None + if config is None + else geography.AtomicSurveyReconstruction.to_bytes(config) + ) + + +def _geography_binding(value, config_payload, preparation_payload): + """Seal fresh reconstruction values; decoded receipts never issue authority.""" + _require( + type(value) is geography.AtomicSurveyGeographyReconstruction, + "GEOGRAPHY_RECONSTRUCTION_TYPE", + ) + document = json.loads(value.receipt) + population_sha256 = geography._population_stamp(value.population) + stages = {stage.node.id: stage for stage in value.stages} + _require( + value.observed_population is stages[geography.observed_graph.NODE].population + and value.expanded_population + is stages[clone.COMBINED_CLONE_CLAIM_NODE].population, + "GEOGRAPHY_STAGE_BINDINGS", + ) + validation = stages["geography.gate"].receipt + _require( + stages["geography.gate"].artifacts == (("validation", validation),), + "GEOGRAPHY_VALIDATION_ARTIFACT", + ) + _require( + value.config_sha256 == _sha(config_payload) + and document["protocol"] == geography.PROTOCOL + and document["preparation_sha256"] == _sha(preparation_payload) + and document["config_sha256"] == value.config_sha256 + and document["frame_sha256"] == source._frame_identity(value.population.frame) + and document["population_sha256"] == population_sha256 + and document["observed_population_sha256"] + == geography._population_stamp(value.observed_population) + and document["expanded_population_sha256"] + == geography._population_stamp(value.expanded_population) + and document["validation_receipt_sha256"] == _sha(validation) + and document["assignment_identity"] + == list(geography.composition.ASSIGNMENT_IDENTITY) + and document["projection_receipt_sha256"] == _sha(value.projection_receipt) + and document["definition_sha256"] == _sha(value.definition) + and document["support_sha256"] == _sha(value.support_payload) + and document["publisher_provenance_established"] is False + and document["source_admission_issued"] is False + and document["population_admission_issued"] is False + and document["release_eligible"] is False, + "GEOGRAPHY_RECONSTRUCTION_SEAL", + ) + # The full receipt above seals this detached object exactly. Frame-store + # replay can canonicalize storage beneath masked nulls, so its physical + # stamp cannot identify an independently reconstructed object across runs. + physical_fields = ( + "population_sha256", + "observed_population_sha256", + "expanded_population_sha256", + ) + semantic_receipt = { + key: item for key, item in document.items() if key not in physical_fields + } + population = value.population + semantic_population = { + "frame_sha256": document["frame_sha256"], + "version": population.version, + "owners": sorted(population.owners.items()), + "weight_kind": [ + (entity, kind.value) for entity, kind in population.weight_kind.items() + ], + "mass_ledger": [asdict(record) for record in population.mass_ledger], + "design_weights": [ + (entity, str(values.dtype), values.shape, values.tobytes().hex()) + for entity, values in population.design_weights.items() + ], + } + return _json( + { + "config_sha256": value.config_sha256, + "reconstruction_semantic_sha256": _sha(_json(semantic_receipt)), + "projection_receipt_sha256": _sha(value.projection_receipt), + "definition_sha256": _sha(value.definition), + "support_sha256": _sha(value.support_payload), + "postclone_geography_population_semantic_sha256": _sha( + _json(semantic_population) + ), + "validation_receipt": json.loads(validation), + "identity_scope": { + "reconstruction_omitted_fields": list(physical_fields), + "population_fields": list(semantic_population), + "null_backing": "retained_in_in_process_physical_seals_only", + }, + "publisher_provenance_established": False, + } + ) + + +def _initial( + view, + allocated, + expanded, + *, + preparation=None, + geography_config=None, + _with_geography_binding=False, +): + _require(type(_with_geography_binding) is bool, "GEOGRAPHY_BINDING_FLAG") + _require( + type(allocated) is Population and type(expanded) is Population, + "LIVE_POPULATION_REQUIRED", + ) + instructions = graph.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + _require(0 < len(instructions) <= MAX_GROUPS, "GROUP_COUNT_BOUND") + design = view.frame.weights_for("household").values.copy() + _weights, _context, allocation, receipt, expected = graph._allocation_output( + view.frame, view.context, instructions, _sha(view.payload) + ) + columns = frame_column_declarations(view.frame) + nodes = graph.survey_population_nodes( + columns, + preparation_sha256=_sha(view.payload), + fraction=view.selection_plan.fraction, + seed=view.selection_plan.seed, + ) + graph._same_frame(expected, allocated.frame) + graph._check_design_anchors(allocated, design) + allocation_ledger = ( + _mass_record( + view.frame, expected, nodes[1], KernelResult(receipt=receipt), "declared" + ), + ) + cells = tuple((e, str(c)) for e in view.frame.entities for c in view.frame.table(e)) + graph._check_population_state( + allocated, + version=graph.ALLOCATION_NODE, + owners=dict.fromkeys(cells, graph.ALLOCATION_NODE), + kind=WeightKind.IMPORTANCE, + ledger=allocation_ledger, + ) + geography_binding = None + if geography_config is not None: + config_payload = geography.AtomicSurveyReconstruction.to_bytes(geography_config) + reconstructed = geography.reconstruct_atomic_survey_geography( + preparation, allocated, geography_config + ) + geography_binding = _geography_binding( + reconstructed, config_payload, view.payload + ) + # Verify the exact clone before assignment, then the complete geography + # population. Geography additions never relax the clone equality rule. + graph._verify_cloned_frame( + reconstructed.observed_population.frame, + reconstructed.expanded_population.frame, + design, + ) + geography.replay.same_replayed_population(reconstructed.population, expanded) + else: + clone_nodes = clone.us_combined_survey_clone_nodes( + columns, base=graph.ALLOCATION_NODE, source_channels=("acs", "asec") + ) + copied_design = graph._verify_cloned_frame(expected, expanded.frame, design) + graph._check_design_anchors(expanded, copied_design) + owners = { + (entity, str(column)): clone.COMBINED_CLONE_NODE + for entity in expected.entities + for column in expected.table(entity) + } + owners.update( + { + (o.entity, o.column): clone.COMBINED_CLONE_CLAIM_NODE + for o in clone_nodes[1].outputs + } + ) + graph._check_population_state( + expanded, + version=clone.COMBINED_CLONE_NODE, + owners=owners, + kind=WeightKind.IMPORTANCE, + ledger=( + *allocation_ledger, + _mass_record( + expected, expanded.frame, clone_nodes[0], KernelResult(), "conserve" + ), + ), + ) + if geography_config is not None: + _require( + geography.AtomicSurveyReconstruction.to_bytes(geography_config) + == config_payload + and _geography_binding(reconstructed, config_payload, view.payload) + == geography_binding, + "FINAL_GEOGRAPHY_RECONSTRUCTION_SEAL", + ) + # Current survey financial and diagnostic hosts consume the historical pair. + # Only the budget owner explicitly requests its additional reconstruction seal. + if _with_geography_binding: + return instructions, allocation, geography_binding + return instructions, allocation + + +def _document( + view, + allocated, + expanded, + instructions, + allocation, + producer, + geography_binding=None, +): + before, after = ( + allocated.frame.table("household"), + expanded.frame.table("household"), + ) + source_column = support_source_id_column("household") + role_column = support_clone_index_column("household") + positions = {int(value): i for i, value in enumerate(before[source_column])} + _require(len(positions) == len(instructions), "ROOT_SOURCE_ID_COLLISION") + members = [[] for _ in instructions] + ids, groups = [], [] + for row in after[["household_id", source_column, role_column]].itertuples( + index=False, name=None + ): + hh_id, source_id, role = (int(value) for value in row) + _require(source_id in positions and role in (0, 1), "CLONE_ORIGIN_ROLE") + group = positions[source_id] + members[group].append((hh_id, role)) + ids.append(hh_id) + groups.append(group) + _require( + len(ids) == 2 * len(instructions) and len(set(ids)) == len(ids), + "CLONE_CARDINALITY", + ) + incoming = allocated.frame.weights_for("household").values + clone_weights = expanded.frame.weights_for("household").values + by_id = {value: i for i, value in enumerate(ids)} + bounds = [] + selected_by_key = {row.key: row for row in view.selection_plan.selected} + + def records(): + for i, (instruction, group_members) in enumerate( + zip(instructions, members, strict=True) + ): + _require( + len(group_members) == 2 and {m[1] for m in group_members} == {0, 1}, + "COMPLETE_CLONE_ROLES", + ) + actual = float(incoming[i]) + _require( + math.fsum(float(clone_weights[by_id[m[0]]]) for m in group_members) + == actual, + "CLONE_INCOMING_CONSERVATION", + ) + reference = _reference( + instruction.original_anchor, + instruction.inclusion_probability, + instruction.share, + actual, + ) + bounds.append(float.fromhex(reference["upper_float64_hex"])) + key = instruction.key + _require( + type(key.native_id) is str and len(key.native_id) <= MAX_SCALAR_CHARS, + "NATIVE_KEY_BOUND", + ) + yield { + "source": key.source.value, + "source_year": key.source_year, + "survey_year": key.survey_year, + "raw_native_id": key.native_id, + "selected_receiving_household_id": instruction.selected_receiving_household_id, + "combined_household_id": instruction.household_id, + "statistical_unit": selected_by_key[instruction.key].statistical_unit, + "original_design_float64_bytes": allocated.design_weights["household"][ + i : i + 1 + ] + .tobytes() + .hex(), + **reference, + "members": [[hh_id, role] for hh_id, role in group_members], + "incoming_clone_float64_bytes": [ + clone_weights[by_id[hh_id] : by_id[hh_id] + 1].tobytes().hex() + for hh_id, _role in group_members + ], + } + + header = { + "protocol": BUDGET_PROTOCOL, + "prescription": PRESCRIPTION, + "coefficient_status": "provisional_development_not_scientific_approval", + "current_share_simplification": "min(8b,4a)=4a; 8b is redundant for current shares", + "preparation_sha256": _sha(view.payload), + "source_producer": view.receipt["producer"], + "source_native": view.receipt["native"], + "source_catalogues": view.receipt["catalogues"], + # One row per supplied household: digested from the stream, never + # held whole. The digest equals _sha(_json(...)) wherever _json admits. + "selection_sha256": graph._json_sha256( + view.receipt["selection"], maximum=MAX_ROSTER_BYTES + ), + "allocation_sha256": _sha(allocation), + "producer": producer, + "allocated_frame_sha256": source._frame_identity(allocated.frame), + "clone_frame_sha256": source._frame_identity(expanded.frame), + "household_ids": ids, + "group_indices": groups, + "group_count": len(instructions), + "release_eligible": False, + } + if geography_binding is not None: + header["atomic_geography"] = json.loads(geography_binding) + # Stream one bounded origin record at a time; never materialize an unbounded + # list of origin dictionaries before the transport cap is checked. The + # pieces accumulate in segments no larger than MAX_PAYLOAD_BYTES and are + # joined once under MAX_ROSTER_BYTES; the bytes are the same ones a single + # accumulation produced. TRANSPORT_LIMIT keeps its code on the condition a + # segmented stream can still reach: one piece larger than one accumulation. + segments, payload, total = [], bytearray(), 0 + + def append(piece): + nonlocal payload, total + if payload and len(payload) + len(piece) > MAX_PAYLOAD_BYTES: + segments.append(bytes(payload)) + payload = bytearray() + _require(len(piece) <= MAX_PAYLOAD_BYTES - len(payload), "TRANSPORT_LIMIT") + _require(total + len(piece) <= MAX_ROSTER_BYTES, "TRANSPORT_ROSTER_LIMIT") + payload.extend(piece) + total += len(piece) + + def append_list(values): + # One entry per clone row; streamed element by element so no single + # piece approaches the accumulation ceiling. Byte-identical to + # _json(values). + append(b"[") + for position, value in enumerate(values): + if position: + append(b",") + append(_json(value)) + append(b"]") + + append(b"{") + keys = sorted((*header, "origins")) + for index, key in enumerate(keys): + if index: + append(b",") + append(_json(key) + b":") + if key == "origins": + append(b"[") + for position, record in enumerate(records()): + if position: + append(b",") + append(_json(record)) + append(b"]") + elif key in ("household_ids", "group_indices"): + append_list(header[key]) + else: + append(_json(header[key])) + append(b"}") + segments.append(bytes(payload)) + document = b"".join(segments) + _require(len(document) == total <= MAX_ROSTER_BYTES, "TRANSPORT_ROSTER_LIMIT") + constraint = group_bounds.GroupedUpperBounds(ids, groups, bounds) + constraint.check(clone_weights, positive=False) + return document, constraint + + +@dataclass(frozen=True) +class _BudgetState: + preparation: object + preparation_entry: tuple + initial_view: source.CheckedSurveyPopulationView + allocated: Population + expanded: Population + preparation_payload: bytes + allocated_identity: tuple + expanded_identity: tuple + geography_config: geography.AtomicSurveyReconstruction | None + geography_config_payload: bytes | None + geography_binding: bytes | None + + +def _preparation_entry(preparation, payload, expected=None): + """Check retained issuance identity without repeating source-file reads.""" + entry = source._ISSUED.get(id(preparation)) + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation + and entry is not None + and (expected is None or entry is expected) + and entry[0]() is preparation + and type(preparation.payload) is bytes + and preparation.payload == entry[1] == payload, + "FINAL_PREPARATION_ISSUANCE", + ) + return entry + + +def _final_budget_state(state): + """Seal retained owners after helper/file I/O, with no recursive borrow. + + Optional support bytes are checked before the source owner's pure final + check. That pure check hashes retained Frames without rereading survey + files, issuing replacement capsules or granting decoded authority. + """ + _producer() + _require( + _config_payload(state.geography_config) == state.geography_config_payload, + "FINAL_GEOGRAPHY_CONFIG_SEAL", + ) + if state.geography_config is not None: + # Finish support I/O before the same pure owner/Population final seals. + # Re-reading exact pinned bytes grants no publisher provenance. + geography._read_support(state.geography_config) + _pure_budget_state(state) + + +def _pure_budget_state(state): + """Check retained source/Population state without a new file borrow.""" + entry = _preparation_entry( + state.preparation, state.preparation_payload, state.preparation_entry + ) + source._pure_final(entry[2]) + _require( + _population_identity(state.allocated) == state.allocated_identity + and _population_identity(state.expanded) == state.expanded_identity, + "FINAL_POPULATION_SEAL", + ) + _require( + ( + state.geography_config is None + and state.geography_config_payload is None + and state.geography_binding is None + ) + or ( + state.geography_config is not None + and geography.AtomicSurveyReconstruction.to_bytes(state.geography_config) + == state.geography_config_payload + ), + "FINAL_GEOGRAPHY_CONFIG_SEAL", + ) + _require(_live() == _LIVE, "FINAL_PRODUCER_SEAL") + _preparation_entry( + state.preparation, state.preparation_payload, state.preparation_entry + ) + + +def _validate_budget(state, payload): + _producer() + _require( + _config_payload(state.geography_config) == state.geography_config_payload, + "INITIAL_GEOGRAPHY_CONFIG_CHANGED", + ) + _require( + _population_identity(state.allocated) == state.allocated_identity + and _population_identity(state.expanded) == state.expanded_identity, + "INITIAL_POPULATION_CHANGED", + ) + # This private retained view is only a reconstruction input. The fresh + # source borrow below authenticates it after the first reconstruction. + view = state.initial_view + _require(view.payload == state.preparation_payload, "SOURCE_PREPARATION_CHANGED") + instructions, allocation, geography_binding = _initial( + view, + state.allocated, + state.expanded, + preparation=state.preparation, + geography_config=state.geography_config, + _with_geography_binding=True, + ) + _require(geography_binding == state.geography_binding, "GEOGRAPHY_RECONSTRUCTION") + expected, constraint = _document( + view, + state.allocated, + state.expanded, + instructions, + allocation, + _producer(), + geography_binding, + ) + _require(expected == payload, "BUDGET_RECONSTRUCTION") + final = source.AuthenticatedSurveyPopulationPreparation.checked_view( + state.preparation + ) + _require( + final.payload == view.payload + and final.frame is view.frame + and final.selection_plan is view.selection_plan, + "FINAL_SOURCE_SEAL", + ) + _instructions, _allocation, final_geography = _initial( + final, + state.allocated, + state.expanded, + preparation=state.preparation, + geography_config=state.geography_config, + _with_geography_binding=True, + ) + _require(final_geography == state.geography_binding, "FINAL_GEOGRAPHY_SEAL") + return constraint + + +def _entry(value, cls): + entry = _ISSUED.get(id(value)) + _require( + type(value) is cls + and entry is not None + and entry[0]() is value + and type(value.payload) is bytes + and value.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + + +def _issue(cls, payload, state): + value = object.__new__(cls) + object.__setattr__(value, "payload", payload) + identifier = id(value) + + def forget(reference): + current = _ISSUED.get(identifier) + if current is not None and current[0] is reference: + _ISSUED.pop(identifier, None) + + reference = weakref.ref(value, forget) + _ISSUED[identifier] = (reference, payload, state) + return value + + +def _check_view_constraint(constraint, document): + """Bind every derived numerical field to an already requalified issued view.""" + reason = "FINAL_BUDGET_VIEW_CONSTRAINT" + _require(type(constraint) is group_bounds.GroupedUpperBounds, reason) + ids = tuple(document["household_ids"]) + groups = document["group_indices"] + bounds_hex = [row["upper_float64_hex"] for row in document["origins"]] + # This is the maintained constraint's stable-ID reduction order. These + # expected buckets come from the issued document, not constructor output. + buckets = [[] for _ in bounds_hex] + for index in sorted( + range(len(ids)), key=lambda i: (isinstance(ids[i], str), ids[i]) + ): + buckets[groups[index]].append(index) + + def immutable_array(value, expected, dtype): + payload = np.asarray(expected, dtype=dtype).tobytes() + return ( + type(value) is np.ndarray + and value.dtype.str == dtype + and value.shape == (len(expected),) + and value.strides == (8,) + and not value.flags.writeable + and not value.flags.owndata + and type(value.base) is bytes + and value.base == payload + and value.tobytes() == payload + ) + + digest_payload = { + "contract": "stable-household-fsum-group-upper-v1", + "household_ids": ids, + "group_indices": groups, + "absolute_bounds_hex": bounds_hex, + } + expected_digest = _sha( + json.dumps( + digest_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + ) + _require( + type(constraint.household_ids) is tuple + and len(constraint.household_ids) == len(ids) + and all( + type(actual) is type(expected) and actual == expected + for actual, expected in zip(constraint.household_ids, ids, strict=True) + ) + and immutable_array(constraint.group_indices, groups, " bool: + return False + + @property + def population_binding_authenticated(self) -> bool: + return False + + @property + def release_eligible(self) -> bool: + return False + + @property + def required_bindings(self) -> tuple[str, ...]: + return ( + "original_source_bytes", + "complete_original_membership", + "original_publisher_weight_authority", + "source_period_and_residence_scope", + "full_receiving_population_and_producer_edges", + "actual_selection_probability", + ) + + @property + def analysis_year(self) -> int: + return 2024 + + @property + def income_reference(self) -> str: + if self.original.key.source is Source.ASEC: + return f"calendar_year_{self.original.key.source_year}" + return "previous_12_months_at_each_2024_interview" + + +def _require(condition: bool, code: str) -> None: + if not condition: + raise DomainInputError(code) + + +def _token(value: str) -> str: + _require(type(value) is str and len(value) <= MAX_TOKEN_CHARS, "LITERAL_TYPE_SIZE") + return value + + +def _unsigned(raw: str, digits: int, maximum: int) -> int | None: + _token(raw) + if re.fullmatch(rf"[0-9]{{1,{digits}}}", raw) is None: + return None + value = int(raw) + return value if value <= maximum else None + + +def _key(key: HouseholdKey) -> tuple[Source, int, int, str | int]: + _require(type(key) is HouseholdKey and type(key.source) is Source, "HOUSEHOLD_KEY") + _require( + type(key.source_year) is int and type(key.survey_year) is int, "COHORT_TYPE" + ) + _token(key.native_id) + if key.source is Source.ACS: + _require((key.source_year, key.survey_year) == (2024, 2024), "COHORT_IDENTITY") + _require( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", key.native_id) is not None + and int(key.native_id[-7:]) > 0, + "HOUSEHOLD_KEY", + ) + native = key.native_id + else: + _require( + key.source_year in (2022, 2023, 2024) + and key.survey_year == key.source_year + 1, + "COHORT_IDENTITY", + ) + native = _unsigned(key.native_id, MAX_TOKEN_CHARS, 99999) + _require(native is not None and native > 0, "HOUSEHOLD_KEY") + return key.source, key.source_year, key.survey_year, native + + +def declaration() -> tuple[DomainShare, ...]: + """The prospectively fixed coefficients, without applying them to weights.""" + return ( + DomainShare( + Domain.SHARED_HOUSING, + "occupied_housing_unit", + Fraction(1, 2), + Fraction(1, 2), + ), + DomainShare( + Domain.RESIDUAL_HOUSING, "occupied_housing_unit", Fraction(1), Fraction(0) + ), + DomainShare(Domain.INSTITUTIONAL_GQ, "person", Fraction(1), Fraction(0)), + DomainShare(Domain.NONINSTITUTIONAL_GQ, "person", Fraction(1), Fraction(0)), + ) + + +def reduce_qualifiers(values: tuple[Qualifier, ...]) -> Qualifier: + """Existential reduction over known roster members, not membership proof.""" + _require( + type(values) is tuple and 0 < len(values) <= MAX_MEMBERS, "QUALIFIER_ROSTER" + ) + _require(all(type(v) is Qualifier for v in values), "QUALIFIER_TYPE") + if Qualifier.TRUE in values: + return Qualifier.TRUE + if Qualifier.UNKNOWN in values: + return Qualifier.UNKNOWN + return Qualifier.FALSE + + +def _acs_state(age: str, raw: str, minimum: int, codes: tuple[str, ...]) -> str: + # Exact vocabulary of acs_person_coverage_columns._field_state. No owner import. + value = _unsigned(age, 2, 99) + if value is None: + return "age_unresolved" + if value < minimum: + return "outside_age_universe" if raw == "" else "value_below_age_universe" + if raw == "": + return "missing_in_universe" + return "observed_code" if raw in codes else "unlabelled_code" + + +def _asec_state(raw: str) -> str: + # Exact vocabulary of asec_person_coverage_source._state. + if raw in ("1", "2", "3"): + return "observed_code" + if raw == "": + return "blank_unresolved" + if raw in ("-4", "-3", "-2", "-1", "0"): + return "unlabelled_in_range" + if re.fullmatch(r"-?(?:0|[1-9][0-9]*)", raw) and raw != "-0": + return "out_of_range" + return "malformed_token" + + +def _person(person: AcsPerson | AsecPerson) -> PersonDecision: + is_acs = type(person) is AcsPerson + age = _unsigned(person.age, 2 if is_acs else 19, 99 if is_acs else 2**63 - 1) + observations = [ + Observation( + "AGEP" if is_acs else "A_AGE", + person.age, + "observed_integer" if age is not None else "age_unresolved", + ) + ] + diagnostics = [] + if is_acs: + _token(person.pwgtp) + for raw, state in ( + (person.esr, person.esr_state), + (person.mil, person.mil_state), + ): + _token(raw) + _token(state) + _require( + person.esr_state + == _acs_state(person.age, person.esr, 16, ("1", "2", "3", "4", "5", "6")), + "FIELD_STATE_MISMATCH", + ) + _require( + person.mil_state + == _acs_state(person.age, person.mil, 17, ("1", "2", "3", "4")), + "FIELD_STATE_MISMATCH", + ) + observations.extend( + ( + Observation("ESR", person.esr, person.esr_state), + Observation("MIL", person.mil, person.mil_state), + ) + ) + true = person.esr in ("1", "2", "3", "6") + false = person.esr in ("4", "5") + if ( + person.mil_state == "observed_code" + and person.esr_state == "observed_code" + and ( + (person.mil == "1" and true) + or (person.mil in ("2", "3", "4") and false) + ) + ): + diagnostics.append("mil_esr_disagreement") + if person.mil_state != "observed_code": + diagnostics.append("mil_" + person.mil_state) + contradiction = False + else: + _token(person.prpertyp) + _token(person.prpertyp_state) + _require( + person.prpertyp_state == _asec_state(person.prpertyp), + "FIELD_STATE_MISMATCH", + ) + observations.append( + Observation("PRPERTYP", person.prpertyp, person.prpertyp_state) + ) + true, false = person.prpertyp == "2", person.prpertyp == "3" + contradiction = age is not None and age >= 16 and person.prpertyp == "1" + if age is None: + qualifier, reason = Qualifier.UNKNOWN, "source_age_unresolved" + elif age < 16: + qualifier, reason = Qualifier.FALSE, "source_age_below_16" + elif contradiction: + qualifier, reason = Qualifier.UNKNOWN, "adult_age_with_child_type" + elif true: + qualifier, reason = Qualifier.TRUE, "source_civilian_proxy" + elif false: + qualifier, reason = Qualifier.FALSE, "source_armed_forces_proxy" + else: + qualifier, reason = Qualifier.UNKNOWN, "required_person_status_unresolved" + return PersonDecision( + person, qualifier, reason, tuple(observations), tuple(diagnostics) + ) + + +def _members(row: AcsHousehold | AsecHousehold) -> tuple[PersonDecision, ...]: + _require( + type(row.persons) is tuple and len(row.persons) <= MAX_MEMBERS, + "MEMBER_TUPLE_BOUND", + ) + household = _key(row.key) + seen_keys, seen_lines = set(), set() + expected = AcsPerson if type(row) is AcsHousehold else AsecPerson + for person in row.persons: + _require(type(person) is expected, "MEMBER_TYPE") + _require(_key(person.household_key) == household, "MEMBER_HOUSEHOLD_KEY") + raw_line = person.sporder if expected is AcsPerson else person.a_lineno + line = _unsigned( + raw_line, + 2 if expected is AcsPerson else 19, + 20 if expected is AcsPerson else 2**63 - 1, + ) + _require( + line is not None and line > 0 and line not in seen_lines, "MEMBER_LINE_KEY" + ) + seen_lines.add(line) + person_key = line if expected is AcsPerson else _token(person.peridnum) + if expected is AsecPerson: + _require( + re.fullmatch(r"[0-9]{22}", person_key) is not None, "MEMBER_PERSON_KEY" + ) + _require(person_key not in seen_keys, "MEMBER_DUPLICATE_KEY") + seen_keys.add(person_key) + return tuple(_person(person) for person in row.persons) + + +def _field( + name: str, raw: str, digits: int, maximum: int, minimum: int = 0 +) -> tuple[int | None, Observation]: + value = _unsigned(raw, digits, maximum) + known = value is not None and value >= minimum + state = ( + "valid" + if known + else "missing" + if raw == "" + else "malformed" + if re.fullmatch(rf"[0-9]{{1,{digits}}}", raw) is None + else "unlabelled" + ) + return value if known else None, Observation(name, raw, state) + + +def classify_household(row: AcsHousehold | AsecHousehold) -> HouseholdDecision: + """Classify supplied complete rows; unresolved results authorize no allocation. + + Key/tuple/count inconsistencies raise. Unknown required observations retain + their literals and produce REVIEW_REQUIRED, never residual by default. + Source and population authority cannot be supplied as a boolean argument. + """ + _require(type(row) in (AcsHousehold, AsecHousehold), "HOUSEHOLD_TYPE") + key = _key(row.key) + is_acs = type(row) is AcsHousehold + _require(key[0] is (Source.ACS if is_acs else Source.ASEC), "HOUSEHOLD_SOURCE") + people = _members(row) + observations = [] + zero = None + + def field(name, raw, digits, maximum, minimum=0): + value, observation = _field(name, raw, digits, maximum, minimum) + observations.append(observation) + return value + + def result(status, reason, domain=None, share=None, unit=None, housing_units=None): + return HouseholdDecision( + row, + status, + domain, + share, + reason, + unit, + housing_units, + zero, + people, + tuple(observations), + ( + Annotation( + DECLARATION, + "source_age_16_ESR_only_or_PRPERTYP", + "joint_MIL_ESR_or_other_age_threshold", + "prespecified_development_choice", + ), + Annotation( + DECLARATION, + "fixed_domain_shares_and_structural_exclusions", + "estimated_overlap_or_optimized_shares", + "not_empirically_validated", + ), + Annotation( + DECLARATION, + reason, + "no_silent_residual_or_donor_admission", + status.value, + ), + ), + ) + + count = field( + "NP" if is_acs else "H_NUMPER", + row.np if is_acs else row.h_numper, + 2, + 20 if is_acs else 16, + ) + if count is not None: + _require(count == len(row.persons), "REPORTED_MEMBERSHIP_COUNT") + if is_acs: + housing = field("TYPEHUGQ", row.typehugq, 1, 3, 1) + weight = field("WGTP", row.wgtp, 4, 9999) + if count is None or housing is None: + return result(Status.REVIEW_REQUIRED, "acs_household_condition_unresolved") + if row.key.native_id[4:6] != ("HU" if housing == 1 else "GQ"): + return result(Status.REVIEW_REQUIRED, "acs_serialno_housing_conflict") + if housing == 1 and count == 0: + return result( + Status.EXCLUDED, "acs_vacancy", share=Fraction(0), housing_units=0 + ) + if housing in (2, 3): + if count != 1 or weight != 0: + return result(Status.REVIEW_REQUIRED, "acs_gq_placeholder_conflict") + person_weight = field("PWGTP", row.persons[0].pwgtp, 4, 9999, 1) + if person_weight is None: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = False + return result( + Status.ELIGIBLE, + "acs_gq_allocation", + Domain.INSTITUTIONAL_GQ if housing == 2 else Domain.NONINSTITUTIONAL_GQ, + Fraction(1), + "person", + 0, + ) + if weight is None or weight <= 0: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = False + else: + interview = field("H_HHTYPE", row.h_hhtype, 1, 3, 1) + household_type = field("HRHTYPE", row.hrhtype, 2, 10) + living = field("H_LIVQRT", row.h_livqrt, 2, 12, 1) + weight = field("HSUP_WGT", row.hsup_wgt, MAX_TOKEN_CHARS, 999999999) + if count is None: + return result(Status.REVIEW_REQUIRED, "asec_membership_count_unresolved") + if row.key.source_year != 2024: + return result(Status.EXCLUDED, "asec_older_cohort", share=Fraction(0)) + if interview in (2, 3): + return result(Status.EXCLUDED, "asec_noninterview", share=Fraction(0)) + if interview is None or living is None or household_type is None: + return result(Status.REVIEW_REQUIRED, "asec_household_condition_unresolved") + if living == 11: + return result( + Status.REVIEW_REQUIRED, + "asec_student_quarters_source_review", + share=Fraction(0), + housing_units=0, + ) + if living >= 8: + return result( + Status.EXCLUDED, "asec_nonhousing", share=Fraction(0), housing_units=0 + ) + if household_type not in range(1, 9) or count == 0: + return result(Status.REVIEW_REQUIRED, "asec_housing_membership_conflict") + if weight is None: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = weight == 0 + predicate = reduce_qualifiers(tuple(person.qualifier for person in people)) + if predicate is Qualifier.UNKNOWN: + return result(Status.REVIEW_REQUIRED, "household_qualifier_unresolved") + if predicate is Qualifier.TRUE: + return result( + Status.ELIGIBLE, + "chosen_shared_housing", + Domain.SHARED_HOUSING, + Fraction(1, 2), + "occupied_housing_unit", + 1, + ) + return result( + Status.ELIGIBLE if is_acs else Status.EXCLUDED, + "acs_known_residual_housing" if is_acs else "asec_residual_zero_allocation", + Domain.RESIDUAL_HOUSING, + Fraction(1) if is_acs else Fraction(0), + "occupied_housing_unit", + 1, + ) + + +def classify_households( + rows: tuple[AcsHousehold | AsecHousehold, ...], +) -> tuple[HouseholdDecision, ...]: + """Classify a bounded batch, rejecting household and cohort-person collisions.""" + _require(type(rows) is tuple and 0 < len(rows) <= MAX_HOUSEHOLDS, "HOUSEHOLD_BATCH") + keys, members = set(), 0 + for row in rows: + _require(type(row) in (AcsHousehold, AsecHousehold), "HOUSEHOLD_TYPE") + key = _key(row.key) + _require(key not in keys, "DUPLICATE_HOUSEHOLD_KEY") + keys.add(key) + _require( + type(row.persons) is tuple and len(row.persons) <= MAX_MEMBERS, + "MEMBER_TUPLE_BOUND", + ) + members += len(row.persons) + _require(members <= MAX_TOTAL_MEMBERS, "BATCH_MEMBER_BOUND") + person_keys = set() + for row in rows: + if type(row) is AsecHousehold: + for person in row.persons: + _require(type(person) is AsecPerson, "MEMBER_TYPE") + literal = _token(person.peridnum) + _require( + re.fullmatch(r"[0-9]{22}", literal) is not None, + "MEMBER_PERSON_KEY", + ) + person_key = (row.key.source_year, literal) + _require(person_key not in person_keys, "DUPLICATE_COHORT_PERSON_KEY") + person_keys.add(person_key) + return tuple(classify_household(row) for row in rows) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py new file mode 100644 index 000000000..de268fa88 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py @@ -0,0 +1,2284 @@ +"""Authenticate complete catalogues, select once, and stack original anchors. + +The selection is a development declaration. Source issuers retain their own +exact-key/unknown-inclusion receipts; this successor binds the separately +demonstrated catalogue sampling relation. No allocation, calibration or donor +operation runs here. Source catalogues/parents still have full-source costs. +""" + +from __future__ import annotations + +import _csv +import csv +import hashlib +import json +import math +import os +import stat +import sys +import weakref +from contextlib import contextmanager +from dataclasses import InitVar, dataclass, fields +from enum import Enum +from fractions import Fraction +from pathlib import Path +from types import FunctionType + +import numpy as np +import pandas as pd + +from microcosm.build import survey_domain_sample +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import population as graph_population +from microcosm.graph.population import dtype_for_token, storage_equal + +from . import acs_native_coverage_binding as acs_native +from . import acs_population_catalogue as acs_catalogue +from . import asec_2024_native_population as asec_native +from . import asec_current_money as current_money +from . import asec_population_catalogue as asec_catalogue +from . import graph_context, graph_sources, spine_assembly +from . import survey_catalogue_selection as selection +from . import survey_observed_age as observed_age +from . import survey_population_domains as domains +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.survey-population-preparation.v2" +REQUEST_PROTOCOL = "microcosm.us.survey-population-request.v1" +SPM_REQUEST_PROTOCOL = "microcosm.us.survey-population-request.v2" +SOURCE_CODEC = "us-survey-population-source-v1" +MAX_REQUEST_BYTES = 4096 +MAX_PAYLOAD_BYTES = 64 * 1024**2 +# One whole-roster receipt carries one record per source household and per +# person, so a full-source document is 1.02 GiB (docs/us-native-scale-transport.md +# §1) and a single bounded encode refuses at 6.1% of the source. The transport +# below keeps every accumulation inside the value the old ceiling allowed -- +# MAX_SEGMENT_BYTES is that ceiling, unchanged -- and adds one explicit resource +# ceiling for the total, which is not a transport shape. +MAX_SEGMENT_BYTES = MAX_PAYLOAD_BYTES +MAX_ROSTER_BYTES = 64 * MAX_SEGMENT_BYTES +ROSTER_PROTOCOL = "microcosm.us.survey-population-roster-transport.v1" +LEGACY_HEAD_COLUMN = "legacy_prepared_is_household_head" +LEGACY_HEAD_RULE = "microcosm.us.legacy-headship-namespace.v1" +_MAX_SCALAR_BYTES = 1024**2 +_FRAME_FAST_STRING_CHARS = 4096 +_SOURCE_ROSTER = ( + "selection-request.json", + "acs/csv_hus.zip", + "acs/csv_pus.zip", + "asec/parent.h5", + "asec/household-attachment.h5", + "asec/person-income-attachment.h5", + "asec/pppub23.csv", + "asec/pppub24.csv", + "asec/pppub25.csv", + "asec/hhpub25.csv", +) +_TOKEN = object() +_ISSUED = {} + + +class SurveyPopulationPreparationError(ValueError): + """Static refusal; messages contain no source observations or paths. + + Every catch-all chains the exception it caught, so a refused run names + what refused; the code itself stays static. + """ + + +def _require(condition, code): + if not condition: + raise SurveyPopulationPreparationError(code) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _check_scalars(value, depth=0): + _require(depth <= 64, "VALUE_DEPTH") + if type(value) is str: + _require(len(value) <= _MAX_SCALAR_BYTES, "SCALAR_LIMIT") + elif type(value) in (tuple, list): + for item in value: + _check_scalars(item, depth + 1) + elif type(value) is dict: + for key, item in value.items(): + _check_scalars(key, depth + 1) + _check_scalars(item, depth + 1) + + +def _chunks(value): + # A single primitive is bounded before JSON quoting. The encoder never + # builds an unbounded complete JSON string before enforcing the transport. + _check_scalars(value) + yield from json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ).iterencode(value) + + +def _encode(value, maximum=MAX_PAYLOAD_BYTES): + result = bytearray() + for chunk in _chunks(value): + encoded = chunk.encode("utf-8") + _require(len(result) + len(encoded) <= maximum, "PAYLOAD_LIMIT") + result.extend(encoded) + return bytes(result) + + +def _digest(value): + digest = hashlib.sha256() + for chunk in _chunks(value): + digest.update(chunk.encode("utf-8")) + return digest.hexdigest() + + +def _roster_segments(value, *, segment=MAX_SEGMENT_BYTES, maximum=MAX_ROSTER_BYTES): + """Encode one whole-roster document as bounded, content-addressed segments. + + ``_encode`` accumulates the complete canonical stream before it enforces + anything, so its bound is a ceiling on the document rather than on the + process's allocation. This walks the same ``_chunks`` stream, closes a + segment whenever the next chunk would carry it past ``segment``, and keeps + ``PAYLOAD_LIMIT`` on each segment at the same number the single encode used. + The total is bounded separately, by ``ROSTER_LIMIT``. + + The running digest is over the chunks in order, so it equals + ``_digest(value)`` exactly -- and therefore equals ``_sha(_encode(value))`` + for every document ``_encode`` accepts, because sha256 is a streaming hash + over those same bytes. The canonical byte stream is unchanged; only its + materialization is bounded. + """ + digest = hashlib.sha256() + segments, table, current, total = [], [], bytearray(), 0 + + def close(): + raw = bytes(current) + digest.update(raw) + table.append((_sha(raw), len(raw))) + segments.append(raw) + current.clear() + + for chunk in _chunks(value): + encoded = chunk.encode("utf-8") + if current and len(current) + len(encoded) > segment: + close() + # A single chunk is one JSON token, and _check_scalars has already + # bounded every string, so this refuses only a token larger than one + # whole segment -- the same refusal, at the same number. + _require(len(current) + len(encoded) <= segment, "PAYLOAD_LIMIT") + _require(total + len(encoded) <= maximum, "ROSTER_LIMIT") + current.extend(encoded) + total += len(encoded) + if current or not segments: + close() + return segments, tuple(table), digest.hexdigest(), total + + +def _absent(path): + """True only when nothing at all sits at ``path``, symlinks included.""" + try: + path.lstat() + except FileNotFoundError: + return True + return False + + +def _spill_roster(root, name, segments, table, digest, total): + """Write the segments beside a header naming them, as the store's own shape. + + ``ContentStore.put_frame`` already writes content-addressed bodies under a + ``meta.json`` payload table of sha256 and size per file. This is that shape + for a receipt: one ``.segment`` per segment and one ``header.json`` + naming the whole-stream digest, the size and the ordered segment table, so + the receipt has a re-verifiable on-disk form that does not depend on the + graph store. + """ + _require(type(name) is str and name.isidentifier(), "ROSTER_NAME") + directory = root / name + directory.mkdir(parents=True, exist_ok=True) + # The inventory's resource contract covers reads, not writes, so nothing + # else in this repository would notice a redirected spill. Every component + # of the path this module writes through is checked here: ``root`` is + # already a validated snapshot root, ``name`` is an identifier, and each + # segment's own name is the sha256 of its bytes. + _require( + not stat.S_ISLNK(directory.lstat().st_mode) and directory.is_dir(), + "ROSTER_SPILL_LOCATION", + ) + for raw, (sha, size) in zip(segments, table, strict=True): + path = directory / (sha + ".segment") + # lstat, not exists(): a broken symlink does not exist and would be + # written straight through. + if _absent(path): + path.write_bytes(raw) + stats = path.lstat() + _require( + not stat.S_ISLNK(stats.st_mode) + and stat.S_ISREG(stats.st_mode) + and stats.st_size == size, + "ROSTER_SEGMENT_CHANGED", + ) + # A segment that was already there is read back and hashed, not trusted + # for its size. The run's own bytes come from memory either way, so this + # protects the claim that the spill is a re-verifiable on-disk form -- + # which it would not be if same-size different content passed. + _require(_sha(path.read_bytes()) == sha, "ROSTER_SEGMENT_CHANGED") + header = { + "protocol": ROSTER_PROTOCOL, + "name": name, + "sha256": digest, + "size": total, + "segments": [[sha, size] for sha, size in table], + } + path = directory / "header.json" + if not _absent(path): + stats = path.lstat() + _require( + not stat.S_ISLNK(stats.st_mode) and stat.S_ISREG(stats.st_mode), + "ROSTER_SPILL_LOCATION", + ) + path.write_bytes(_encode(header)) + return header + + +def _roster_payload( + value, *, spill=None, name=None, segment=MAX_SEGMENT_BYTES, maximum=MAX_ROSTER_BYTES +): + """One whole-roster receipt's bytes, and the header that names its segments. + + The bytes are identical to ``_encode(value)`` wherever ``_encode`` accepts + the document, and exist wherever it does not. They are materialized once, + because ``KernelResult.artifacts`` is a mapping of ``bytes`` and + ``ContentStore.put_bytes`` takes whole bytes; removing that copy needs a + streaming artifact channel in ``microcosm-graph``, which is not this + module's to change. + """ + segments, table, digest, total = _roster_segments( + value, segment=segment, maximum=maximum + ) + header = None + if spill is not None: + header = _spill_roster(spill, name, segments, table, digest, total) + for raw, (sha, size) in zip(segments, table, strict=True): + _require(len(raw) == size and _sha(raw) == sha, "ROSTER_SEGMENT_CHANGED") + payload = b"".join(segments) + _require(len(payload) == total <= maximum, "ROSTER_LIMIT") + _require(_sha(payload) == digest, "ROSTER_DIGEST") + return payload, header + + +def _catalogue_fast_path(value): + """Recognize immutable, bounded raw records without changing admission.""" + if type(value) is not tuple or len(value) != 2: + return False + for group in value: + if type(group) is not tuple: + return False + for record in group: + if type(record) is not tuple or len(record) != 7: + return False + people = record[6] + if type(people) is not tuple or len(people) > 20: + return False + if any(type(person) is not tuple or len(person) != 9 for person in people): + return False + characters = 0 + for row in (record[:6], *people): + for item in row: + if type(item) is str: + if not item.isascii(): + return False + characters += len(item) + if characters > 65_536: + return False + elif type(item) is int: + if not -(2**63) <= item < 2**63: + return False + else: + return False + return True + + +def _catalogue_chunks(value): + # Decide for the whole value before encoding. Unexpected shapes retain + # the generic encoder's full-depth scalar pass and error precedence. + if not _catalogue_fast_path(value): + yield from _chunks(value) + return + _check_scalars(value) + encoder = json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ) + yield "[" + for group_index, group in enumerate(value): + if group_index: + yield "," + yield "[" + for record_index, record in enumerate(group): + if record_index: + yield "," + # encode uses the ordinary one-shot C provider when available. + # At most 186 leaves and 65,536 ASCII characters bound this + # individual JSON record below 400 KiB, including escaping. + yield encoder.encode(record) + yield "]" + yield "]" + + +def _catalogue_digest(value): + digest = hashlib.sha256() + for chunk in _catalogue_chunks(value): + digest.update(chunk.encode("utf-8")) + return digest.hexdigest() + + +def _catalogue_memo(value, expected_digest): + """Retain one proved immutable tree, never an unchecked caller digest. + + The exact-tuple/int/ASCII-str eligibility test also excludes mutable leaves + and subclasses. Strong references prevent identity reuse. Rehash once while + creating the memo to bind those exact immutable roots to the earlier seal. + A replacement tree or an ineligible shape keeps the original digest path. + """ + if not _catalogue_fast_path(value): + return None + if _catalogue_digest(value) != expected_digest: + return None + return (value[0], value[1], expected_digest) + + +def _memoized_catalogue_digest(value, memo=None, *, expected_digest=None): + """Reuse only the issued preparation's unchanged immutable record roots. + + This private memo is not source authority. The owning preparation still + performs every source/producer borrow and full physical Frame seal. Its + original nested digest is checked independently of the memo's digest. + """ + if ( + type(value) is tuple + and len(value) == 2 + and type(memo) is tuple + and len(memo) == 3 + and value[0] is memo[0] + and value[1] is memo[1] + and type(memo[2]) is str + and memo[2] == expected_digest + ): + return memo[2] + return _catalogue_digest(value) + + +def _value(value): + if isinstance(value, Enum): + return value.value + if type(value) is Fraction: + return [value.numerator, value.denominator] + if hasattr(type(value), "__dataclass_fields__"): + return { + field.name: _value(getattr(value, field.name)) for field in fields(value) + } + if type(value) in (tuple, list): + return [_value(item) for item in value] + if type(value) is dict: + return {key: _value(item) for key, item in value.items()} + if isinstance(value, np.generic): + return value.item() + _require(value is None or type(value) in (str, int, bool, float), "VALUE_TYPE") + return value + + +def _plan_document(plan): + _require(type(plan) is selection.CatalogueSelectionPlan, "SELECTION_TYPE") + result = { + "fraction": _value(plan.fraction), + "seed": plan.seed, + "supplied_households": plan.supplied_households, + } + budget = [0] + for name in ("selected", "excluded", "cells"): + result[name] = [] + for row in getattr(plan, name): + _bounded_append(result[name], _value(row), budget) + return result + + +def _root(value, *, allow_missing=False): + _require(isinstance(value, (str, Path)), "PATH_TYPE") + path = Path(value).absolute() + for component in (path, *path.parents): + if allow_missing: + try: + component.lstat() + except FileNotFoundError: + continue + _require(not stat.S_ISLNK(component.lstat().st_mode), "SOURCE_SYMLINK") + return path + + +def _stat_identity(info): + return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +@contextmanager +def _regular_reader(path, maximum): + """Never follow a replaced final symlink or block on a substituted FIFO.""" + _root(path.parent) + before = path.lstat() + _require( + stat.S_ISREG(before.st_mode) and 0 <= before.st_size <= maximum, + "SOURCE_REGULAR_FILE", + ) + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + try: + opened = os.fstat(descriptor) + _require( + stat.S_ISREG(opened.st_mode) + and _stat_identity(opened) == _stat_identity(before), + "SOURCE_DESCRIPTOR_CHANGED", + ) + with os.fdopen(descriptor, "rb", closefd=False) as stream: + yield stream, opened + after = os.fstat(descriptor) + current = path.lstat() + _require( + stat.S_ISREG(current.st_mode) + and _stat_identity(opened) + == _stat_identity(after) + == _stat_identity(current), + "SOURCE_CHANGED", + ) + finally: + os.close(descriptor) + + +def _request(source_dir): + root = _root(source_dir) + path = root / "selection-request.json" + info = path.lstat() + _require( + stat.S_ISREG(info.st_mode) and info.st_size <= MAX_REQUEST_BYTES, "REQUEST_FILE" + ) + with _regular_reader(path, MAX_REQUEST_BYTES) as (stream, _opened): + payload = stream.read(MAX_REQUEST_BYTES + 1) + _require(len(payload) <= MAX_REQUEST_BYTES, "REQUEST_LIMIT") + document = json.loads(payload) + _require(type(document) is dict, "REQUEST_FIELDS") + fields = {"protocol", "declaration", "fraction", "seed"} + if document.get("protocol") == SPM_REQUEST_PROTOCOL: + fields.add("acs_spm_construction") + _require(set(document) == fields, "REQUEST_FIELDS") + _require( + document["protocol"] in (REQUEST_PROTOCOL, SPM_REQUEST_PROTOCOL) + and document["declaration"] == domains.DECLARATION, + "REQUEST_DECLARATION", + ) + pair = document["fraction"] + _require( + type(pair) is list + and len(pair) == 2 + and all(type(v) is int for v in pair) + and pair[1] > 0, + "REQUEST_FRACTION", + ) + fraction = Fraction(*pair) + _require( + 0 < fraction <= 1 and [fraction.numerator, fraction.denominator] == pair, + "REQUEST_FRACTION", + ) + seed = document["seed"] + _require(type(seed) is int and 0 <= seed < 2**64, "REQUEST_SEED") + _require(_encode(document, MAX_REQUEST_BYTES) == payload, "REQUEST_CANONICAL") + _request_construction(document) + return root, document, payload, fraction, seed + + +def _request_construction(document): + """Decode only explicit v2 options; v1 never imports an SPM helper.""" + if document["protocol"] == REQUEST_PROTOCOL: + return None + options = document["acs_spm_construction"] + _require( + type(options) is dict and set(options) == {"policy", "minor_partner_role"}, + "REQUEST_SPM_OPTIONS", + ) + from microcosm.build.acs_spm_source_assembly import AcsSpmSourceAssemblyOptions + + return AcsSpmSourceAssemblyOptions(**options) + + +def read_survey_population_request(source_dir): + """Decode bounded request arguments, without granting source authority.""" + try: + _path, _document, _payload, fraction, seed = _request(source_dir) + return fraction, seed + except SurveyPopulationPreparationError: + raise + except Exception as error: + raise SurveyPopulationPreparationError("REQUEST_REFUSED") from error + + +def _file_limits(): + # Limits bound only transport. Actual closed owners independently verify + # their pinned source bytes and stricter member/row/receipt contracts. + return { + **{name: asec_native._MAX_CHECKPOINT_BYTES for name in _SOURCE_ROSTER}, + **{ + f"asec/pppub{year}.csv": asec_native.coverage_owner.literal.MAX_MEMBER_BYTES + for year in (23, 24, 25) + }, + "asec/hhpub25.csv": asec_native.anchor_owner._MAX_MEMBER_BYTES, + "selection-request.json": MAX_REQUEST_BYTES, + "acs/csv_hus.zip": acs_catalogue._ARCHIVE_BYTES, + "acs/csv_pus.zip": acs_catalogue._ARCHIVE_BYTES, + } + + +def _source_files(root): + _root(root) + _require(stat.S_ISDIR(root.lstat().st_mode), "SOURCE_ROOT") + found = [] + for directory, expected in ( + (root, {"acs", "asec", "selection-request.json"}), + (root / "acs", {"csv_hus.zip", "csv_pus.zip"}), + ( + root / "asec", + {Path(n).name for n in _SOURCE_ROSTER if n.startswith("asec/")}, + ), + ): + _require(stat.S_ISDIR(directory.lstat().st_mode), "SOURCE_DIRECTORY") + children = tuple(directory.iterdir()) + _require( + {p.name for p in children} == expected and len(children) == len(expected), + "SOURCE_ROSTER", + ) + for path in children: + mode = path.lstat().st_mode + _require(not stat.S_ISLNK(mode), "SOURCE_SYMLINK") + if path.name in {"acs", "asec"} and path.parent == root: + _require(stat.S_ISDIR(mode), "SOURCE_DIRECTORY") + else: + _require(stat.S_ISREG(mode), "SOURCE_REGULAR_FILE") + found.append(path.relative_to(root).as_posix()) + _require(set(found) == set(_SOURCE_ROSTER), "SOURCE_ROSTER") + limits = _file_limits() + _require( + sum( + (root / name).lstat().st_size + for name in ("acs/csv_hus.zip", "acs/csv_pus.zip") + ) + <= acs_catalogue._ARCHIVE_BYTES, + "ARCHIVE_LIMIT", + ) + result = [] + for name in _SOURCE_ROSTER: + path = root / name + digest, size = hashlib.sha256(), 0 + with _regular_reader(path, limits[name]) as (stream, opened): + while block := stream.read(1024**2): + size += len(block) + _require(size <= limits[name], "SOURCE_FILE_LIMIT") + digest.update(block) + _require(size == opened.st_size, "SOURCE_CHANGED") + result.append([name, size, digest.hexdigest()]) + return tuple(tuple(row) for row in result) + + +def _file_stats(root): + _root(root) + result = [] + # Directory ctime/mtime also bind the exact admitted roster across the last + # producer I/O, including a newly added unlisted entry in either source arm. + for directory in (root, root / "acs", root / "asec"): + info = directory.lstat() + _require(stat.S_ISDIR(info.st_mode), "SOURCE_DIRECTORY") + result.append(_stat_identity(info)) + for name in _SOURCE_ROSTER: + path = root / name + _root(path.parent) + info = path.lstat() + _require(stat.S_ISREG(info.st_mode), "SOURCE_REGULAR_FILE") + result.append(_stat_identity(info)) + return tuple(result) + + +def _modules(): + modules = ( + sys.modules[__name__], + domains, + selection, + survey_domain_sample, + spine_assembly, + graph_sources, + graph_context, + graph_population, + observed_age, + acs_catalogue, + asec_catalogue, + acs_native, + acs_native.housing, + acs_native.coverage, + acs_native.coverage.literal, + sys.modules[acs_native.housing.build_acs_pums_unit_frame.__module__], + sys.modules[acs_native.housing.map_acs_native_inputs.__module__], + *asec_native._modules(), + ) + return tuple({module.__name__: module for module in modules}.values()) + + +def _runtime_marker(value, depth=0): + """Detach simple callable configuration, retaining opaque dependency identity.""" + _require(depth <= 16, "PRODUCER_CONFIGURATION_DEPTH") + if type(value) in (type(None), bool, int, float, str, bytes): + return type(value), value + if type(value) in (tuple, list): + return type(value), tuple(_runtime_marker(v, depth + 1) for v in value) + if type(value) is dict: + return tuple( + (_runtime_marker(k, depth + 1), _runtime_marker(v, depth + 1)) + for k, v in value.items() + ) + if isinstance(value, FunctionType): + return value, value.__code__ + return type(value), id(value) + + +def _function_seal(function, depth=0): + # contextmanager keeps executable code in both __wrapped__ and a closure; + # the public wrapper's __code__ alone cannot bind that implementation. + _require(depth <= 16, "PRODUCER_WRAPPER_DEPTH") + wrapped = getattr(function, "__wrapped__", None) + return ( + function, + function.__code__, + _runtime_marker(function.__defaults__), + _runtime_marker(function.__kwdefaults__), + tuple(_runtime_marker(c.cell_contents) for c in function.__closure__ or ()), + _function_seal(wrapped, depth + 1) + if isinstance(wrapped, FunctionType) + else _runtime_marker(wrapped), + ) + + +def _live(): + result = {} + for module in _modules(): + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[(module.__name__, name)] = _function_seal(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + result[(module.__name__, name)] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[(module.__name__, name, method)] = _function_seal( + function + ) + result["rng"] = (np.random.Generator, np.random.PCG64, np.random.SeedSequence) + result["csv"] = (csv.reader, _csv.reader) + # The bounded catalogue path uses JSONEncoder.encode and its C provider; + # frame cells also use the same encode_basestring provider as _chunks. + # Bind their actual live identities through this existing seal. + result["json_catalogue"] = ( + json.JSONEncoder, + json.encoder, + tuple( + _function_seal(function) + if isinstance(function, FunctionType) + else _runtime_marker(function) + for function in ( + getattr(json.JSONEncoder, name, None) + for name in ("__init__", "encode", "iterencode") + ) + ), + getattr(json.encoder, "c_make_encoder", None), + getattr(json.encoder, "encode_basestring", None), + ) + result["contract"] = ( + PROTOCOL, + REQUEST_PROTOCOL, + SPM_REQUEST_PROTOCOL, + acs_native.housing._SPM_ASSEMBLER_SHA256, + acs_native.housing._SPM_EVIDENCE_MAX_BYTES, + SOURCE_CODEC, + MAX_REQUEST_BYTES, + MAX_PAYLOAD_BYTES, + MAX_SEGMENT_BYTES, + MAX_ROSTER_BYTES, + ROSTER_PROTOCOL, + LEGACY_HEAD_COLUMN, + LEGACY_HEAD_RULE, + _MAX_SCALAR_BYTES, + _FRAME_FAST_STRING_CHARS, + _SOURCE_ROSTER, + domains.DECLARATION, + domains.MAX_MEMBERS, + domains.MAX_HOUSEHOLDS, + domains.MAX_TOTAL_MEMBERS, + selection._BATCH_HOUSEHOLDS, + selection._BATCH_PEOPLE, + observed_age.RULE, + observed_age.AGE_CONVENTION, + observed_age.MAX_ROWS, + observed_age.MAX_EXACT_FLOAT64_INTEGER, + ) + return result + + +def _code_bytes(): + return { + module.__name__: _sha(Path(module.__file__).read_bytes()) + for module in _modules() + } + + +def _producer(): + _require(_live() == _LIVE and _code_bytes() == _BYTES, "PRODUCER_CHANGED") + return { + "implementation": _BYTES, + "acs": acs_catalogue._producer(), + "asec": _sha(_encode(asec_catalogue._implementation())), + "numpy": np.__version__, + "rng": "numpy.random.PCG64/SeedSequence", + "declaration": domains.DECLARATION, + } + + +def _authority(): + return _encode( + { + "asec": asec_catalogue._source_authority().hex(), + "acs": acs_native.housing._ARCHIVE_PINS, + } + ) + + +def _cell(value): + if ( + value is None + or value is pd.NA + or (isinstance(value, (float, np.floating)) and math.isnan(value)) + ): + return None + if isinstance(value, np.generic): + value = value.item() + _require(type(value) in (str, int, bool, float), "FRAME_CELL_TYPE") + if type(value) is float: + _require(math.isfinite(value), "FRAME_NONFINITE") + return ["float", value.hex()] + return value + + +def _frame_cell_encode(value, maximum=MAX_PAYLOAD_BYTES): + """Encode one normalized cell with the exact _encode preimage and limits. + + _cell admits only None, exact bool/int/str, or its own freshly constructed + ["float", finite_float.hex()] list. Keep normalization and the scalar walk + before encoding; large strings/integers retain the original slow path. + No Frame, Series, index, or normalized value is retained between calls. + """ + value = _cell(value) + kind = type(value) + if not ( + value is None + or kind is bool + or (kind is int and -(2**63) <= value < 2**64) + or (kind is str and len(value) <= _FRAME_FAST_STRING_CHARS) + or kind is list + ): + return _encode(value, maximum) + + _check_scalars(value) + if value is None: + encoded = b"null" + elif kind is bool: + encoded = b"true" if value else b"false" + elif kind is int: + # Match JSON's exact-int formatter, including its decimal digit policy. + encoded = int.__repr__(value).encode("ascii") + elif kind is str: + # _chunks selects this very provider with ensure_ascii=False. UTF-8 + # conversion still precedes the payload check, including surrogates. + # At most 6 * 4096 + 2 encoded bytes, even for all control characters. + encoded = json.encoder.encode_basestring(value).encode("utf-8") + else: + # Only _cell can construct this list: both strings are ASCII without + # JSON escapes. Keep hex spelling (especially -0.0), never decimalize. + encoded = b'["float","' + value[1].encode("ascii") + b'"]' + _require(len(encoded) <= maximum, "PAYLOAD_LIMIT") + return encoded + + +_HEX_DIGITS = np.frombuffer(b"0123456789abcdef", dtype=np.uint8) +# ``["float","`` -0x1. <13 mantissa nibbles> p +<=4 exponent digits> "]\n +_FLOAT_WIDTH = 37 +_FLOAT_HEAD = np.frombuffer(b'["float","', dtype=np.uint8) +_FLOAT_LEAD = np.stack( + ( + np.frombuffer(b"0x1.", dtype=np.uint8), + np.frombuffer(b"0x0.", dtype=np.uint8), + ) +) +_FLOAT_TAIL = np.frombuffer(b'"]\n', dtype=np.uint8) +_FLOAT_NULL = np.frombuffer(b"null\n", dtype=np.uint8) +_FLOAT_EXPONENT_DIGITS = 4 +_MANTISSA_SHIFTS = np.arange(48, -1, -4, dtype=np.uint64) +_BOOL_CELLS = np.array([b"false\n", b"true\n", b"null\n"], dtype=object) +_INTEGER_DTYPES = frozenset( + ("int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64") +) +_NULLABLE_INTEGER_DTYPES = frozenset( + ("Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", "UInt64") +) + + +def _float_cells(values): + """``_frame_cell_encode`` + a newline for one float64 column, per column. + + The record is laid out at fixed offsets with a per-position keep mask, so + the variable-length spellings ``float.hex()`` produces are cut out of one + padded matrix rather than assembled a cell at a time: + + 0..9 ``["float","`` always + 10 ``-`` iff the sign bit is set + 11..14 ``0x1.`` / ``0x0.`` normal / subnormal-or-zero + 15..27 13 mantissa nibbles all 13, or only column 15 (``0``) for zero + 28 ``p`` always + 29 ``+`` / ``-`` sign of the unbiased exponent + 30..33 4 exponent digits only the last significant ones + 34..36 ``"]`` and newline always + + A NaN cell spells ``null`` instead, which is what ``_cell`` makes of it. + ``_cell`` refuses a non-finite cell, so an infinity refuses here too, with + the same code; it is raised for the column rather than at the first + offending cell, and the digest is discarded either way. + """ + _require(not np.isinf(values).any(), "FRAME_NONFINITE") + count = values.shape[0] + if count == 0: + return b"" + bits = values.view(np.uint64) + missing = np.isnan(values) + sign = (bits >> np.uint64(63)).astype(bool) + biased = ((bits >> np.uint64(52)) & np.uint64(0x7FF)).astype(np.int64) + mantissa = bits & np.uint64(0xFFFFFFFFFFFFF) + subnormal = biased == 0 + zero = subnormal & (mantissa == 0) + exponent = np.where(subnormal, np.int64(-1022), biased - 1023) + exponent = np.where(zero, np.int64(0), exponent) + + cells = np.zeros((count, _FLOAT_WIDTH), dtype=np.uint8) + keep = np.zeros((count, _FLOAT_WIDTH), dtype=bool) + cells[:, 0:10] = _FLOAT_HEAD + cells[:, 10] = np.uint8(0x2D) + keep[:, 10] = sign + cells[:, 11:15] = _FLOAT_LEAD[subnormal.astype(np.intp)] + cells[:, 15:28] = _HEX_DIGITS[ + ((mantissa[:, None] >> _MANTISSA_SHIFTS[None, :]) & np.uint64(0xF)).astype( + np.uint8 + ) + ] + cells[zero, 15] = np.uint8(0x30) + cells[:, 28] = np.uint8(0x70) + cells[:, 29] = np.where(exponent < 0, np.uint8(0x2D), np.uint8(0x2B)) + magnitude = np.abs(exponent) + for position in range(_FLOAT_EXPONENT_DIGITS): + power = 10 ** (_FLOAT_EXPONENT_DIGITS - 1 - position) + cells[:, 30 + position] = np.uint8(0x30) + (magnitude // power % 10).astype( + np.uint8 + ) + cells[:, 34:37] = _FLOAT_TAIL + + keep[:, 0:10] = True + keep[:, 11:28] = True + keep[zero, 16:28] = False + keep[:, 28:30] = True + significant = np.where( + magnitude >= 1000, + 4, + np.where(magnitude >= 100, 3, np.where(magnitude >= 10, 2, 1)), + ) + keep[:, 30:34] = ( + np.arange(_FLOAT_EXPONENT_DIGITS)[None, :] + >= (_FLOAT_EXPONENT_DIGITS - significant)[:, None] + ) + keep[:, 34:37] = True + if missing.any(): + cells[missing, 0:5] = _FLOAT_NULL + keep[missing, :] = False + keep[missing, 0:5] = True + return cells[keep].tobytes() + + +def _integer_cells(values): + """One integer column's cells. ``str`` on a Python int is ``int.__repr__``. + + Every value an integer dtype can hold lies inside the fast path's + ``-2**63 <= value < 2**64`` window and spells at most twenty ASCII bytes, + so neither ``_encode`` nor ``PAYLOAD_LIMIT`` is reachable from here. + """ + if values.shape[0] == 0: + return b"" + return ("\n".join(map(str, values.tolist())) + "\n").encode("ascii") + + +def _nullable_integer_cells(series): + """One nullable-integer column's cells; ``pd.NA`` is ``_cell``'s ``None``. + + The unsigned widths keep an unsigned carrier: ``UInt64`` holds values above + ``2**63 - 1`` that no signed carrier can spell, and the fast path admits + every integer below ``2**64``. + """ + if len(series) == 0: + return b"" + carrier = "uint64" if str(series.dtype).startswith("U") else "int64" + present = series.to_numpy(dtype=carrier, na_value=0) + spelled = list(map(str, present.tolist())) + for position in np.flatnonzero(np.asarray(series.isna())).tolist(): + spelled[position] = "null" + return ("\n".join(spelled) + "\n").encode("ascii") + + +def _boolean_cells(series): + """One boolean column's cells: three spellings, selected by index.""" + if len(series) == 0: + return b"" + missing = np.asarray(series.isna()) + present = series.to_numpy(dtype=bool, na_value=False) + codes = np.where(missing, np.intp(2), present.astype(np.intp)) + return b"".join(_BOOL_CELLS[codes].tolist()) + + +def _string_cells(series): + """One string column's cells, encoding each distinct value once. + + ``StringDtype`` compares and hashes exactly, so grouping equal values + cannot merge two whose encodings differ. ``object`` and ``category`` + deliberately do not qualify: ``1``, ``True`` and ``1.0`` are equal and hash + alike there while ``_frame_cell_encode`` spells them three different ways, + so those dtypes keep the cell-at-a-time walk. + """ + if len(series) == 0: + return b"" + codes, uniques = pd.factorize(series, use_na_sentinel=False) + encoded = np.empty(len(uniques), dtype=object) + for position, value in enumerate(uniques): + encoded[position] = _frame_cell_encode(value) + b"\n" + return b"".join(encoded[codes].tolist()) + + +def _cells_blob(series): + """The exact bytes a cell-at-a-time walk over ``series`` would digest. + + Every branch below is proved equal to that walk, byte for byte, in + ``test_us_survey_population_frame_seal.py``; a dtype with no proved branch + takes the walk itself rather than a guess. + """ + name = str(series.dtype) + if name == "float64": + return _float_cells(series.to_numpy(dtype=np.float64, copy=False)) + if name in _INTEGER_DTYPES: + return _integer_cells(series.to_numpy(copy=False)) + if name in _NULLABLE_INTEGER_DTYPES: + return _nullable_integer_cells(series) + if name in ("bool", "boolean"): + return _boolean_cells(series) + if name == "string": + return _string_cells(series) + return b"".join(_frame_cell_encode(value) + b"\n" for value in series) + + +def _index_blob(index): + """The same, for an axis, which the walk reads exactly as it reads a column.""" + name = str(index.dtype) + if name in _INTEGER_DTYPES: + return _integer_cells(index.to_numpy(copy=False)) + if name == "float64": + return _float_cells(index.to_numpy(dtype=np.float64, copy=False)) + return b"".join(_frame_cell_encode(value) + b"\n" for value in index) + + +def _frame_identity(frame): + _require( + isinstance(frame, Frame) and frame.schema == US_SCHEMA and not frame.links, + "FRAME_TYPE", + ) + digest = hashlib.sha256() + + def update(value): + digest.update(_encode(value)) + digest.update(b"\n") + + update([list(frame.entities), list(frame.weighted_entities)]) + for entity in frame.entities: + table = frame.table(entity) + _require( + type(table) is pd.DataFrame and not table.columns.has_duplicates, + "FRAME_TABLE", + ) + update( + [ + entity, + list(table.columns), + type(table.columns).__name__, + str(table.columns.dtype), + list(table.columns.names), + type(table.index).__name__, + str(table.index.dtype), + list(table.index.names), + ] + ) + digest.update(_index_blob(table.index)) + for column in table: + series = table[column] + dtype = series.dtype + update( + [ + column, + str(dtype), + getattr(dtype, "storage", None), + str(getattr(dtype, "na_value", "")), + ] + ) + digest.update(_cells_blob(series)) + update( + [ + str(frame.strata.dtype), + type(frame.strata.index).__name__, + str(frame.strata.index.dtype), + list(frame.strata.index.names), + getattr(frame.strata.dtype, "storage", None), + frame.strata.name, + ] + ) + digest.update(_index_blob(frame.strata.index)) + digest.update(_cells_blob(frame.strata)) + for entity in frame.weighted_entities: + weights = frame.weights_for(entity) + update( + [ + entity, + weights.kind.value, + str(weights.values.dtype), + list(weights.values.shape), + ] + ) + digest.update(weights.values.tobytes()) + digest.update(graph_context.encode_us_frame_context(frame)) + return digest.hexdigest() + + +def _copy_source(frame): + result = Frame( + {e: frame.table(e).copy(deep=True) for e in frame.entities}, + frame.schema, + { + e: Weights(frame.weights_for(e).values.copy(), frame.weights_for(e).kind) + for e in frame.weighted_entities + }, + frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + # Only physical string storage changes on owned copies, before the stack's + # exact shared-dtype check. Actual values and missing masks stay identical. + for entity in result.entities: + for column in result.table(entity): + original = result.table(entity)[column] + if isinstance(original.dtype, pd.StringDtype): + changed = original.astype(dtype_for_token("string")) + _same_values(original, changed) + result.table(entity)[column] = changed + return result + + +def _same_values(left, right): + _require(len(left) == len(right), "ROW_COUNT") + _require(list(map(_cell, left)) == list(map(_cell, right)), "SOURCE_VALUE_CHANGED") + + +def _normalized_source_copy(frame: Frame) -> Frame: + _require( + LEGACY_HEAD_COLUMN not in frame.person, "LEGACY_HEADSHIP_NAMESPACE_COLLISION" + ) + result = _copy_source(frame) + # These prepared booleans are not the raw reference-person observations. + # Preserve every value under a diagnostic name; the role source operator + # will own the nullable canonical column from A_EXPRRP / RELSHIPP. + if "is_household_head" in result.person: + result.person.columns = _headship_namespace_axis(result.person.columns) + _require("A_AGE" in frame.person, "OBSERVED_AGE_REQUIRED") + result.person["age"] = observed_age.normalize_observed_age( + frame.person["A_AGE"], frame.person.get("age") + ) + _verify_normalized_copy(frame, result) + return result + + +def _headship_namespace_axis(columns): + return pd.Index( + [LEGACY_HEAD_COLUMN if c == "is_household_head" else c for c in columns], + dtype=columns.dtype, + name=columns.name, + ) + + +def _verify_normalized_copy(original: Frame, normalized: Frame): + """Only age normalization, the lossless headship name and string storage differ.""" + _require( + original.schema == normalized.schema + and original.entities == normalized.entities + and original.links == normalized.links == () + and original.metadata == normalized.metadata + and original.mass_log == normalized.mass_log + and original.weighted_entities == normalized.weighted_entities, + "NORMALIZED_FRAME_CONTEXT", + ) + for entity in original.entities: + before, after = original.table(entity), normalized.table(entity) + expected_axis = before.columns + if entity == "person": + _require( + LEGACY_HEAD_COLUMN not in before, "LEGACY_HEADSHIP_NAMESPACE_COLLISION" + ) + expected_axis = _headship_namespace_axis(before.columns) + expected_columns = list(expected_axis) + renamed_axis = after.columns + if entity == "person" and "age" not in before: + expected_columns.append("age") + renamed_axis = after.columns[:-1] + _require( + list(after) == expected_columns + and expected_axis.identical(renamed_axis) + and before.index.identical(after.index), + "NORMALIZED_ORDERED_COLUMNS", + ) + for column in before: + if entity == "person" and column == "age": + continue # Independently reconstructed from raw observations below. + target = ( + LEGACY_HEAD_COLUMN + if entity == "person" and column == "is_household_head" + else column + ) + left, right = before[column], after[target] + if isinstance(left.dtype, pd.StringDtype): + _require( + right.dtype == dtype_for_token("string"), + "NORMALIZED_STRING_DTYPE", + ) + _same_values(left, right) + else: + _require(storage_equal(left, right), "NORMALIZED_SOURCE_STORAGE") + _require( + original.strata.index.identical(normalized.strata.index) + and original.strata.name == normalized.strata.name + and storage_equal(original.strata, normalized.strata), + "NORMALIZED_STRATA", + ) + for entity in original.weighted_entities: + left, right = original.weights_for(entity), normalized.weights_for(entity) + _require( + left.kind is right.kind + and left.values.dtype == right.values.dtype + and left.values.tobytes() == right.values.tobytes(), + "NORMALIZED_WEIGHTS", + ) + expected = observed_age.normalize_observed_age( + original.person["A_AGE"], original.person.get("age") + ) + _require( + storage_equal(expected, normalized.person["age"]), "NORMALIZED_AGE_IDENTITY" + ) + + +def _bounded_append(rows, row, budget): + # A bound on the Python row list, not on a transport: one record per source + # household or per person, so a full-source roster is 1.02 GiB. The code and + # the per-row encode are unchanged; the ceiling is the explicit resource one. + size = len(_encode(row)) + 1 + _require(budget[0] + size <= MAX_ROSTER_BYTES, "ORIGIN_LIMIT") + budget[0] += size + rows.append(row) + + +def _origins(frame, sources, selected, native_receipts): + selected_by_key = {(r.key.source.value, r.key.native_id): r for r in selected} + native_hh, native_people = {}, {} + for row in native_receipts["asec"]["households"]: + native_hh[("asec", row["household_id"])] = row["source"]["H_SEQ"] + asec_rows = {row["person_id"]: row for row in native_receipts["asec"]["persons"]} + for channel, source in sources.items(): + for row in source.table("household").itertuples(index=False): + if channel == "acs": + native_hh[(channel, int(row.household_id))] = row.SERIALNO + for row in source.person.itertuples(index=False): + if channel == "acs": + person_key, line = str(row.SPORDER), str(row.SPORDER) + else: + original = asec_rows[int(row.person_id)] + person_key, line = original["PERIDNUM"], str(original["A_LINENO"]) + native_people[(channel, int(row.person_id))] = ( + person_key, + line, + int(row.person_household_id), + ) + maps, entities, households, people, budget = {}, {}, [], [], [0] + household_positions = {} + for channel, source in sources.items(): + ids = source.table("household").household_id + positions = {int(value): index for index, value in enumerate(ids)} + _require(len(positions) == len(ids), "SOURCE_ID_COLLISION") + household_positions[channel] = positions + for entity in frame.entities: + table = frame.table(entity) + ids = US_SCHEMA.entity_id_column(entity) + channels, previous = ( + support_channel_column(entity), + spine_source_id_column(entity), + ) + provenance = set(spine_assembly._support_metadata_columns(entity)) + expected_columns = provenance | { + column for source in sources.values() for column in source.table(entity) + } + _require(set(table) == expected_columns, "UNDECLARED_STACK_COLUMN") + records, mapping = [], {} + for new_id, channel, source_id in table[[ids, channels, previous]].itertuples( + index=False, name=None + ): + key = (channel, int(source_id)) + _require(key not in mapping, "SOURCE_ID_COLLISION") + mapping[key] = int(new_id) + _bounded_append(records, [int(new_id), channel, int(source_id)], budget) + expected = { + (channel, int(i)) + for channel, source in sources.items() + for i in source.table(entity)[ids] + } + _require(set(mapping) == expected, "COMPLETE_ENTITY_ORIGIN") + maps[entity], entities[entity] = mapping, records + for entity in frame.entities: + table = frame.table(entity) + id_column = US_SCHEMA.entity_id_column(entity) + for channel, source in sources.items(): + arm = table.loc[table[support_channel_column(entity)].eq(channel)] + old_ids = arm[spine_source_id_column(entity)].to_numpy() + original = ( + source.table(entity).set_index(id_column, drop=False).loc[old_ids] + ) + for column in original: + if column == id_column: + expected = [ + maps[entity][(channel, int(v))] for v in original[column] + ] + elif entity == "person" and column in { + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + }: + group = next( + e + for e in US_SCHEMA.group_entities + if US_SCHEMA.membership_column(e) == column + ) + expected = [ + maps[group][(channel, int(v))] for v in original[column] + ] + else: + expected = original[column] + _same_values(expected, arm[column]) + for column in ( + set(arm) + - set(original) + - set(spine_assembly._support_metadata_columns(entity)) + ): + _require(arm[column].isna().all(), "ABSENT_SOURCE_VALUE_INVENTED") + if entity == "person": + source_positions = { + int(value): index + for index, value in enumerate(source.person.person_id) + } + expected_strata = source.strata.iloc[ + [source_positions[int(value)] for value in old_ids] + ] + receiving_positions = np.flatnonzero( + table[support_channel_column(entity)].eq(channel).to_numpy() + ) + _same_values(expected_strata, frame.strata.iloc[receiving_positions]) + household_table = frame.table("household") + for position, (new_id, channel, source_id) in enumerate(entities["household"]): + raw = native_hh[(channel, source_id)] + chosen = selected_by_key.pop((channel, raw), None) + _require(chosen is not None, "SELECTED_SOURCE_IDENTITY") + original = chosen.original_design_weight + source = sources[channel] + source_position = household_positions[channel][source_id] + _require( + source.weights_for("household").kind is WeightKind.DESIGN + and frame.weights_for("household").kind is WeightKind.DESIGN, + "ORIGINAL_WEIGHT_KIND", + ) + expected = np.array([float(original)], dtype=np.float64).tobytes() + _require( + source.weights_for("household") + .values[source_position : source_position + 1] + .tobytes() + == expected + == frame.weights_for("household").values[position : position + 1].tobytes(), + "ORIGINAL_ANCHOR_CHANGED", + ) + _bounded_append( + households, + { + "household_id": new_id, + "source": channel, + "source_year": 2024, + "survey_year": 2024 if channel == "acs" else 2025, + "raw_native_id": raw, + "selected_receiving_household_id": source_id, + "original_anchor": _value(original), + }, + budget, + ) + _require( + not selected_by_key and len(households) == len(household_table), + "SELECTED_HOUSEHOLD_ROSTER", + ) + for new_id, channel, source_id in entities["person"]: + person_key, line, household_id = native_people[(channel, source_id)] + _bounded_append( + people, + [ + new_id, + channel, + 2024, + 2024 if channel == "acs" else 2025, + native_hh[(channel, household_id)], + person_key, + line, + source_id, + household_id, + maps["household"][(channel, household_id)], + ], + budget, + ) + return { + "households": households, + "entities": entities, + "persons": { + "columns": [ + "person_id", + "source", + "source_year", + "survey_year", + "raw_native_household_id", + "raw_native_person_id", + "native_line_numeric_original", + "selected_receiving_person_id", + "selected_receiving_household_id", + "household_id", + ], + "rows": people, + }, + } + + +# --------------------------------------------------------------------------- +# Verification epochs: validate once per run, and again in full at the end. +# +# Outside an epoch nothing below is reached and every capsule behaves exactly as +# it did before: one complete validation per accessor use. Inside an epoch each +# borrow still pays a cheap tier in full -- the live authority, the attached +# owner payloads and the producer encoding -- and the expensive tier is skipped +# only while a signature over every file those checks read, the whole source +# roster's stat identities included, and every live buffer they digest, is +# identical to the signature recorded when that tier last ran in full. The +# roster's stat identities are read into that signature and are deliberately +# not compared in the cheap tier: a signature that moved is a memo miss, not a +# refusal, so the complete validation runs and raises whatever it would have +# raised, with the same code, at the same borrow. Leaving the epoch re-validates +# every capsule it memoised, in full, with the memo bypassed, and records no +# memo answer for anything whose signature moved while that validation ran. +# --------------------------------------------------------------------------- + +_EPOCHS = [] +_MEMO = {} +_EPOCH_RECORD = None + + +def _array_witness(values): + """A read-free identity of one column's live storage. + + Buffer address, shape, strides, dtype and the writeable flag move whenever + a column is replaced, reindexed, retyped or rebound. An extension array + that exposes no numpy buffer contributes its object identity, which pandas + keeps stable for the arrays that have one. + """ + + data = getattr(values, "_data", None) + mask = getattr(values, "_mask", None) + if isinstance(data, np.ndarray) and isinstance(mask, np.ndarray): + return ("masked", _buffer_witness(data), _buffer_witness(mask)) + array = getattr(values, "_ndarray", None) + if isinstance(array, np.ndarray): + return ("array", _buffer_witness(array)) + if isinstance(values, np.ndarray): + return ("raw", _buffer_witness(values)) + return ("opaque", type(values).__name__, id(values), len(values)) + + +def _buffer_witness(array): + return ( + array.ctypes.data, + array.shape, + array.strides, + array.dtype.str, + array.flags.writeable, + ) + + +def _frame_witness(frame): + """A read-free structural identity of one Frame's live storage.""" + + if not isinstance(frame, Frame): + return ("not-a-frame", type(frame).__name__) + tables = [] + for entity in sorted(frame._tables): + table = frame._tables[entity] + columns = tuple(str(column) for column in table.columns) + tables.append( + ( + entity, + id(table), + len(table), + columns, + tuple(str(table[column].dtype) for column in table.columns), + tuple(_array_witness(table[column].array) for column in table.columns), + _array_witness(table.index), + id(table.index), + ) + ) + weights = tuple( + ( + entity, + frame._weights[entity].kind.value, + _array_witness(frame._weights[entity].values), + ) + for entity in sorted(frame._weights) + ) + strata = frame.strata + return ( + tuple(tables), + weights, + ( + id(strata), + len(strata), + str(strata.dtype), + strata.name, + _array_witness(strata.array), + _array_witness(strata.index), + ), + tuple(sorted(frame._link_tables)), + ) + + +def _verified_source_stats(state): + """Stat identities of every path the memoised tier re-reads, per owner.""" + + return ( + _file_stats(state.root), + acs_catalogue._verified_source_stats(state.catalogues[0]), + asec_catalogue._verified_source_stats(state.catalogues[1]), + acs_native._verified_source_stats(state.native[0]), + asec_native._verified_source_stats(state.native[1]), + ) + + +def _memo_signature(state): + """Everything cheap the memoised tier's answer can depend on.""" + + borrowed = (*state.catalogues, *state.native) + return ( + state.files, + _verified_source_stats(state), + tuple((id(value.payload), len(value.payload)) for value in borrowed), + tuple(id(value) for value in borrowed), + _frame_witness(state.frame), + tuple(_frame_witness(frame) for frame in state.source_frames), + asec_native._parent_frame_witness(state.native[1]), + ( + id(state.plan), + len(state.plan.selected), + len(state.plan.excluded), + len(state.plan.cells), + ), + ) + + +def _cheap_checks(state): + """The tier every borrow pays whether or not the expensive tier is skipped. + + The source roster's stat identities are read on every borrow, but they are + read into the *signature* and are deliberately not compared here. A moved + stat field is a memo miss, and the complete validation the miss runs raises + the refusal it raises today, with the same code, at the same borrow -- + which is exactly what this design promises. Comparing them here as well + would refuse first and put SOURCE_STAT_CHANGED in front of the code an + unmemoised borrow surfaces, so an appended archive would read as a moved + timestamp rather than as the ACS catalogue refusing its own bytes. + """ + + _require( + _live() == _LIVE and _authority() == state.authority, "FINAL_AUTHORITY_CHANGED" + ) + _require(_attached(state) == state.attached, "ATTACHED_EVIDENCE_CHANGED") + _require(_encode(_producer()) == state.producer, "PRODUCER_CHANGED") + + +def _memoized_validate(owner, state): + """Validate ``state`` unless this epoch already validated it unchanged.""" + + if not _EPOCHS: + _validate(state) + return + _cheap_checks(state) + try: + signature = _memo_signature(state) + except Exception: + # A signature that cannot even be taken -- a roster entry removed, or + # no longer a regular file -- is still a miss rather than a refusal of + # its own: the complete validation runs first, so the borrow raises + # today's code. `_validate` ends with the same `_file_stats` call and + # so cannot pass where this failed; re-raising is the fail-closed + # remainder, never the refusal a caller sees in practice. Only an + # error is handled here: an interrupt or a process exit landing in the + # signature is not a miss and must not buy a complete re-validation + # before it propagates. + _validate(state) + raise + entry = _MEMO.get(id(owner)) + if entry is not None and entry[0]() is owner and entry[1] == signature: + _EPOCH_RECORD["hits"] += 1 + return + _validate(state) + _MEMO[id(owner)] = (weakref.ref(owner), signature, state) + _EPOCH_RECORD["misses"] += 1 + + +def _signature_or_none(state): + """This state's signature, or ``None`` when one cannot be taken. + + ``None`` never equals a signature, so a close that cannot take one records + no memo answer and the borrow that follows runs the complete validation -- + which is what refuses, with today's code, for whatever made the signature + unavailable. At the outermost close no borrow follows, so ``_finalize_epoch`` + pays that validation itself rather than leaving it to one. + """ + + try: + return _memo_signature(state) + except Exception: # noqa: BLE001 - an absent signature is a miss, not a verdict + return None + + +def _finalize_epoch(): + """Re-validate every memoised capsule in full, with the memo bypassed. + + The refusal class is the one a borrow would have raised: this is the borrow + the epoch deferred, so a foreign owner's error is translated exactly as + ``_checked`` translates it rather than escaping raw to the run. + + An inner nested close keeps its memo -- ``_MEMO`` is cleared only when the + outermost epoch leaves -- so the signature recorded here is what the outer + epoch's next borrows are answered against, and it must not absorb anything + that moved while this close was validating. ``_validate`` compares the + roster stats and then runs a whole trailing ``_pure_final``; a stat that + moves in that window would become the new normal and be answered as a hit + until the outer close. The signature is therefore taken before validating + and again after, and when the two differ no memo answer is recorded at all, + so the next borrow is a miss and pays the complete validation -- which + raises the code it raises today, ``SOURCE_STAT_CHANGED`` for a touched + roster file. The entry itself stays: the outermost close re-validates every + capsule the memo still holds, and dropping it here would drop that pass for + a capsule nothing borrows again. + + Recording nothing is the whole answer only while a borrow can still follow. + At the **outermost** close there is none -- ``_MEMO`` is cleared as this + returns -- so leaving the deferred validation to the next borrow would end + the run without refusing at all. The outermost close therefore pays that + validation here, inside the same translation block, whenever the signature + moved or could not be taken: ``_validate`` re-reads the same roster and so + raises the same refusal the next borrow's miss would have raised, with no + code of this function's own. When it passes, nothing moved that the + complete validation can see, which is exactly the verdict a miss would have + reached, and the close continues. + """ + + for key, entry in list(_MEMO.items()): + owner = entry[0]() + if owner is None: + del _MEMO[key] + continue + before = _signature_or_none(entry[2]) + try: + _validate(entry[2]) + _EPOCH_RECORD["final_validations"] += 1 + after = _signature_or_none(entry[2]) + moved = before is None or after is None or after != before + if moved and not _EPOCHS: + # The outermost close: the borrow that would have paid for this + # never comes. Pay it here, so the run refuses on the way out + # instead of ending clean. Not counted as a final validation -- + # that count is of the unconditional pass above. + _validate(entry[2]) + except SurveyPopulationPreparationError: + raise + except Exception as error: + raise SurveyPopulationPreparationError( + "PREPARATION_VERIFICATION_REFUSED" + ) from error + _MEMO[key] = (entry[0], None if moved else after, entry[2]) + + +@contextmanager +def verification_epoch(*, join=False): + """Validate each native capsule once per run, and again in full at the end. + + The memo is opt-in and scoped: nothing outside this context manager changes + behaviour, which is why every existing borrow, refusal and trace-based + mutation test keeps its exact meaning. Leaving the epoch re-validates every + capsule it memoised in full; a capsule whose source or live storage moved + while a memo was warm therefore still makes the run refuse, before the + caller receives anything. + + The record it yields is this epoch's own receipt: how many borrows were + answered from the memo, how many re-ran the complete validation, and how + many unconditional final re-validations closed it. + + ``join=True`` reuses the innermost open epoch without closing or clearing + it. The caller must retain that outer scope until its result can escape: + changes invisible to the memo signature are refused at the outer close. + With no open epoch, joining creates and closes a normal epoch. The default + retains independent nested finalization. + """ + + global _EPOCH_RECORD + _require(type(join) is bool, "VERIFICATION_EPOCH_JOIN") + if join and _EPOCHS: + yield _EPOCH_RECORD + return + record = { + "protocol": PROTOCOL + "/verification-epoch/1", + "hits": 0, + "misses": 0, + "final_validations": 0, + "capsules": 0, + } + outer = _EPOCH_RECORD + _EPOCH_RECORD = record + _EPOCHS.append(record) + asec_native._epoch_enter() + failed = False + try: + yield record + except BaseException: + failed = True + raise + finally: + _EPOCHS.pop() + # The nested capsule's epoch closes first. At the outermost close its + # depth reaches zero and its memo is cleared, so this owner's own final + # validation reaches the nested population's complete file check; at an + # inner nested close the depth stays above zero and `_epoch_exit` + # refreshes the nested memo's signature instead -- or records none at + # all, if that signature moved while it was validating -- so this + # owner's validation is a memo hit there, or a miss that re-runs the + # nested capsule's complete check. Nothing is skipped in net either way, + # because the nested capsule's own exit has just re-validated it in + # full. Both refusals are collected and this owner's is preferred, + # because this owner's error class is the one every borrow through it + # raises; the other is chained onto it, never dropped. + nested = None + try: + asec_native._epoch_exit(failed) + except BaseException as error: # noqa: BLE001 - re-raised below + nested = error + own = None + try: + if not failed: + record["capsules"] = len(_MEMO) + _finalize_epoch() + except BaseException as error: # noqa: BLE001 - re-raised below + own = error + finally: + if not _EPOCHS: + _MEMO.clear() + _EPOCH_RECORD = outer + if not failed and (own is not None or nested is not None): + if own is not None and nested is not None: + raise own from nested + raise own if own is not None else nested + + +def epoch_record(): + """The innermost open epoch's record, or None outside an epoch.""" + + return _EPOCH_RECORD if _EPOCHS else None + + +@dataclass(frozen=True) +class _State: + root: Path + files: tuple + file_stats: tuple + producer: bytes + authority: bytes + catalogues: tuple + native: tuple + attached: tuple + source_frames: tuple + source_frame_seals: tuple + frame: Frame + identity: str + context: bytes + plan: selection.CatalogueSelectionPlan + plan_sha256: str + nested: tuple + acs_catalogue_memo: tuple | None = None + + +def _attached(state): + return tuple(value.payload for value in (*state.catalogues, *state.native)) + + +def _nested_seals( + catalogues, native, *, catalogue_memo=None, expected_catalogue_digest=None +): + """Pure final borrowed-object seals after all source/producer file checks. + + These inspect the actual retained issued owners, never construct issuer + authority from decoded records. The ancestor Frames still impose their + existing full-parent verification cost. + """ + acs = acs_native._owned(native[0]) + acs_cat = acs_catalogue._lookup(catalogues[0]) + values = [ + id(acs), + id(acs.prepared), + id(acs.prepared.source), + id(acs.literal), + acs.prepared.receipt_json, + acs.prepared.source.receipt_json, + acs.prepared.source.projection_json, + acs.literal.payload, + _encode( + [ + [str(path) for _, path in sorted(paths.items())] + for paths in acs.snapshots + ] + ), + _memoized_catalogue_digest( + (acs_cat.records, acs_cat.vacancies), + catalogue_memo, + expected_digest=expected_catalogue_digest, + ), + acs.prepared.construction_evidence_json, + ] + for module, value in ((asec_catalogue, catalogues[1]), (asec_native, native[1])): + entry = module._ISSUED.get(id(value)) + _require( + entry is not None and entry[0]() is value and entry[1] == value.payload, + "ANCESTOR_ISSUANCE_CHANGED", + ) + state = entry[2] + attached = asec_native._attached_evidence( + state.parent, state.coverage, state.anchors, state.fields + ) + values.extend( + ( + id(state), + id(state.parent), + id(attached[0]), + *attached[1:], + asec_native._frame_identity(state.parent.frame), + ) + ) + if module is asec_catalogue: + values.append(asec_catalogue._current_records_identity(state)) + return tuple(values) + + +def _pure_final(state): + _require( + _live() == _LIVE and _authority() == state.authority, "FINAL_AUTHORITY_CHANGED" + ) + _require(_attached(state) == state.attached, "ATTACHED_EVIDENCE_CHANGED") + _require( + _nested_seals( + state.catalogues, + state.native, + catalogue_memo=state.acs_catalogue_memo, + expected_catalogue_digest=state.nested[9], + ) + == state.nested, + "NESTED_EVIDENCE_CHANGED", + ) + _require(_digest(_plan_document(state.plan)) == state.plan_sha256, "PLAN_CHANGED") + _require( + _frame_identity(state.frame) == state.identity + and graph_context.encode_us_frame_context(state.frame) == state.context, + "PREPARED_FRAME_CHANGED", + ) + _require( + tuple( + ( + acs_native._frame_sha256(frame) + if i == 0 + else asec_native._frame_identity(frame) + ) + for i, frame in enumerate(state.source_frames) + ) + == state.source_frame_seals, + "SOURCE_FRAME_CHANGED", + ) + + +def _validate(state): + _pure_final(state) + acs_catalogue.verify_acs_source_catalogue(state.catalogues[0]) + asec_catalogue.verify_asec_source_catalogue(state.catalogues[1]) + acs_native.verify_acs_native_coverage(state.native[0], state.source_frames[0]) + state.native[1].validate() + _require(_source_files(state.root) == state.files, "SOURCE_CHANGED") + _require(_encode(_producer()) == state.producer, "PRODUCER_CHANGED") + _require(_file_stats(state.root) == state.file_stats, "SOURCE_STAT_CHANGED") + _pure_final(state) + + +@dataclass(frozen=True, slots=True, weakref_slot=True, eq=False) +class AuthenticatedSurveyPopulationPreparation: + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "ISSUANCE_CONSTRUCTOR") + + def _checked(self): + try: + entry = _ISSUED.get(id(self)) + _require( + type(self) is AuthenticatedSurveyPopulationPreparation + and entry is not None + and entry[0]() is self + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + _memoized_validate(self, entry[2]) + _require( + _ISSUED.get(id(self)) is entry + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + except SurveyPopulationPreparationError: + raise + except Exception as error: + raise SurveyPopulationPreparationError( + "PREPARATION_VERIFICATION_REFUSED" + ) from error + + def validate(self): + self._checked() + + def checked_view(self): + """Borrow one checked bundle; the view itself grants no authority.""" + _reference, payload, state = self._checked() + return CheckedSurveyPopulationView( + payload, state.context, state.frame, state.plan, json.loads(payload) + ) + + @property + def frame(self): + return self._checked()[2].frame + + @property + def context(self): + return self._checked()[2].context + + @property + def selection_plan(self): + return self._checked()[2].plan + + @property + def receipt(self): + return json.loads(self._checked()[1]) + + @property + def acs_spm_construction_evidence(self): + """Defensive native evidence; use receipt origins for assembled IDs. + + These records keep the selected native registry's IDs. The existing + per-entity origin map explicitly relates them to stacked Frame IDs. + A decoded copy is never a new source-authority capsule. + """ + state = self._checked()[2] + return state.native[0].construction_evidence + + def to_bytes(self): + return self._checked()[1] + + +@dataclass(frozen=True, slots=True) +class CheckedSurveyPopulationView: + """A checked borrow's values, not an issued source or reusable certificate.""" + + payload: bytes + context: bytes + frame: Frame + selection_plan: selection.CatalogueSelectionPlan + receipt: dict + + +def verify_survey_population_preparation(value): + _require( + type(value) is AuthenticatedSurveyPopulationPreparation, "PREPARATION_TYPE" + ) + value._checked() + return value + + +def verify_materialized_survey_population(preparation, frame): + _require( + type(preparation) is AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + _require(_frame_identity(frame) == entry[2].identity, "MATERIALIZED_FRAME_CHANGED") + _memoized_validate(preparation, entry[2]) + _require(_frame_identity(frame) == entry[2].identity, "MATERIALIZED_FRAME_CHANGED") + _require(preparation.payload == entry[1], "UNISSUED_OR_CHANGED") + + +def prepare_authenticated_survey_population( + source_dir, *, snapshot_root, fraction, seed, candidate=None +): + """Reconstruct actual source authority before accepting candidate bytes.""" + try: + _require(type(candidate) is bytes or candidate is None, "CANDIDATE_TYPE") + _require( + candidate is None or len(candidate) <= MAX_ROSTER_BYTES, "CANDIDATE_LIMIT" + ) + root, request, request_bytes, requested_fraction, requested_seed = _request( + source_dir + ) + spm_construction = _request_construction(request) + acs_native.housing._spm_implementation(spm_construction) + _require( + type(fraction) is Fraction + and fraction == requested_fraction + and type(seed) is int + and seed == requested_seed, + "REQUEST_ARGUMENTS", + ) + snapshots = _root(snapshot_root, allow_missing=True) + _require( + not snapshots.is_relative_to(root) and not root.is_relative_to(snapshots), + "SNAPSHOT_LOCATION", + ) + snapshots.mkdir(parents=True, exist_ok=True) + authority, files_before, producer = ( + _authority(), + _source_files(root), + _encode(_producer()), + ) + _require( + files_before[0] + == ("selection-request.json", len(request_bytes), _sha(request_bytes)), + "REQUEST_CHANGED", + ) + file_stats = _file_stats(root) + kwargs = dict( + parent_path=root / "asec/parent.h5", + household_attachment_path=root / "asec/household-attachment.h5", + person_income_attachment_path=root / "asec/person-income-attachment.h5", + person_member_paths={ + 2022: root / "asec/pppub23.csv", + 2023: root / "asec/pppub24.csv", + 2024: root / "asec/pppub25.csv", + }, + household_member_path=root / "asec/hhpub25.csv", + ) + acs = acs_catalogue.issue_acs_source_catalogue( + root / "acs", snapshot_root=snapshots + ) + asec = asec_catalogue.issue_asec_source_catalogue(**kwargs) + acs_rows, asec_rows = acs.households, asec.households + plan = selection.plan_catalogue_selection( + acs_households=acs_rows, + asec_households=asec_rows, + fraction=fraction, + seed=seed, + ) + acs_keys = tuple( + r.key.native_id for r in plan.selected if r.key.source is domains.Source.ACS + ) + asec_keys = tuple( + (2024, int(r.key.native_id)) + for r in plan.selected + if r.key.source is domains.Source.ASEC + ) + _require( + acs_keys and asec_keys and len(set(asec_keys)) == len(asec_keys), + "SELECTED_SOURCE_SUPPORT", + ) + actual_acs = acs_native.issue_acs_native_coverage( + root / "acs", + snapshot_root=snapshots, + serialnos=acs_keys, + spm_construction=spm_construction, + ) + actual_asec = asec_native.load_authenticated_asec_2024_native_population( + **kwargs, selected_households=asec_keys + ) + native_frames = actual_acs.frame, actual_asec.frame + source_seals = ( + acs_native._frame_sha256(native_frames[0]), + asec_native._frame_identity(native_frames[1]), + ) + source_copies = { + channel: _normalized_source_copy(frame) + for channel, frame in zip(("acs", "asec"), native_frames, strict=True) + } + stacked = spine_assembly.stack_survey_spines(source_copies) + frame = stacked.frame + transitions = graph_sources._canonical_assembly( + frame, tuple(source_copies.values()) + ) + context = graph_context.encode_us_frame_context(frame) + native_receipts = {"acs": actual_acs.receipt, "asec": actual_asec.receipt} + for channel, original in zip(("acs", "asec"), native_frames, strict=True): + _verify_normalized_copy(original, source_copies[channel]) + origins = _origins(frame, source_copies, plan.selected, native_receipts) + origins["storage_transitions"] = list(transitions) + origins["legacy_headship_namespace"] = { + "rule": LEGACY_HEAD_RULE, + "from": "is_household_head", + "to": LEGACY_HEAD_COLUMN, + "values_changed": False, + "canonical_authority": False, + "sources_present": { + channel: "is_household_head" in native_frames[i].person + for i, channel in enumerate(("acs", "asec")) + }, + } + origins["observed_age_normalization"] = { + "rule": observed_age.rule_document(), + "sources": { + channel: { + "native_frame_sha256": source_seals[i], + "normalized_frame_sha256": _frame_identity(source_copies[channel]), + "common_age_preexisting": "age" in native_frames[i].person, + } + for i, channel in enumerate(("acs", "asec")) + }, + } + documents = {"acs": acs.receipt, "asec": asec.receipt} + catalogues = { + "acs": { + "receipt_sha256": _sha(acs.to_bytes()), + "records_sha256": documents["acs"]["counts"]["canonical_record_sha256"], + "counts": documents["acs"]["counts"], + }, + "asec": { + "receipt_sha256": _sha(asec.to_bytes()), + "records_sha256": documents["asec"]["records"]["sha256"], + "counts": documents["asec"]["counts"], + }, + } + identity = _frame_identity(frame) + payload, _ = _roster_payload( + { + "protocol": PROTOCOL, + "request": request, + "request_sha256": _sha(request_bytes), + "source_files": files_before, + "producer": json.loads(producer), + "catalogues": catalogues, + "native": { + channel: { + "receipt_sha256": _sha(value.payload), + "frame_sha256": source_seals[i], + "households": native_frames[i].n("household"), + "persons": native_frames[i].n("person"), + } + for i, (channel, value) in enumerate( + (("acs", actual_acs), ("asec", actual_asec)) + ) + }, + "selection": _plan_document(plan), + "origins": origins, + "frame_sha256": identity, + "context_sha256": _sha(context), + "release_eligible": False, + }, + spill=snapshots, + name="preparation", + ) + nested = _nested_seals((acs, asec), (actual_acs, actual_asec)) + acs_owned = acs_catalogue._lookup(acs) + catalogue_memo = _catalogue_memo( + (acs_owned.records, acs_owned.vacancies), nested[9] + ) + state = _State( + root, + files_before, + file_stats, + producer, + authority, + (acs, asec), + (actual_acs, actual_asec), + tuple(value.payload for value in (acs, asec, actual_acs, actual_asec)), + native_frames, + source_seals, + frame, + identity, + context, + plan, + _digest(_plan_document(plan)), + nested, + catalogue_memo, + ) + _validate(state) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + result = AuthenticatedSurveyPopulationPreparation(payload, _token=_TOKEN) + key = id(result) + + def cleanup(reference): + entry = _ISSUED.get(key) + if entry is not None and entry[0] is reference: + del _ISSUED[key] + + _ISSUED[key] = (weakref.ref(result, cleanup), payload, state) + _pure_final(state) + _require( + type(result.payload) is bytes + and result.payload == payload + and _ISSUED[key][0]() is result, + "UNISSUED_OR_CHANGED", + ) + return result + except SurveyPopulationPreparationError: + raise + except Exception as error: + # The code stays static; the cause is chained so a refused run names + # what refused instead of only that something did. + raise SurveyPopulationPreparationError( + "PREPARATION_ISSUANCE_REFUSED" + ) from error + + +_BYTES = _code_bytes() + + +def _current_survey_wage_projection(preparation, entry): + """Project one just-checked retained owner; bytes grant no source authority. + + The caller must have obtained ``entry`` from ``preparation._checked()`` in + the current operation, and must seal its receiving Populations afterwards. + The one ready() call retains the existing all-field/full-parent admission. + No old reported-income contract or prior-wage column is manufactured. + """ + _require( + type(preparation) is AuthenticatedSurveyPopulationPreparation + and _ISSUED.get(id(preparation)) is entry + and entry[0]() is preparation + and type(preparation.payload) is bytes + and preparation.payload == entry[1], + "WAGE_PREPARATION_ISSUANCE", + ) + state = entry[2] + native_entry = asec_native._ISSUED.get(id(state.native[1])) + _require( + native_entry is not None + and native_entry[0]() is state.native[1] + and state.native[1].payload == native_entry[1], + "WAGE_NATIVE_ISSUANCE", + ) + parent = native_entry[2].parent + ready = parent.ready() # Exactly once, outside both row and feature loops. + field = ready.field("WSAL_VAL") + domain = next(d for d in parent.spec.fields if d.name == "WSAL_VAL") + scope = parent.scope + positions = {pid: i for i, pid in enumerate(scope.person_ids)} + _require(len(positions) == len(scope.person_ids), "WAGE_PARENT_ROSTER") + native_ids = set(state.source_frames[1].person.person_id) + rows, budget = [], [0] + people = state.frame.person + selected = people.loc[people[support_channel_column("person")].eq("asec")] + _require( + set(selected[spine_source_id_column("person")]) == native_ids, + "WAGE_SELECTED_ROSTER", + ) + for stacked, native in selected[ + ["person_id", spine_source_id_column("person")] + ].itertuples(index=False, name=None): + _require(int(native) in positions, "WAGE_PARENT_MEMBER") + i = positions[int(native)] + _require(scope.person_years[i] == 2024, "WAGE_CURRENT_COHORT") + amount = field.amount_bytes[8 * i : 8 * (i + 1)] + number = np.frombuffer(amount, dtype="= 0, + "WAGE_CURRENT_FEATURE", + ) + _bounded_append( + rows, + [ + int(stacked), + int(native), + 2024, + amount.hex(), + field.status_bytes[i], + field.validity_bytes[i], + field.zero_origin_bytes[i], + ], + budget, + ) + document = { + "preparation_sha256": _sha(entry[1]), + "asec_native_sha256": _sha(native_entry[1]), + "money_header": json.loads(ready.header), + "money_header_sha256": _sha(ready.header), + "domain": {f.name: getattr(domain, f.name) for f in fields(domain)}, + "zero_origin_code": int(current_money._origin_code(parent.spec, "WSAL_VAL")), + "columns": [ + "stacked_person_id", + "native_person_id", + "income_year", + "amount_f64le_hex", + "status", + "validity", + "zero_origin", + ], + "rows": rows, + } + # One row per selected wage earner, so this is a whole-roster receipt too. + payload, _ = _roster_payload(document) + # ready() may perform I/O. Recheck the actual retained entries and all pure + # owner seals after it; this does not reread the national catalogues. + _pure_final(state) + _require( + _ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1] + and asec_native._ISSUED.get(id(state.native[1])) is native_entry + and state.native[1].payload == native_entry[1] + and native_entry[2].parent is parent, + "WAGE_FINAL_ISSUANCE", + ) + return payload + + +_LIVE = _live() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py new file mode 100644 index 000000000..dd55a5e65 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py @@ -0,0 +1,208 @@ +"""Directional ContentStore replay checks, never source or population authority. + +The caller must separately retain the original source/population seals. These +comparisons permit the store's missing-buffer normalization and scalar object +encoding only at a named materialization boundary. They are not mutation seals. +""" + +from __future__ import annotations + +from dataclasses import asdict + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, MassChangeRecord, WeightKind +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import MassRecord, Population +from microcosm.graph.store import ( + _axis_name_payload, + _encode_frame_metadata, + _encode_object_scalar, +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_POPULATION_REPLAY_" + reason) + + +def _array_bytes_equal(left, right): + return ( + type(left) is np.ndarray + and type(right) is np.ndarray + and left.dtype == right.dtype + and not left.dtype.hasobject + and left.shape == right.shape + and left.tobytes() == right.tobytes() + ) + + +def _object_bytes(value): + try: + return _encode_object_scalar(value) + except (TypeError, ValueError, OverflowError, UnicodeError): + raise ValueError("SURVEY_POPULATION_REPLAY_UNSUPPORTED_OBJECT") from None + + +def _series(expected, actual): + _require( + type(expected) is pd.Series + and type(actual) is pd.Series + and type(expected.dtype) is type(actual.dtype) + and expected.dtype == actual.dtype + and len(expected) == len(actual), + "SERIES_DTYPE_OR_LENGTH", + ) + dtype = expected.dtype + left, right = expected.array, actual.array + if isinstance(dtype, pd.BooleanDtype) or ( + isinstance(dtype, pd.api.extensions.ExtensionDtype) + and pd.api.types.is_integer_dtype(dtype) + ): + # The v1 country profile supports pandas' actual masked integer/bool + # storage. Other extension-array implementations must be reviewed first. + ld, rd = getattr(left, "_data", None), getattr(right, "_data", None) + lm, rm = getattr(left, "_mask", None), getattr(right, "_mask", None) + _require( + _array_bytes_equal(lm, rm) + and lm.dtype == np.dtype(np.bool_) + and lm.shape == (len(expected),) + and type(ld) is np.ndarray + and type(rd) is np.ndarray + and ld.dtype == rd.dtype + and not ld.dtype.hasobject + and ld.shape == rd.shape == lm.shape, + "MASKED_STORAGE", + ) + _require(_array_bytes_equal(ld[~lm], rd[~rm]), "PRESENT_BITS") + _require( + _array_bytes_equal(ld, rd) or bool(np.all(rd[rm] == 0)), + "NONCANONICAL_NULL_BACKING", + ) + return + if isinstance(dtype, pd.StringDtype): + _require( + dtype.storage == actual.dtype.storage + and (dtype.na_value is pd.NA) == (actual.dtype.na_value is pd.NA), + "STRING_POLICY", + ) + lm = expected.isna().to_numpy(dtype=np.bool_, copy=False) + rm = actual.isna().to_numpy(dtype=np.bool_, copy=False) + _require(_array_bytes_equal(lm, rm), "STRING_MASK") + for missing, a, b in zip(lm, expected, actual, strict=True): + if not missing: + _require(type(a) is str and type(b) is str and a == b, "STRING_VALUE") + return + if pd.api.types.is_object_dtype(dtype): + # Reuse the store's typed scalar bytes. None/pd.NA/NaT, bool/int, + # float signed zero and NaN payloads remain distinguished by that codec. + # Object pointers and np scalar wrappers are not portable identities. + for a, b in zip(expected, actual, strict=True): + _require(_object_bytes(a) == _object_bytes(b), "OBJECT_VALUE") + return + _require( + not isinstance(dtype, pd.api.extensions.ExtensionDtype), + "UNSUPPORTED_EXTENSION_DTYPE", + ) + _require( + _array_bytes_equal(expected.to_numpy(copy=False), actual.to_numpy(copy=False)), + "NATIVE_BITS", + ) + + +def _name_bytes(value): + try: + return canonical_json(_axis_name_payload(value)) + except (TypeError, ValueError, OverflowError, UnicodeError): + raise ValueError("SURVEY_POPULATION_REPLAY_UNSUPPORTED_AXIS_NAME") from None + + +def _axis(expected, actual): + _require( + type(expected) is type(actual) + and expected.identical(actual) + and not isinstance(expected, pd.MultiIndex), + "AXIS", + ) + _require(_name_bytes(expected.name) == _name_bytes(actual.name), "AXIS_NAME") + _series(pd.Series(expected.array, copy=False), pd.Series(actual.array, copy=False)) + + +def same_replayed_frame(expected: Frame, actual: Frame) -> None: + """Compare actual Frames, allowing only the named store representation rule. + + No I/O or allocation of another Frame. Temporary buffers are proportional + to individual columns and context metadata, without a total input RAM cap. + """ + _require(isinstance(expected, Frame) and isinstance(actual, Frame), "FRAME_TYPE") + _require( + expected.schema == actual.schema + and expected.entities == actual.entities + and expected.links == actual.links == () + and canonical_json(_encode_frame_metadata(expected.metadata)) + == canonical_json(_encode_frame_metadata(actual.metadata)) + and all( + type(r) is MassChangeRecord for r in (*expected.mass_log, *actual.mass_log) + ) + and canonical_json([asdict(r) for r in expected.mass_log]) + == canonical_json([asdict(r) for r in actual.mass_log]) + and expected.weighted_entities == actual.weighted_entities, + "FRAME_CONTEXT", + ) + for entity in expected.entities: + left, right = expected.table(entity), actual.table(entity) + _require( + type(left) is pd.DataFrame + and type(right) is pd.DataFrame + and left.flags == right.flags, + "TABLE_TYPE_OR_FLAGS", + ) + _axis(left.index, right.index) + _axis(left.columns, right.columns) + for column in left: + _series(left[column], right[column]) + _axis(expected.strata.index, actual.strata.index) + _require( + _name_bytes(expected.strata.name) == _name_bytes(actual.strata.name), + "STRATA_NAME", + ) + _series(expected.strata, actual.strata) + for entity in expected.weighted_entities: + left, right = expected.weights_for(entity), actual.weights_for(entity) + _require( + left.kind is right.kind and _array_bytes_equal(left.values, right.values), + "WEIGHT_BYTES", + ) + + +def same_replayed_population(expected: Population, actual: Population) -> None: + """Require complete receiving state as well as the directional Frame check.""" + _require( + type(expected) is Population and type(actual) is Population, "POPULATION_TYPE" + ) + same_replayed_frame(expected.frame, actual.frame) + _require( + type(expected.version) is type(actual.version) is str + and expected.version == actual.version + and tuple(sorted(expected.owners.items())) + == tuple(sorted(actual.owners.items())) + and all(type(v) is str for v in actual.owners.values()) + and tuple(expected.weight_kind.items()) == tuple(actual.weight_kind.items()) + and all(type(v) is WeightKind for v in actual.weight_kind.values()) + and type(expected.mass_ledger) is type(actual.mass_ledger) is tuple + and all( + type(r) is MassRecord for r in (*expected.mass_ledger, *actual.mass_ledger) + ) + and canonical_json([asdict(r) for r in expected.mass_ledger]) + == canonical_json([asdict(r) for r in actual.mass_ledger]) + and tuple(expected.design_weights) == tuple(actual.design_weights), + "POPULATION_CONTEXT", + ) + for entity in expected.design_weights: + _require( + _array_bytes_equal( + expected.design_weights[entity], actual.design_weights[entity] + ), + "DESIGN_BYTES", + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py new file mode 100644 index 000000000..adf73f3ab --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py @@ -0,0 +1,176 @@ +"""Explicit source-report Social Security component basis. + +These are numerical owners, not source admission APIs. A graph host must bind +the live source projection and any actual fitted completion artifact before +using the returned values. Missing components never become zero. Report-record totals are not individual +beneficiary incidence. This module neither replaces totals with PUF draws nor +allocates combined family payments to individual beneficiaries. +""" + +from __future__ import annotations + +import numpy as np + +COMPONENTS = ( + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", +) +PROTOCOL = "microcosm.us.survey-social-security-basis.v1" +REASON_COMPONENTS = { + 1: (0,), + 2: (1,), + 3: (3,), + 4: (2,), + 5: (3,), + 6: (2,), + # The published code mixes surviving, dependent and disabled children. + 7: (1, 2, 3), + 8: (0, 1, 2, 3), +} + + +def require(condition, reason): + if not condition: + raise ValueError("SURVEY_SOCIAL_SECURITY_" + reason) + + +def _float64(values, *, shape, nullable=False): + require(type(values) is np.ndarray, "ARRAY_TYPE") + require(values.dtype == np.dtype("float64") and values.shape == shape, "ARRAY_AXIS") + require(not np.isinf(values).any(), "NONFINITE") + require(nullable or not np.isnan(values).any(), "UNKNOWN") + require((values[~np.isnan(values)] >= 0).all(), "NEGATIVE") + return values.copy() + + +def asec_reason_basis(total, reason_1, reason_2): + """Map a source total and reasons without inventing component amounts. + + Callers qualify the actual money and literal reason observations. Codes + 3/5 share survivors and 4/6 share dependents. Multiple distinct components, + code 7, code 8, or missing reasons leave the positive amount unallocated. + A single component receives the total as an explicit mapping judgment. + No age fallback or priority between reported reasons is applied. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + reasons = [] + for values in (reason_1, reason_2): + arr = _float64(values, shape=total.shape, nullable=True) + require( + ( + (arr[np.isfinite(arr)] == np.floor(arr[np.isfinite(arr)])) + & (arr[np.isfinite(arr)] <= 8) + ).all(), + "REASON_DOMAIN", + ) + reasons.append(arr) + out = np.full((len(amount), 4), np.nan, dtype=np.float64) + allowed = np.ones((len(amount), 4), dtype=np.bool_) + labels = [] + for i, value in enumerate(amount): + if value == 0: + out[i] = 0 + allowed[i] = False + labels.append("known_total_zero") + continue + codes = [arr[i] for arr in reasons] + if any(np.isnan(code) for code in codes): + labels.append("unresolved_missing_reason") + continue + nonzero = [int(code) for code in codes if code != 0] + if not nonzero: + labels.append("unresolved_niu_reason_with_positive_total") + continue + possible = set().union(*(REASON_COMPONENTS[code] for code in nonzero)) + allowed[i] = False + allowed[i, list(possible)] = True + if len(possible) == 1: + out[i] = 0 + out[i, next(iter(possible))] = value + labels.append("source_reason_total_allocation") + else: + labels.append("unresolved_component_split") + return out, allowed, tuple(labels) + + +def asec_reporting_basis(total, age, recipiency, reason_1, reason_2): + """Respect the age-15 reporting universe without inferring beneficiaries. + + SS_VAL=0 outside the question universe is a source sentinel, not observed + absence of benefits. Retain that literal upstream and return unknown total + and shares here. A reporting adult may receive combined family payments; + the returned component basis remains at that source report's grain. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + ages = _float64(age, shape=total.shape) + receipt = _float64(recipiency, shape=total.shape, nullable=True) + require(((ages == np.floor(ages)) & (ages <= 99)).all(), "AGE_DOMAIN") + require( + np.isin(receipt[np.isfinite(receipt)], [0, 1, 2]).all(), "RECIPIENCY_DOMAIN" + ) + eligible = ages >= 15 + require( + ((amount[~eligible] == 0) & (receipt[~eligible] == 0)).all(), + "OUTSIDE_REPORTING_UNIVERSE_OBSERVATION", + ) + require(np.isin(receipt[eligible], [1, 2]).all(), "IN_UNIVERSE_RECIPIENCY_UNKNOWN") + require(not ((amount > 0) & (receipt != 1)).any(), "RECIPIENCY_CONTRADICTION") + basis, allowed, labels = asec_reason_basis(amount, reason_1, reason_2) + amount[~eligible] = np.nan + basis[~eligible] = np.nan + allowed[~eligible] = True + labels = tuple( + label if inside else "asec_below15_outside_reporting_universe" + for label, inside in zip(labels, eligible, strict=True) + ) + return amount, basis, allowed, labels + + +def complete_positive_basis(total, source_basis, allowed, modeled_scores): + """Convert actual model scores to shares only for unresolved positive rows. + + The graph owner must authenticate the fitted artifact. This numerical API + cannot do so. Existing complete rows must have no modeled scores; eligible + rows require a complete nonnegative four-vector with positive allowed mass. + Source totals and excluded component possibilities are retained exactly. + A missing/zero model vector refuses instead of creating equal shares. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + shape = (len(amount), 4) + basis = _float64(source_basis, shape=shape, nullable=True) + scores = _float64(modeled_scores, shape=shape, nullable=True) + require( + type(allowed) is np.ndarray + and allowed.dtype == np.dtype("bool") + and allowed.shape == shape, + "ALLOWED_AXIS", + ) + complete = np.isfinite(basis).all(axis=1) + require((complete | np.isnan(basis).all(axis=1)).all(), "PARTIAL_COMPONENT_BASIS") + require(np.isnan(scores[complete]).all(), "MODELED_SOURCE_OVERWRITE") + require( + np.array_equal(basis[complete].sum(axis=1), amount[complete]), + "SOURCE_TOTAL_IDENTITY", + ) + require((basis[complete][~allowed[complete]] == 0).all(), "SOURCE_SUPPORT") + missing = ~complete + require((amount[missing] > 0).all(), "UNRESOLVED_ZERO_TOTAL") + require(np.isfinite(scores[missing]).all(), "COMPLETION_UNKNOWN") + selected = scores[missing].copy() + selected[~allowed[missing]] = 0 + mass = selected.sum(axis=1) + require(np.isfinite(mass).all() and (mass > 0).all(), "COMPLETION_NO_ALLOWED_MASS") + filled = selected / mass[:, None] * amount[missing, None] + # Put the floating residual on the largest supported component. This keeps + # excluded components exactly zero and avoids systematic lost pennies. + if len(filled): + j = filled.argmax(axis=1) + filled[np.arange(len(filled)), j] += amount[missing] - filled.sum(axis=1) + require(np.isfinite(filled).all() and (filled >= 0).all(), "COMPLETION_ARITHMETIC") + basis[missing] = filled + return basis diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/target_geography_view.py b/packages/microcosm-build/src/microcosm/build/us_runtime/target_geography_view.py new file mode 100644 index 000000000..303e4b761 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/target_geography_view.py @@ -0,0 +1,413 @@ +"""The national/state/CD view axis for US calibration-target comparisons. + +``native_survey_handoff.REQUIRED_RELEASE_EVIDENCE`` names +``national_and_cd_target_fit`` as required release evidence, and no producer +existed: the two-artifact yardstick +(:mod:`tools.score_us_release_head_to_head`) emitted ``family``, ``entity``, +``value_basis`` and ``period`` per row but no geographic identifier, so a +national-versus-congressional-district view of a head-to-head could not be +computed from the scorecard even after a real run. This module is that +missing classification and nothing else. + +It decides **nothing**. There is no threshold, tolerance, band, verdict or +promotion rule here; the comparison stays the owner's flip decision. The one +judgment encoded is refusal: a row whose geographic scope no declared evidence +identifies is reported as :data:`UNRESOLVED_GEOGRAPHY_VIEW_LEVEL`, never +folded into the national view. A silently national-labelled CD row would make +the national view look better than the artifact is. + +The level vocabulary is the one the native measurement kernel already +advertises for its household scopes --- ``national``/``state``/ +``congressional_district`` +(:mod:`microcosm.build.us_runtime.graph_fiscal_measurement`, ``_LEVELS`` and +``FiscalGeographyScope``) --- so the measurement side and the comparison side +name the same three views. That kernel is frozen for the active replay and is +not edited here; ``test_us_release_head_to_head_scorer`` pins the two +vocabularies equal instead. + +Ledger facts spell the national level ``country``. The alias table below is +the same rename the US fiscal target compiler already performs when it turns a +ledger geography level into a target's ``geography_scope`` +(``us_runtime/fiscal_targets.py:2816-2826``). + +Congressional-district membership additionally has a shared union-of-evidence +classifier, :func:`microcosm.data.us_critical_targets. +is_congressional_district_target`, which the release builder uses +(``tools/build_us_fiscal_refresh_release.py:7136-7139``). It is reused here +rather than restated: as the fallback when no explicit level is declared, and +as a cross-check that is *reported* — never silently preferred — when it +disagrees with an explicit level. + +Identifiers +----------- + +Level and identifier are not independent choices, and reading them +independently names the wrong area. The compiler stamps a district's *parent* +state on congressional-district rows (``fiscal_targets.py:2825-2830`` derives +``state_fips`` as ``congressional_district_geoid[:2]``), and a hierarchy +geography can sit at a level this axis does not advertise as a view at all. So +an identifier is read only from a declaration whose *own* declared level is +the row's resolved level, in the order below; when no declaration at that +level carries one the identifier stays empty with +:data:`UNBOUND_GEOGRAPHY_ID_SOURCE`, never substituted from another level. + +The US compiler supplies hierarchy seeds and requires nonempty, consistent +fact geography. Those checks do not validate an identifier's encoding: both +hierarchy and ledger fields can still contain bare or unrecognized text. +Canonical counts therefore validate the bound value at its resolved level. +No actual registry has been evaluated by this classifier's invented tests. + +Two identifier encodings exist, and this module treats exactly one of them as +the canonical name of an area: + +* **canonical** --- a validated prefixed census GEOID (``0100000US``, ``0400000US06``, + ``5001900US0601``) that the ledger compiler copies verbatim from the + Chronicle fact into both ``hierarchy.geography.id`` + (``ledger_targets.py:947,978-986``) and ``metadata['ledger_geography_id']`` + (``ledger_targets.py:3232-3233``). Because those two are one fact field read + twice --- and ``_calibration_hierarchy`` already raises unless the member + facts agree on ``(level, id)`` (``ledger_targets.py:934-947``) --- they are + comparable to each other, and + :attr:`TargetGeographyView.geography_id_declarations_conflict` is the + defensive assertion that their text agrees. On the compiled path it is expected + to be false for every row; a true value means a producer bypassed that + constructor. +* **bare** --- ``state_fips`` (``fiscal_targets.py:3452-3459``) and + ``congressional_district_geoid`` (``fiscal_targets.py:3462-3477``), which + the US target compiler derives from the canonical GEOID by stripping its + summary-level prefix. This is a restatement, not a second declaration of an + area, so it is never compared against a canonical id and never counted as a + conflict. + +Bare values can also occur in the hierarchy and ledger fields; the source +field name alone never establishes the encoding. Validation reuses the fiscal +target compiler's state/CD parsers and the national activation's identifier. +Unrecognized nonempty values stay visible in their own count, outside both +canonical and bare counts. Values are never rewritten. + +The bare form is **not** an interchangeable spelling, and this module makes no +claim that it is. Stripping is lossless for a state (one prefix, +``0400000US``), but lossy for a congressional district: both the 117th-Congress +``5001700US`` and the current ``5001900US`` prefix strip to the same four +digits (``congressional_district_vintage.py:19-20``), and the packaged vintage +crosswalk splits ``5001700US0101`` across three different current districts. +So a bare-bound row's area is not comparable to a canonically-bound row's, and +mapping between them would need a prefix rule this module deliberately does not +own --- ``"0400000US"`` is an unnamed literal repeated across four production +modules today, with no shared constant to import. + +Consequence for the scorecard, and the reason the binding alone is not enough: +an identifier bound from the bare form must never enter a distinct-area count +beside canonical ones, or one area would be counted twice under two spellings +--- the same wrong count the level binding exists to prevent, one step along. +:attr:`TargetGeographyView.geography_id_is_canonical` is how a consumer keeps +the two apart; ``score_us_release_head_to_head`` counts distinct *canonical* +ids and reports bare-bound and unbound rows as their own counts beside it. +Every declaration at the level stays visible in +:attr:`TargetGeographyView.declared_geography_ids`, merged with nothing. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + +from microcosm.build.us_runtime.fiscal_targets import ( + _congressional_district_geoid, + _state_fips, +) +from microcosm.build.us_runtime.national_age_activation import NATIONAL_GEOGRAPHY_ID +from microcosm.calibrate.geography_constants import US_STATE_FIPS_TO_POSTAL +from microcosm.data.us_critical_targets import is_congressional_district_target + +__all__ = [ + "GEOGRAPHY_VIEW_LEVEL_ALIASES", + "UNBOUND_GEOGRAPHY_ID_SOURCE", + "UNRESOLVED_GEOGRAPHY_VIEW_LEVEL", + "US_TARGET_GEOGRAPHY_VIEW_LEVELS", + "US_TARGET_GEOGRAPHY_VIEW_ORDER", + "TargetGeographyView", + "us_target_geography_view", + "us_target_spec_geography_view", +] + +#: The three household scopes the native fiscal measurement kernel advertises. +US_TARGET_GEOGRAPHY_VIEW_LEVELS: tuple[str, ...] = ( + "national", + "state", + "congressional_district", +) + +#: A row whose scope no declared evidence identifies. Never a silent national. +UNRESOLVED_GEOGRAPHY_VIEW_LEVEL = "unresolved" + +#: No declaration at the row's resolved level carries an identifier. +UNBOUND_GEOGRAPHY_ID_SOURCE = "none" + +#: Rendering/rollup order: the declared views first, refusals last. +US_TARGET_GEOGRAPHY_VIEW_ORDER: tuple[str, ...] = ( + *US_TARGET_GEOGRAPHY_VIEW_LEVELS, + UNRESOLVED_GEOGRAPHY_VIEW_LEVEL, +) + +#: Ledger spells the national level ``country``; the compiler already renames +#: it (``us_runtime/fiscal_targets.py:2816-2826``). Only that one rename is +#: declared: an unknown level stays unknown rather than being guessed into a +#: view. +GEOGRAPHY_VIEW_LEVEL_ALIASES: Mapping[str, str] = {"country": "national"} + +_EXPLICIT_LEVEL_METADATA_KEYS = ("ledger_geography_level", "geography_scope") + +#: The two sources that carry the Chronicle fact's ``geography.id`` verbatim, +#: in binding order. Same field, read twice, so they are comparable to each +#: other; the compiler already refuses to let them diverge +#: (``ledger_targets.py:934-952``), which is what makes a difference between +#: them an invariant violation rather than an encoding difference. +_CANONICAL_GEOGRAPHY_ID_SOURCES = ("hierarchy_geography", "ledger_geography_id") + +#: The bare, prefix-stripped restatement of that same identifier, one key per +#: level it can name. ``state_fips`` names a state wherever it appears --- on a +#: district row it is the district's parent (``fiscal_targets.py:2825-2830``) +#: --- so it is only ever an identifier for the level that owns it. +_LEVEL_OWNED_ID_METADATA_KEYS: Mapping[str, str] = { + "state": "state_fips", + "congressional_district": "congressional_district_geoid", +} + + +@dataclass(frozen=True) +class TargetGeographyView: + """One target row's comparison view, with the evidence that named it.""" + + level: str + geography_id: str + #: Which evidence named ``level``; ``"none"`` when nothing did. + level_source: str + #: The shared CD classifier's independent reading of the same row. + congressional_district_evidence: bool + #: Which declaration ``geography_id`` was read from; + #: :data:`UNBOUND_GEOGRAPHY_ID_SOURCE` when no declaration at ``level`` + #: carried one. + geography_id_source: str = UNBOUND_GEOGRAPHY_ID_SOURCE + #: Every ``(source, identifier)`` declared *at* ``level``, in binding + #: order. Kept whole rather than merged. + declared_geography_ids: tuple[tuple[str, str], ...] = () + + @property + def resolved(self) -> bool: + return self.level != UNRESOLVED_GEOGRAPHY_VIEW_LEVEL + + @property + def geography_id_bound(self) -> bool: + """An identifier was read from a declaration at this row's level.""" + + return self.geography_id_source != UNBOUND_GEOGRAPHY_ID_SOURCE + + @property + def geography_id_is_canonical(self) -> bool: + """``geography_id`` is a prefixed census GEOID, not the bare form. + + A consumer counting distinct areas must count only canonical + identifiers: the bare restatement is a different spelling of an area + (and, for a district, a vintage-lossy one), so mixing the two in one + set counts one area twice. + """ + + if ( + self.geography_id_source not in _CANONICAL_GEOGRAPHY_ID_SOURCES + or not self.geography_id.isascii() + ): + return False + if self.level == "national": + return self.geography_id == NATIONAL_GEOGRAPHY_ID + fact = {"geography": {"id": self.geography_id}} + if self.level == "state": + return _state_fips(fact) is not None + if self.level == "congressional_district": + return _congressional_district_geoid(fact) is not None + return False + + @property + def geography_id_is_bare(self) -> bool: + """Recognize a bare code without assigning its missing prefix/vintage.""" + + if self.level == "state": + return self.geography_id in US_STATE_FIPS_TO_POSTAL + return ( + self.level == "congressional_district" + and len(self.geography_id) == 4 + and self.geography_id.isascii() + and self.geography_id.isdigit() + and self.geography_id[:2] in US_STATE_FIPS_TO_POSTAL + ) + + @property + def geography_id_declarations_conflict(self) -> bool: + """The two verbatim copies of the fact's geography id disagree. + + Only the prefixed-GEOID sources are compared, because they are the + same Chronicle field read twice. The bare ``state_fips`` / + ``congressional_district_geoid`` restatement is derived from that same + id and is never counted as a second declaration of an area. + + Reported, never resolved: preferring either copy would move a row to a + different area on undeclared grounds. + """ + + declared = { + source: value + for source, value in self.declared_geography_ids + if source in _CANONICAL_GEOGRAPHY_ID_SOURCES + } + return len(set(declared.values())) > 1 + + @property + def disagrees_with_congressional_district_evidence(self) -> bool: + """The shared CD classifier contradicts an explicitly declared level. + + Reported, never resolved here: silently preferring either side would + move rows between the national and CD views on undeclared grounds. + """ + + return self.congressional_district_evidence and self.level not in ( + "congressional_district", + UNRESOLVED_GEOGRAPHY_VIEW_LEVEL, + ) + + +def _text(raw: object) -> str: + return str(raw or "").strip() + + +def _normalized_level(raw: object) -> str: + level = _text(raw) + if not level: + return "" + level = GEOGRAPHY_VIEW_LEVEL_ALIASES.get(level, level) + return level if level in US_TARGET_GEOGRAPHY_VIEW_LEVELS else "" + + +def _hierarchy_geography(hierarchy: object) -> tuple[str, str]: + geography = getattr(hierarchy, "geography", None) + if geography is None: + return "", "" + return _text(getattr(geography, "level", "")), _text(getattr(geography, "id", "")) + + +def _declared_geography_ids( + *, + level: str, + metadata: Mapping[str, object], + hierarchy_level: str, + hierarchy_id: str, +) -> tuple[tuple[str, str], ...]: + """Every identifier available at ``level``, in binding order. + + The two canonical sources contribute only at the level *they* declare: a + hierarchy geography at ``county`` declares a different level and is + therefore not an identifier for this row. + + The bare key is different, and deliberately so: ``state_fips`` and + ``congressional_district_geoid`` carry no level of their own, so the key is + admitted purely because *the row* resolved to the level that owns it --- by + whatever evidence did so, including the shared CD classifier's substring + fallback. That is what keeps a district's parent ``state_fips`` out of the + district's own declarations, and it is the whole of the guarantee: the key + is not independently attested. + """ + + declared: list[tuple[str, str]] = [] + if hierarchy_id and _normalized_level(hierarchy_level) == level: + declared.append(("hierarchy_geography", hierarchy_id)) + ledger_id = _text(metadata.get("ledger_geography_id")) + if ledger_id and _normalized_level(metadata.get("ledger_geography_level")) == level: + declared.append(("ledger_geography_id", ledger_id)) + owned_key = _LEVEL_OWNED_ID_METADATA_KEYS.get(level, "") + if owned_key: + owned_id = _text(metadata.get(owned_key)) + if owned_id: + declared.append((owned_key, owned_id)) + return tuple(declared) + + +def us_target_geography_view( + *, + name: object, + metadata: Mapping[str, object] | None, + hierarchy: object | None = None, +) -> TargetGeographyView: + """Classify one target row into a national/state/CD comparison view. + + Explicit declared level evidence is preferred, in the order the US target + compiler produces it: the calibration hierarchy's geography tier, then the + ledger geography level, then the compiler's ``geography_scope`` metadata. + Only when no explicit level exists does the shared congressional-district + classifier act as evidence of its own. An unsupported explicit level is + not missing evidence and cannot be replaced by that fallback. Anything else is + :data:`UNRESOLVED_GEOGRAPHY_VIEW_LEVEL`. + + The identifier is then read only from a declaration at that same resolved + level: the hierarchy geography, then the ledger identifier, then the bare + identifier the level owns. No identifier is carried across levels, and an + unresolved row reports none at all. + """ + + metadata = metadata if isinstance(metadata, Mapping) else {} + cd_evidence = bool(is_congressional_district_target(name, metadata)) + + hierarchy_level, hierarchy_id = _hierarchy_geography(hierarchy) + level = _normalized_level(hierarchy_level) + level_source = "hierarchy_geography" if level else "" + if not level: + for key in _EXPLICIT_LEVEL_METADATA_KEYS: + level = _normalized_level(metadata.get(key)) + if level: + level_source = key + break + # An unsupported declaration is evidence of another scope, not missing + # evidence that a name substring may replace. A supported declaration + # above can still resolve the row (e.g. county hierarchy + state scope). + has_explicit_level = bool(_text(hierarchy_level)) or any( + _text(metadata.get(key)) for key in _EXPLICIT_LEVEL_METADATA_KEYS + ) + if not level and not has_explicit_level and cd_evidence: + level = "congressional_district" + level_source = "congressional_district_evidence" + if not level: + return TargetGeographyView( + level=UNRESOLVED_GEOGRAPHY_VIEW_LEVEL, + geography_id="", + level_source="none", + congressional_district_evidence=cd_evidence, + geography_id_source=UNBOUND_GEOGRAPHY_ID_SOURCE, + declared_geography_ids=(), + ) + + declared = _declared_geography_ids( + level=level, + metadata=metadata, + hierarchy_level=hierarchy_level, + hierarchy_id=hierarchy_id, + ) + geography_id_source, geography_id = ( + declared[0] if declared else (UNBOUND_GEOGRAPHY_ID_SOURCE, "") + ) + return TargetGeographyView( + level=level, + geography_id=geography_id, + level_source=level_source, + congressional_district_evidence=cd_evidence, + geography_id_source=geography_id_source, + declared_geography_ids=declared, + ) + + +def us_target_spec_geography_view(spec: object) -> TargetGeographyView: + """:func:`us_target_geography_view` for a compiled ``TargetSpec``.""" + + metadata = getattr(spec, "metadata", None) + return us_target_geography_view( + name=getattr(spec, "name", ""), + metadata=metadata if isinstance(metadata, Mapping) else {}, + hierarchy=getattr(spec, "hierarchy", None), + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py index 134ec403e..15107d378 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py @@ -957,8 +957,7 @@ def _inventory( "A_MARITL", "A_SPOUSE", "A_HSCOL", - "WSAL_VAL", - "SEMP_VAL", + "A_LFSR", "MCARE", "CAID", "IHSFLG", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index 5e336b231..38459934b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "eb328a5cf6cc0bc567c209fce03294f4f3b0b5e59adf47dfa7ff4173545e03be" + "95e8297d5656292fa5c0d01983b3a06959dfb99a92e94bf1cefa864ddb149f61" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/packages/microcosm-build/tests/ss_report_source_fixture.py b/packages/microcosm-build/tests/ss_report_source_fixture.py new file mode 100644 index 000000000..51a07c83c --- /dev/null +++ b/packages/microcosm-build/tests/ss_report_source_fixture.py @@ -0,0 +1,248 @@ +"""Invented original SS reports for the real survey preparation/qualification path. + +This helper constructs raw source bytes and their matching parent checkpoints +before fresh issuance. It does not replace an issuer, qualification, or model. +The private source pins describe invented fixtures, never Census evidence. +""" + +import copy +import hashlib +import shutil +from dataclasses import replace +from fractions import Fraction +from pathlib import Path + +import numpy as np +import pandas as pd +from test_us_survey_population_preparation import fixture + +from microcosm.build.frame_checkpoint import ( + load_frame_checkpoint, + write_frame_checkpoint, +) +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_current_money_source as money +from microcosm.build.us_runtime import asec_household_observations as household_owner +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import native_household_origin as origin +from microcosm.frame import Frame + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def ss_report_source_arguments( + root: Path, monkeypatch, *, no_recipients: bool = False +) -> dict: + """Return arguments for ``prepare_authenticated_survey_population``. + + Native ASEC household 7 contains 105 (retirement, age 55), 106 (NIU, + age 14), 109 (disability, age 45), and 110 (dependents, age 18). + Household 8 contains 107 (ambiguous reason 7, age 55), 108 (NIU, + age 14), 111 (survivors, age 65), and 112 (known zero, age 40). + The maintained fraction 2/3, seed 41 selection keeps household 8 and + omits household 7; original DESIGN weights remain 2552.12 and 100. + Every ACS source person reports positive SSP so any selected ACS person + supplies an unresolved positive report. ``no_recipients`` instead encodes + coherent zero reports in both surveys, retaining the under-15 NIU cases. + + The historical ASEC parent rows, raw members, and cohort files are + preserved. New people join existing groups; raw source household IDs + 7/8 remain distinct from parent-internal PH_SEQ/group IDs 3/4. + """ + arguments = fixture( + root, + monkeypatch, + fraction=Fraction(2, 3), + seed=41, + zero=False, + acs_ssp_values={ + key: 0 if no_recipients else 1200 + for key in ( + ("2024HU0000001", 1), + ("2024HU0000001", 2), + ("2024HU0000002", 1), + ("2024GQ0000001", 1), + ("2024GQ0000002", 1), + ) + }, + ) + source_root = arguments["source_dir"] / "asec" + parent_path = source_root / "parent.h5" + household_path = source_root / "household-attachment.h5" + loaded = load_frame_checkpoint(parent_path) + tables = {e: loaded.frame.table(e).copy(deep=True) for e in loaded.frame.entities} + people = tables["person"] + strata = loaded.frame.strata.copy(deep=True) + historical = people.loc[people.source_year.ne(2024)].copy(deep=True) + + # Copy the dependent record in each existing household, preserving every + # group link. Only raw construction is changed, before any new issuance. + additions, added_strata = [], [] + for pid, template_pid, line, age in ( + (109, 106, 3, 45), + (110, 106, 4, 18), + (111, 108, 3, 65), + (112, 108, 4, 40), + ): + positions = np.flatnonzero(people.person_id.to_numpy() == template_pid) + assert len(positions) == 1 + position = positions[0] + person = people.iloc[[position]].copy(deep=True) + person["person_id"] = np.array([pid], dtype=np.int64) + person["PERIDNUM"] = pd.array([str(pid - 100).zfill(22)], dtype="string") + person["A_LINENO"] = line + person["A_AGE"] = age + person["WSAL_VAL"] = 0.0 + person["SEMP_VAL"] = 0.0 + additions.append(person) + added_strata.append(strata.iloc[[position]].copy(deep=True)) + people = pd.concat([people, *additions]) + strata = pd.concat([strata, *added_strata]) + # Report reason 7 is ambiguous; it is not the person's age. + reports = { + 105: (12000, 1), + 106: (0, 0), + 107: (3000, 7), + 108: (0, 0), + 109: (6000, 2), + 110: (2400, 4), + 111: (9000, 3), + 112: (0, 0), + } + for pid, (amount, _) in reports.items(): + selected = people.person_id.eq(pid) + assert selected.sum() == 1 + people.loc[selected, "SS_VAL"] = 0.0 if no_recipients else float(amount) + pd.testing.assert_frame_equal( + people.loc[people.source_year.ne(2024)], historical, check_exact=True + ) + tables["person"] = people + frame = Frame( + tables, + loaded.frame.schema, + {e: loaded.frame.weights_for(e) for e in loaded.frame.weighted_entities}, + strata, + ) + metadata = copy.deepcopy(loaded.metadata) + for name in ("identity", "source_construction_identity"): + metadata[name] = frame_identity(frame).to_payload() + sources = tuple( + household_owner.AsecHouseholdObservationSource( + entry["year"], Path(entry["path"]), entry["sha256"] + ) + for entry in metadata["source_receipt"]["sources"] + ) + assert [s.year for s in sources] == [2022, 2023, 2024] + assert all(_sha(s.path) == s.sha256 for s in sources) + write_frame_checkpoint(parent_path, frame, metadata=metadata) + parent_sha = _sha(parent_path) + attachment = household_owner.with_asec_household_observations( + frame, + checkpoint_metadata=metadata, + checkpoint_sha256=parent_sha, + sources=sources, + ) + write_frame_checkpoint( + household_path, + attachment.frame, + metadata={ + "schema_version": 1, + "artifact_kind": "microcosm.asec_household_observations_source", + "parent_checkpoint_sha256": parent_sha, + "household_observations": attachment.receipt, + }, + ) + monkeypatch.setattr( + money, + "_SOURCE_PINS", + ( + parent_sha, + _sha(household_path), + tuple((s.year, s.sha256) for s in sources), + ), + ) + + members, pins = {}, [] + current = people.loc[people.source_year.eq(2024)].set_index("PERIDNUM") + for year, member, archive, old_digest, old_rows, old_size in coverage._MEMBER_PINS: + path = source_root / member + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + if year == 2024: + extra = [] + for pid, template_pid in ((109, 106), (110, 106), (111, 108), (112, 108)): + record = raw.loc[ + raw.PERIDNUM.eq(str(template_pid - 100).zfill(22)) + ].copy(deep=True) + assert len(record) == 1 + record["PERIDNUM"] = str(pid - 100).zfill(22) + extra.append(record) + raw = pd.concat([raw, *extra], ignore_index=True) + assert raw.PERIDNUM.is_unique and set(raw.PERIDNUM) == set(current.index) + for field, parent_field in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ("SS_VAL", "SS_VAL"), + ("WSAL_VAL", "WSAL_VAL"), + ("SEMP_VAL", "SEMP_VAL"), + ): + values = current.loc[raw.PERIDNUM, parent_field].to_numpy(np.float64) + assert np.isfinite(values).all() and (values == np.floor(values)).all() + raw[field] = [str(int(v)) for v in values] + raw["SS_YN"] = [ + "0" if int(age) < 15 else "1" if int(amount) > 0 else "2" + for age, amount in zip(raw.A_AGE, raw.SS_VAL, strict=True) + ] + raw["RESNSS1"] = [ + "0" if no_recipients else str(reports[int(key) + 100][1]) + for key in raw.PERIDNUM + ] + for field in ("RESNSS2", "RESNSSA", "I_SSVAL", "I_SSYN"): + raw[field] = "0" + # Deliberately differ from the parent order: qualification must join + # native source keys, not rely on row positions. + raw.iloc[::-1].to_csv(path, index=False) + else: + assert (_sha(path), len(raw), path.stat().st_size) == ( + old_digest, + old_rows, + old_size, + ) + pins.append((year, member, archive, _sha(path), len(raw), path.stat().st_size)) + members[year] = path + for owner in (coverage, restoration): + monkeypatch.setattr(owner, "_MEMBER_PINS", tuple(pins)) + + household_member = source_root / "hhpub25.csv" + raw_households = pd.read_csv(household_member, dtype=str, keep_default_na=False) + assert raw_households.H_SEQ.map(int).tolist() == [7, 8] + raw_households["H_NUMPER"] = "4" + raw_households.to_csv(household_member, index=False) + assert len(origin._ASEC_MEMBER_PINS) == 1 + monkeypatch.setattr( + origin, + "_ASEC_MEMBER_PINS", + ( + replace( + origin._ASEC_MEMBER_PINS[0], + member_sha256=_sha(household_member), + size_bytes=household_member.stat().st_size, + rows=len(raw_households), + ), + ), + ) + restored = root / "ss-report-restored-money" + restoration.restore_asec_person_income_source( + parent_path, + household_path, + member_paths=members, + output_dir=restored, + ) + shutil.copyfile( + restored / restoration.CHECKPOINT_FILENAME, + source_root / "person-income-attachment.h5", + ) + return arguments diff --git a/packages/microcosm-build/tests/test_atomic_geography.py b/packages/microcosm-build/tests/test_atomic_geography.py new file mode 100644 index 000000000..ccc4b172c --- /dev/null +++ b/packages/microcosm-build/tests/test_atomic_geography.py @@ -0,0 +1,511 @@ +"""Invented support controls; no survey, Census or administrative files are read.""" + +from copy import deepcopy + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import atomic_geography as geo +from microcosm.build.graph_atomic_geography import ( + ATOMIC_GEOGRAPHY_VALIDATION_TYPE, + atomic_geography_nodes, + register_atomic_geography_kernels, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactType, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + + +def support_parts(system="invented_block", *, district=True): + columns = { + name: { + "kind": "code", + "source": "invented-lookup-v1", + "vintage": "invented-2020", + "relation": "official_tabulation" if name == "district" else "exact", + } + for name in ("area", "region", "puma", *(("district",) if district else ())) + } + columns.update( + { + name: {"kind": "weight", "source": "invented-counts-v1", "basis": basis} + for name, basis in (("population", "persons"), ("households", "households")) + } + ) + arrays = { + "area": np.asarray(["0001", "0002", "0003", "0004"]), + "region": np.asarray(["R", "R", "R", "S"]), + "puma": np.asarray(["A", "A", "B", "C"]), + "population": np.asarray([0, 2, 6, 1]), + "households": np.asarray([1, 3, 4, 1]), + } + if district: + arrays["district"] = np.asarray(["01", "02", "02", "03"]) + return { + "version": 1, + "system": system, + "level": "block", + "code_system": "invented", + "vintage": "invented-2020", + "columns": columns, + }, arrays + + +def fixture(system="invented_block", *, district=True): + metadata, arrays = support_parts(system, district=district) + payload = geo.encode_atomic_support(metadata, arrays) + support = geo.decode_atomic_support(payload) + spec = { + "version": 1, + "identity": ["source_record"], + "stream": ["sha256-u53-v1", "invented-geography", 0, 21], + "outputs": {"area": "block", "system": "area_system", "basis": "area_basis"}, + "systems": [ + { + "id": system, + "level": "block", + "code_system": "invented", + "vintage": "invented-2020", + "source": system + "_source", + "selector": {}, + "constraints": [ + {"input": "observed_region", "support": "region", "required": True}, + {"input": "observed_puma", "support": "puma", "required": False}, + ], + "observed_area": None, + "stages": [{"level": "area", "weight": "population"}], + "layers": ( + [ + { + "input": "district", + "output": "district_code", + "vintage": "invented-2020", + "relation": "official_tabulation", + "source": "invented-lookup-v1", + } + ] + if district + else [] + ), + } + ], + } + households = pd.DataFrame( + { + "household_id": np.arange(1, 9, dtype=np.int64), + "source_record": pd.array( + ["source-" + str(i) for i in range(8)], dtype="string" + ), + "observed_region": pd.array(["R"] * 7 + ["S"], dtype="string"), + "observed_puma": pd.array( + [None, "A", "B", None, "A", None, "B", "C"], dtype="string" + ), + } + ) + return households, spec, {system: support}, payload + + +def complete(households, spec, supports): + assigned = pd.concat( + [households, geo.assign_atomic(households, spec, supports)], axis=1 + ) + return pd.concat([assigned, geo.derive_geography(assigned, spec, supports)], axis=1) + + +def test_codec_is_deterministic_and_decoded_values_are_immutable(): + metadata, arrays = support_parts() + first = geo.encode_atomic_support(metadata, arrays) + assert first == geo.encode_atomic_support( + metadata, dict(reversed(list(arrays.items()))) + ) + support = geo.decode_atomic_support(first) + assert support.arrays["area"].tolist() == ["0001", "0002", "0003", "0004"] + with pytest.raises(ValueError): + support.arrays["area"].setflags(write=True) + with pytest.raises(TypeError): + support.metadata["columns"]["area"]["source"] = "changed" + + +def test_integer_inverse_cdf_respects_zero_mass_and_boundaries(): + _, _, supports, _ = fixture() + index = geo._SupportIndex(supports["invented_block"]) + stage = {"level": "area", "weight": "population"} + cell = (("region", "R"),) + assert index.pick(cell, stage, 0.0) == "0002" + assert index.pick(cell, stage, 0.25 - 1 / 2**53) == "0002" + assert index.pick(cell, stage, 0.25) == "0003" + assert index.pick(cell, stage, 1 - 1 / 2**53) == "0003" + + +def test_draws_are_stable_under_order_subset_extra_columns_and_unrelated_rows(): + households, spec, supports, _ = fixture() + before = households.copy(deep=True) + expected = geo.assign_atomic(households, spec, supports) + reordered = households.iloc[[7, 2, 0, 5]] + actual = geo.assign_atomic(reordered.assign(unrelated=42), spec, supports) + pd.testing.assert_frame_equal(actual, expected.loc[reordered.index]) + extra = households.iloc[[0]].copy() + extra.index = [100] + extra["source_record"] = "unrelated-record" + grown = geo.assign_atomic(pd.concat([households, extra]), spec, supports) + pd.testing.assert_frame_equal(grown.loc[households.index], expected) + pd.testing.assert_frame_equal(households, before) + + +def test_stage_law_uses_declared_weights_at_each_level(): + households, spec, supports, _ = fixture() + spec["systems"][0]["stages"] = [ + {"level": "puma", "weight": "households"}, + {"level": "area", "weight": "population"}, + ] + result = complete(households, spec, supports) + assert result.loc[1, "block"] == "0002" + assert result.loc[2, "block"] == "0003" + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + # A/B household mass is 4/4, but population mass is 2/6. + index = geo._SupportIndex(supports["invented_block"]) + assert index.pick((("region", "R"),), spec["systems"][0]["stages"][0], 0.49) == "A" + assert index.pick((("region", "R"),), spec["systems"][0]["stages"][0], 0.5) == "B" + + +def test_qualified_observed_area_is_retained_even_with_zero_sampling_mass(): + households, spec, supports, _ = fixture() + households["observed_block"] = pd.array(["0001"] + [None] * 7, dtype="string") + spec["systems"][0]["observed_area"] = "observed_block" + result = complete(households, spec, supports) + assert result.loc[0, "block"] == "0001" + assert result.loc[0, "area_basis"] == "observed" + assert result.loc[0, "district_code"] == "01" + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + + +@pytest.mark.parametrize( + "defect", + [ + "missing_region", + "wrong_region", + "inconsistent_puma", + "duplicate_identity", + "null_identity", + "existing_output", + "unknown_observed_area", + "conflicting_observed_area", + ], +) +def test_assignment_refuses_invalid_or_conflicting_households(defect): + households, spec, supports, _ = fixture() + if defect == "missing_region": + households.loc[0, "observed_region"] = pd.NA + elif defect == "wrong_region": + households.loc[0, "observed_region"] = "missing" + elif defect == "inconsistent_puma": + households.loc[0, "observed_puma"] = "C" + elif defect == "duplicate_identity": + households.loc[0, "source_record"] = households.loc[1, "source_record"] + elif defect == "null_identity": + households.loc[0, "source_record"] = pd.NA + elif defect == "existing_output": + households["block"] = "untouched" + else: + households["observed_block"] = pd.array( + ["unknown" if defect == "unknown_observed_area" else "0004"] + [None] * 7, + dtype="string", + ) + spec["systems"][0]["observed_area"] = "observed_block" + with pytest.raises(ValueError): + geo.assign_atomic(households, spec, supports) + + +@pytest.mark.parametrize( + "defect", + [ + "duplicate_area", + "negative_weight", + "float_weight", + "bool_weight", + "overflow_weight", + "all_zero_weight", + "unknown_relation", + "wrong_area_vintage", + "empty_code", + "wrong_length", + ], +) +def test_support_refuses_ambiguous_or_invalid_artifacts(defect): + metadata, arrays = support_parts() + if defect == "duplicate_area": + arrays["area"][1] = arrays["area"][0] + elif defect == "negative_weight": + arrays["population"][0] = -1 + elif defect == "float_weight": + arrays["population"] = arrays["population"].astype(float) + elif defect == "bool_weight": + arrays["population"] = arrays["population"].astype(bool) + elif defect == "overflow_weight": + arrays["population"] = np.asarray([2**63] * 4, dtype=np.uint64) + elif defect == "all_zero_weight": + arrays["population"] *= 0 + elif defect == "unknown_relation": + metadata["columns"]["district"]["relation"] = "exact_enough" + elif defect == "wrong_area_vintage": + metadata["columns"]["area"]["vintage"] = "different" + elif defect == "empty_code": + arrays["area"][0] = "" + elif defect == "wrong_length": + arrays["region"] = arrays["region"][:-1] + with pytest.raises(ValueError): + geo.encode_atomic_support(metadata, arrays) + + +@pytest.mark.parametrize("field", ["source", "relation", "vintage"]) +def test_layer_metadata_must_match_the_pinned_support(field): + households, spec, supports, _ = fixture() + spec["systems"][0]["layers"][0][field] = ( + "exact" if field == "relation" else "different" + ) + with pytest.raises(ValueError): + geo.assign_atomic(households, spec, supports) + + +def test_three_nation_systems_use_the_same_operations_and_mark_absent_layers(): + households, spec, supports, _ = fixture() + households["nation"] = pd.array( + ["one"] * 3 + ["two"] * 3 + ["three"] * 2, dtype="string" + ) + spec["systems"][0]["selector"] = {"nation": ["one"]} + for name, selector, district in ( + ("invented_oa", "two", True), + ("invented_dz", "three", False), + ): + _, other, added, _ = fixture(name, district=district) + system = other["systems"][0] + system["selector"] = {"nation": [selector]} + spec["systems"].append(system) + supports.update(added) + result = complete(households, spec, supports) + assert ( + result["area_system"].tolist() + == ["invented_block"] * 3 + ["invented_oa"] * 3 + ["invented_dz"] * 2 + ) + assert result.loc[[6, 7], "district_code"].isna().all() + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + spec["systems"][1]["selector"] = {"nation": ["one", "two"]} + with pytest.raises(ValueError, match="exactly one"): + geo.assign_atomic(households, spec, supports) + + +def test_clone_inheritance_and_pruned_mapping_checks_do_not_redraw(): + households, spec, supports, _ = fixture() + original = complete(households, spec, supports) + clone = original.iloc[[2, 0, 5]].copy() + clone["household_id"] += 100 + clone["source_record"] += "-clone" + assert geo.validate_geography(clone, spec, supports)["households"] == 3 + clone.iloc[0, clone.columns.get_loc("district_code")] = "wrong" + with pytest.raises(ValueError, match="derived geography differs"): + geo.validate_geography(clone, spec, supports) + + +@pytest.mark.parametrize("defect", ["atomic", "basis", "observed", "system"]) +def test_final_integrity_gate_refuses_changed_geography(defect): + households, spec, supports, _ = fixture() + result = complete(households, spec, supports) + if defect == "atomic": + result.loc[0, "block"] = "missing" + elif defect == "basis": + result.loc[0, "area_basis"] = "observed" + elif defect == "observed": + result.loc[0, "observed_region"] = "wrong" + elif defect == "system": + result.loc[0, "area_system"] = pd.NA + with pytest.raises(ValueError): + geo.validate_geography(result, spec, supports) + + +class _InventedSpine(KernelBase): + ref = "invented.atomic_spine@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + households, _, _, _ = fixture() + person = pd.DataFrame( + { + "person_id": np.arange(1, 9, dtype=np.int64), + "person_household_id": households["household_id"], + "age": np.arange(21, 29, dtype=np.int64), + } + ) + return KernelResult( + frame=Frame( + {"person": person, "household": households}, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.ones(8), WeightKind.DESIGN)}, + pd.Series(["invented"] * 8, name="stratum"), + ) + ) + + +@pytest.mark.parametrize("emit_validation_artifact", (False, True)) +def test_real_executor_cold_warm_and_changed_source_identity( + tmp_path, emit_validation_artifact +): + households, spec, supports, payload = fixture() + columns = ( + Owned("person", "age", "int64"), + *( + Owned("household", name, "string") + for name in ("source_record", "observed_region", "observed_puma") + ), + ) + base = Node( + "spine", + _InventedSpine.ref, + sources=("invented_spine",), + structural=StructuralDelta.CREATE, + outputs=columns, + ) + nodes = atomic_geography_nodes( + spec, columns, base=base.id, emit_validation_artifact=emit_validation_artifact + ) + graph = compile_graph( + Graph( + "invented", + ( + SourceRef("invented_spine", "raw-bytes-v1"), + SourceRef("invented_block_source", "raw-bytes-v1"), + ), + (base, *nodes), + ) + ) + registry = KernelRegistry() + registry.register(_InventedSpine()) + register_atomic_geography_kernels(registry) + raw_spine = tmp_path / "invented-spine.bin" + raw_spine.write_bytes(b"invented test source declaration") + raw_support = tmp_path / "invented-support.npz" + raw_support.write_bytes(payload) + sources = {"invented_spine": raw_spine, "invented_block_source": raw_support} + store = ContentStore(tmp_path / "store") + cold = run_graph(graph, sources=sources, store=store, kernels=registry) + warm = run_graph( + graph, sources=sources, store=store, kernels=registry, resume="require" + ) + actual = cold.population(base.id).table("household") + expected = complete(households, spec, supports) + for column in expected: + left, right = actual[column], expected[column] + if isinstance(expected[column].dtype, pd.StringDtype): + # Source checkpoints and emitted columns may use different string + # backends. Check the nullable-string contract, then compare every + # value and missing cell using one explicit physical representation. + assert isinstance(left.dtype, pd.StringDtype) + assert left.dtype.na_value is pd.NA + left = left.astype(pd.StringDtype(storage="python")) + right = right.astype(pd.StringDtype(storage="python")) + pd.testing.assert_series_equal(left, right, check_index_type=False) + assert cold.nodes["geography.assign"].key == warm.nodes["geography.assign"].key + assert cold.nodes["geography.gate"].receipt["outcome"] == "pass" + gate = cold.nodes["geography.gate"] + if emit_validation_artifact: + output = graph.graph.node("geography.gate").artifact_outputs + assert len(output) == 1 and output[0].type == ATOMIC_GEOGRAPHY_VALIDATION_TYPE + key = gate.opaque_artifacts["validation"] + assert key == opaque_artifact_key(gate.key, "validation") + assert store.load_bytes(key) == canonical_json( + geo.validate_geography(expected, spec, supports) + ) + assert warm.nodes["geography.gate"].opaque_artifacts["validation"] == key + else: + assert not gate.opaque_artifacts + + np.testing.assert_array_equal( + cold.population(base.id).weights_for("household").values, np.ones(8) + ) + metadata, arrays = support_parts() + arrays["population"][1] += 1 + raw_support.write_bytes(geo.encode_atomic_support(metadata, arrays)) + changed = run_graph(graph, sources=sources, store=store, kernels=registry) + assert changed.nodes["geography.assign"].key != cold.nodes["geography.assign"].key + + +def test_graph_builder_refuses_input_overwrites_and_duplicate_inventory(): + _, spec, _, _ = fixture() + columns = tuple( + Owned("household", c, "string") + for c in ("source_record", "observed_region", "observed_puma") + ) + with pytest.raises(ValueError, match="overwrite"): + atomic_geography_nodes( + spec, (*columns, Owned("household", "block", "string")), base="spine" + ) + with pytest.raises(ValueError, match="repeats"): + atomic_geography_nodes(spec, (*columns, columns[0]), base="spine") + with pytest.raises(ValueError, match="missing"): + atomic_geography_nodes(spec, columns[1:], base="spine") + altered = deepcopy(spec) + altered["outputs"]["area"] = "observed_region" + with pytest.raises(ValueError, match="overwritten"): + atomic_geography_nodes(altered, columns, base="spine") + + +@pytest.mark.parametrize("defect", ("missing", "type")) +def test_typed_geography_gate_edge_refuses_invalid_producer_contract(defect): + _, spec, _, _ = fixture() + columns = tuple( + Owned("household", name, "string") + for name in ("source_record", "observed_region", "observed_puma") + ) + base = Node( + "spine", + _InventedSpine.ref, + sources=("invented_spine",), + structural=StructuralDelta.CREATE, + outputs=columns, + ) + nodes = atomic_geography_nodes( + spec, columns, base=base.id, emit_validation_artifact=(defect != "missing") + ) + edge_type = ( + ArtifactType("invented.wrong_gate", 1) + if defect == "type" + else ATOMIC_GEOGRAPHY_VALIDATION_TYPE + ) + consumer = Node( + "consumer", + "invented.consumer@1", + population=base.id, + artifact_inputs=( + ArtifactInput("validation", "geography.gate", "validation", edge_type), + ), + ) + with pytest.raises(ValueError, match="no declared artifact|type does not match"): + compile_graph( + Graph( + "invented", + ( + SourceRef("invented_spine", "raw-bytes-v1"), + SourceRef("invented_block_source", "raw-bytes-v1"), + ), + (base, *nodes, consumer), + ) + ) diff --git a/packages/microcosm-build/tests/test_ci_test_groups.py b/packages/microcosm-build/tests/test_ci_test_groups.py new file mode 100644 index 000000000..90c2a72ed --- /dev/null +++ b/packages/microcosm-build/tests/test_ci_test_groups.py @@ -0,0 +1,246 @@ +"""Stdlib-only controls for CI selection; never collect or import data tests. + +Also runnable directly with an isolated Python interpreter, without pytest or +the workspace conftest, to verify the inventory and workflow partition contract. +""" + +import contextlib +import importlib.util +import io +import json +import re +import subprocess +import unittest +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[3] +SPEC = importlib.util.spec_from_file_location( + "ci_test_groups", ROOT / "tools/ci_test_groups.py" +) +groups = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(groups) + + +class ShardSelectionTests(unittest.TestCase): + def test_cli_accepts_single_and_maximum_shard_boundaries(self): + self.assertEqual(groups.parse_shard("1/1"), (1, 1)) + self.assertEqual(groups.parse_shard("64/64"), (64, 64)) + + def test_round_robin_keeps_whole_files_and_covers_each_once(self): + files = [f"packages/demo/tests/test_{i:02d}.py" for i in range(17)] + pieces = [groups.shard_files(reversed(files), i, 4) for i in range(1, 5)] + self.assertEqual(pieces, [files[i::4] for i in range(4)]) + self.assertEqual(sorted(p for piece in pieces for p in piece), files) + self.assertEqual(max(map(len, pieces)) - min(map(len, pieces)), 1) + + def test_single_shard_preserves_sorted_inventory(self): + self.assertEqual(groups.shard_files(["b", "a"], 1, 1), ["a", "b"]) + + def test_invalid_indices_counts_duplicates_and_empty_shards_refuse(self): + for index, count in ((0, 4), (5, 4), (1, 0), (1, 65), (True, 4), (1, 2.0)): + with self.subTest(index=index, count=count): + with self.assertRaises(ValueError): + groups.shard_files(["a", "b", "c", "d"], index, count) + for files, count in (([], 1), (["a"], 2), (["a", "a"], 1)): + with self.subTest(files=files, count=count): + with self.assertRaises(ValueError): + groups.shard_files(files, 1, count) + + def test_cli_rejects_bad_shards_and_incompatible_actions(self): + for token in ("0/4", "5/4", "1/0", "1/65", "1", "1/2/3", "1.0/4", "01/4"): + with self.subTest(token=token), contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as caught: + groups.main(["--list", "rest", "--shard", token]) + self.assertNotEqual(caught.exception.code, 0) + for action in (["--verify"], ["--procs", "us-am"]): + with self.subTest(action=action), contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as caught: + groups.main([*action, "--shard", "1/4"]) + self.assertNotEqual(caught.exception.code, 0) + + def test_empty_selection_emits_no_paths_and_fails(self): + output = io.StringIO() + with ( + patch.object(groups, "tracked_test_files", return_value=()), + contextlib.redirect_stdout(output), + ): + with self.assertRaises(SystemExit): + groups.main(["--list", "rest", "--shard", "1/4"]) + self.assertEqual(output.getvalue(), "") + + def test_insufficient_selected_files_emits_no_paths_and_fails(self): + output = io.StringIO() + with ( + patch.object( + groups, + "tracked_test_files", + return_value=("packages/microcosm-fit/tests/test_fit.py",), + ), + contextlib.redirect_stdout(output), + ): + with self.assertRaises(SystemExit): + groups.main(["--list", "rest", "--shard", "1/4"]) + self.assertEqual(output.getvalue(), "") + + def test_inventory_command_failure_cannot_become_an_empty_success(self): + output = io.StringIO() + failure = subprocess.CalledProcessError(7, ["git", "ls-files"]) + with ( + patch.object(groups, "tracked_test_files", side_effect=failure), + contextlib.redirect_stdout(output), + ): + with self.assertRaises(subprocess.CalledProcessError): + groups.main(["--list", "wheels", "--shard", "1/4"]) + self.assertEqual(output.getvalue(), "") + + def test_wheels_retain_engine_only_files(self): + files = (*groups.ENGINE_ONLY, "packages/microcosm-fit/tests/test_fit.py") + with patch.object(groups, "tracked_test_files", return_value=files): + self.assertEqual(groups.selected_files("wheels"), sorted(files)) + self.assertEqual(groups.selected_files("rest"), [files[-1]]) + + +class InventoryTests(unittest.TestCase): + def test_nested_files_are_excluded_then_reported_even_with_broad_git_output(self): + flat = "packages/microcosm-build/tests/test_flat.py" + nested = "packages/microcosm-build/tests/test_util/test_nested.py" + completed = subprocess.CompletedProcess( + args=[], returncode=0, stdout=f"{nested}\n{flat}\n" + ) + with ( + patch.object(groups.subprocess, "run", return_value=completed) as run, + patch.object(Path, "is_file", return_value=True), + ): + files = groups.tracked_test_files() + self.assertEqual(files, (flat,)) + self.assertEqual( + run.call_args.args[0], + ["git", "ls-files", "--", ":(glob)packages/*/tests/test_*.py"], + ) + self.assertEqual(groups.stray_nested_test_files(files), (nested,)) + self.assertEqual( + run.call_args.args[0], ["git", "ls-files", "--", "packages"] + ) + + def test_deleted_tracked_files_do_not_enter_inventory(self): + completed = subprocess.CompletedProcess( + args=[], + returncode=0, + stdout="packages/microcosm-build/tests/test_deleted.py\n", + ) + with ( + patch.object(groups.subprocess, "run", return_value=completed), + patch.object(Path, "is_file", return_value=False), + ): + self.assertEqual(groups.tracked_test_files(), ()) + self.assertEqual(groups.stray_nested_test_files(()), ()) + + +class WorkflowCoverageTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.files = groups.tracked_test_files() + cls.workflow = (ROOT / ".github/workflows/test.yml").read_text() + + def job(self, name): + match = re.search( + rf"^ {re.escape(name)}:\n(.*?)(?=^ [\w-]+:|\Z)", + self.workflow, + re.MULTILINE | re.DOTALL, + ) + self.assertIsNotNone(match) + return match.group(1) + + def test_actual_matrix_shards_cover_each_tier_without_process_overlap(self): + for job, tier in ( + ("fast", groups.FAST_GROUPS), + ("engine-us", groups.US_GROUPS), + ("wheels", ("wheels",)), + ): + with self.subTest(job=job): + # Matrix rows are JSON objects (also valid YAML), keeping this + # coverage proof stdlib-only and tied to the actual workflow. + rows = [ + json.loads(row) + for row in re.findall( + r"^\s+- (\{\"group\".*\})$", self.job(job), re.MULTILINE + ) + ] + expected = [ + { + "group": group, + "shard": f"{i}/{groups.SHARD_COUNTS.get(group, 1)}", + } + for group in tier + for i in range(1, groups.SHARD_COUNTS.get(group, 1) + 1) + ] + self.assertEqual(rows, expected) + seen = [] + for row in rows: + index, count = groups.parse_shard(row["shard"]) + for proc in groups.PROCESSES[row["group"]]: + selected = groups.selected_files( + f"{row['group']}:{proc}", shard=(index, count) + ) + self.assertTrue(selected) + self.assertEqual( + {groups.process_for(row["group"], p) for p in selected}, + {proc}, + ) + seen.extend(selected) + if job == "fast": + complete = [p for p in self.files if p not in groups.ENGINE_ONLY] + elif job == "engine-us": + complete = [ + p + for p in self.files + if groups.engine_group(p) in groups.US_GROUPS + ] + else: + complete = list(self.files) + self.assertEqual(sorted(seen), sorted(complete)) + self.assertEqual(len(seen), len(set(seen))) + + def test_versions_and_us_process_isolation_are_retained(self): + for job in ("fast", "engine-us", "engine-uk", "wheels"): + self.assertIn('python-version: ["3.13", "3.14"]', self.job(job)) + self.assertIn('python-version: ["3.13", "3.14.4"]', self.job("engine-shared")) + self.assertEqual(groups.PROCESSES["us-am"], ("build", "other-shards")) + us = self.job("engine-us") + self.assertIn('for proc in "${procs[@]}"; do', us) + self.assertIn('pytest "${files[@]}"', us) + self.assertIn("--extra us --extra uk", us) + + def test_selectors_fail_closed_without_process_substitution(self): + for job in ("fast", "engine-shared", "engine-us", "engine-uk", "wheels"): + body = self.job(job) + self.assertNotIn("< <(", body) + self.assertIn("set -euo pipefail", body) + self.assertIn('test -n "$selected"', body) + self.assertIn('mapfile -t files <<< "$selected"', body) + for job in ("engine-us", "engine-uk"): + self.assertIn('test -n "$selected_procs"', self.job(job)) + + def test_wheel_smokes_extras_and_engine_absence_checks_remain(self): + wheel = self.job("wheels") + for text in ( + 'uv build --wheel "$shard"', + "uv venv /tmp/wheels-venv", + "dist/*.whl pytest", + "source import escaped wheel venv", + "tools/spec_envelope_digests.py be uk", + "microcosm-frame[us] @ file://", + "microcosm-build[source-io] @ file://", + 'pd.read_hdf(path, key="person")', + "policyengine-us must be absent from the base wheel gate", + "env -u PYTHONPATH /tmp/wheels-venv/bin/python -I -m pytest", + ): + self.assertIn(text, wheel) + self.assertIn('--list wheels --shard "${{ matrix.lane.shard }}"', wheel) + self.assertNotIn("-m requires", wheel) + self.assertNotIn("--ignore", wheel) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/microcosm-build/tests/test_frame_checkpoint.py b/packages/microcosm-build/tests/test_frame_checkpoint.py index 7c79ba3f8..d5e052a40 100644 --- a/packages/microcosm-build/tests/test_frame_checkpoint.py +++ b/packages/microcosm-build/tests/test_frame_checkpoint.py @@ -576,6 +576,205 @@ def test_schema_3_checkpoint_requires_nullable_boolean_encoding( load_frame_checkpoint(path) +@pytest.mark.parametrize( + "dtype", ["Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", "UInt64"] +) +@pytest.mark.parametrize("has_missing", [False, True]) +def test_nullable_integer_round_trip_preserves_width_values_and_null_mask( + tmp_path: Path, dtype: str, has_missing: bool +) -> None: + frame = _nullable_boolean_checkpoint_frame() + numpy_dtype = pd.api.types.pandas_dtype(dtype).numpy_dtype + bounds = np.iinfo(numpy_dtype) + frame.person["nullable_integer"] = pd.Series( + [int(bounds.min), pd.NA if has_missing else 0, int(bounds.max)], + index=frame.person.index, + dtype=dtype, + ) + first_path = tmp_path / "integer-first.h5" + second_path = tmp_path / "integer-second.h5" + + write_frame_checkpoint(first_path, frame) + loaded = load_frame_checkpoint(first_path) + write_frame_checkpoint(second_path, loaded.frame) + + assert first_path.read_bytes() == second_path.read_bytes() + pd.testing.assert_frame_equal(loaded.frame.person, frame.person, check_exact=True) + pd.testing.assert_frame_equal(loaded.frame.link("jobs"), frame.link("jobs")) + h5py = pytest.importorskip("h5py") + with h5py.File(first_path, mode="r") as h5: + metadata, spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + assert metadata["schema_version"] == 4 + assert spec == { + "name": "nullable_integer", + "dtype": dtype, + "encoding": "nullable_integer_v1", + "has_null_mask": has_missing, + } + values = np.asarray(group["values"]) + assert values.dtype == numpy_dtype + assert values.tolist() == [int(bounds.min), 0, int(bounds.max)] + if has_missing: + mask = np.asarray(group["null_mask"]) + assert mask.dtype == np.dtype(np.uint8) + assert mask.tolist() == [0, 1, 0] + else: + assert "null_mask" not in group + + +def _nullable_integer_checkpoint_frame() -> Frame: + frame = _checkpoint_frame() + frame.person["nullable_integer"] = pd.Series( + [1, pd.NA, 3], index=frame.person.index, dtype="Int64" + ) + return frame + + +def test_nullable_integer_masked_storage_is_canonical(tmp_path: Path) -> None: + paths = [tmp_path / "hidden-zero.h5", tmp_path / "hidden-nonzero.h5"] + for hidden, path in zip((0, 97), paths, strict=True): + frame = _nullable_integer_checkpoint_frame() + frame.person["nullable_integer"] = pd.Series( + pd.arrays.IntegerArray( + np.asarray([1, hidden, 3], dtype=np.int64), + np.asarray([False, True, False], dtype=np.bool_), + ), + index=frame.person.index, + ) + write_frame_checkpoint(path, frame) + assert paths[0].read_bytes() == paths[1].read_bytes() + + +@pytest.mark.parametrize( + "damage", + ["missing", "all_zero", "nonbinary", "wrong_length", "wrong_dtype", "wrong_rank"], +) +def test_nullable_integer_checkpoint_rejects_malformed_null_mask( + tmp_path: Path, damage: str +) -> None: + path = tmp_path / "malformed-integer-mask.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + del group["null_mask"] + if damage != "missing": + masks = { + "all_zero": np.asarray([0, 0, 0], dtype=np.uint8), + "nonbinary": np.asarray([0, 2, 0], dtype=np.uint8), + "wrong_length": np.asarray([0, 1], dtype=np.uint8), + "wrong_dtype": np.asarray([0, 1, 0], dtype=np.int16), + "wrong_rank": np.asarray([[0, 1, 0]], dtype=np.uint8), + } + group.create_dataset("null_mask", data=masks[damage], track_times=False) + with pytest.raises(ValueError, match="null mask"): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize("damage", ["wrong_dtype", "wrong_rank", "hidden_nonzero"]) +def test_nullable_integer_checkpoint_rejects_malformed_values( + tmp_path: Path, damage: str +) -> None: + path = tmp_path / "malformed-integer-values.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + del group["values"] + values = { + "wrong_dtype": np.asarray([1, 0, 3], dtype=np.uint64), + "wrong_rank": np.asarray([[1, 0, 3]], dtype=np.int64), + "hidden_nonzero": np.asarray([1, 2, 3], dtype=np.int64), + } + group.create_dataset("values", data=values[damage], track_times=False) + message = ( + "noncanonical nonzero" + if damage == "hidden_nonzero" + else "nullable integer values" + ) + with pytest.raises(ValueError, match=message): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize("version", [2, 3]) +def test_legacy_checkpoint_cannot_smuggle_nullable_integer_encoding( + tmp_path: Path, version: int +) -> None: + path = tmp_path / "forged-legacy-integer.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata, _spec, _group = _checkpoint_series_group( + root, table="person", column="nullable_integer" + ) + metadata["schema_version"] = version + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match=f"schema version {version}.*nullable integer"): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("has_null_mask", "yes", "has_null_mask"), + ("dtype", "int64", "must pair a supported pandas integer dtype"), + ("encoding", "numpy", "must pair a supported pandas integer dtype"), + ], +) +def test_nullable_integer_checkpoint_rejects_malformed_spec( + tmp_path: Path, field: str, value: object, message: str +) -> None: + path = tmp_path / "malformed-integer-spec.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata, spec, _group = _checkpoint_series_group( + root, table="person", column="nullable_integer" + ) + spec[field] = value + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match=message): + load_frame_checkpoint(path) + + +def test_maskless_nullable_integer_rejects_unexpected_null_mask(tmp_path: Path) -> None: + frame = _nullable_integer_checkpoint_frame() + frame.person["nullable_integer"] = frame.person["nullable_integer"].fillna(0) + path = tmp_path / "unexpected-integer-mask.h5" + write_frame_checkpoint(path, frame) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + group.create_dataset( + "null_mask", data=np.zeros(3, dtype=np.uint8), track_times=False + ) + with pytest.raises(ValueError, match="unexpected null mask"): + load_frame_checkpoint(path) + + +def test_schema_4_checkpoint_requires_nullable_integer_encoding(tmp_path: Path) -> None: + path = tmp_path / "forged-schema-4.h5" + write_frame_checkpoint(path, _nullable_boolean_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata = json.loads(np.asarray(root["metadata_json"]).tobytes()) + metadata["schema_version"] = 4 + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match="schema version 4.*nullable integer"): + load_frame_checkpoint(path) + + def test_frame_checkpoint_fsyncs_parent_directory_after_rename( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -656,10 +855,9 @@ def test_frame_checkpoint_preserves_range_indexes_and_column_axis_name( @pytest.mark.parametrize( "unsupported", [ - pd.Series([1, pd.NA, 3], dtype="Int64"), pd.Series(["a", "b", "a"], dtype="category"), ], - ids=["nullable_integer", "categorical"], + ids=["categorical"], ) def test_frame_checkpoint_rejects_unsupported_dtype_without_replacing_destination( tmp_path: Path, diff --git a/packages/microcosm-build/tests/test_reform_validation.py b/packages/microcosm-build/tests/test_reform_validation.py index 263b54f91..c3014cfaa 100644 --- a/packages/microcosm-build/tests/test_reform_validation.py +++ b/packages/microcosm-build/tests/test_reform_validation.py @@ -7,10 +7,13 @@ from __future__ import annotations import json +from copy import deepcopy +from dataclasses import replace import pytest import microcosm.build.us_runtime.reform_validation as reform_validation_module +from microcosm.build.us_runtime.fiscal_targets import SimpleTaxExpenditureReform from microcosm.build.us_runtime.reform_validation import ( REFORM_VALIDATION_SCHEMA_VERSION, ReformValidationSpec, @@ -40,6 +43,176 @@ def calculate(self, measure: str, period): # noqa: ARG002 return _FakeSeries(self._totals[measure]) +def test_obbba_claim_comparisons_match_executed_worlds(monkeypatch): + # Invented patches and callback totals: two measures and two periods must + # retain separate stacks, even when their rows are interleaved. + specs = tuple( + replace( + _oos_spec(-1, category="OBBBA"), + id=name, + parameter_changes={f"gov.example.{name}": {"2027-01-01": 0}}, + budget_measure=measure, + period=period, + effect_direction="baseline_minus_reform", + ) + for name, measure, period in ( + ("a", "example_tax", 2027), + ("c", "other_tax", 2027), + ("b", "example_tax", 2027), + ("d", "example_tax", 2028), + ) + ) + original = deepcopy(specs) + patches = { + key: value for spec in specs for key, value in spec.parameter_changes.items() + } + + def world(names): + return {f"gov.example.{name}": patches[f"gov.example.{name}"] for name in names} + + expected_calls = [ + (world("abcd"), "example_tax", 2027, 1000), + (world("bcd"), "example_tax", 2027, 900), + (world("cd"), "example_tax", 2027, 920), + (world("abcd"), "other_tax", 2027, 50), + (world("abd"), "other_tax", 2027, 70), + (world("abcd"), "example_tax", 2028, 1100), + (world("abc"), "example_tax", 2028, 1080), + ] + calls = [] + monkeypatch.setattr(reform_validation_module, "_build_parameter_reform", deepcopy) + + def simulate(reform): + changes, measure, at_period, total = expected_calls[len(calls)] + assert reform == changes + calls.append(deepcopy(reform)) + + class FakeSimulation: + def calculate(self, actual_measure, actual_period): + assert (actual_measure, actual_period) == (measure, at_period) + return _FakeSeries(total) + + return FakeSimulation() + + payload = reform_validation_payload(specs, period=2026, simulate=simulate) + rows = {row["id"]: row for row in payload["reforms"]} + assert len(calls) == len(expected_calls) + assert payload["obbba_pre_baseline_parameter_changes"] == patches + assert payload["baseline_period"] == 2026 + for name, before, after, base_total, reform_total in ( + ("a", "abcd", "bcd", 1000, 900), + ("c", "abcd", "abd", 50, 70), + ("b", "bcd", "cd", 900, 920), + ("d", "abcd", "abc", 1100, 1080), + ): + comparison = rows[name]["scoring_comparison"] + assert comparison == { + "baseline": { + "framework": "policyengine_us", + "parameter_changes": world(before), + "neutralized_variables": None, + }, + "reform": { + "framework": "policyengine_us", + "parameter_changes": world(after), + "neutralized_variables": None, + }, + "effect_direction": "reform_minus_baseline", + } + assert rows[name]["microcosm"]["baseline_total"] == base_total + assert rows[name]["microcosm"]["reform_total"] == reform_total + assert rows[name]["microcosm"]["budget_effect"] == reform_total - base_total + # The configured repeal definition keeps its original convention; + # the executed comparison explicitly records the enactment convention. + assert rows[name]["reform"]["parameter_changes"] == world(name) + assert rows[name]["reform"]["effect_direction"] == "baseline_minus_reform" + + rows["a"]["scoring_comparison"]["baseline"]["parameter_changes"]["gov.example.b"][ + "2027-01-01" + ] = 99 + rows["a"]["scoring_comparison"]["reform"]["parameter_changes"]["gov.example.b"][ + "2027-01-01" + ] = 98 + payload["obbba_pre_baseline_parameter_changes"]["gov.example.a"]["2027-01-01"] = 97 + assert rows["b"]["scoring_comparison"]["baseline"]["parameter_changes"] == world( + "bcd" + ) + assert rows["a"]["reform"]["parameter_changes"] == world("a") + assert specs == original + assert calls == [call[0] for call in expected_calls] + + +def test_obbba_claim_current_law_endpoint_and_identical_shared_patch(monkeypatch): + # Identical overlapping reverts are accepted by the existing scorer. Once + # the first row enacts the shared path, both later worlds are current law. + first = _oos_spec(-1, category="OBBBA") + second = replace(first, id="second") + monkeypatch.setattr(reform_validation_module, "_build_parameter_reform", deepcopy) + calls = [] + + def simulate(reform): + calls.append(deepcopy(reform)) + return _FakeSim({"income_tax": 30 if reform else 20}) + + payload = reform_validation_payload([first, second], period=2024, simulate=simulate) + first_row, second_row = payload["reforms"] + current_law = { + "framework": "policyengine_us", + "parameter_changes": None, + "neutralized_variables": None, + } + assert calls == [first.parameter_changes, None, None] + assert first_row["scoring_comparison"]["reform"] == current_law + assert second_row["scoring_comparison"]["baseline"] == current_law + assert second_row["scoring_comparison"]["reform"] == current_law + assert first_row["microcosm"]["budget_effect"] == -10 + assert second_row["microcosm"]["budget_effect"] == 0 + + +@pytest.mark.parametrize("category", ["OBBBA", "Other"]) +def test_obbba_claim_unscored_rows_do_not_claim_executed_baseline(category): + spec = _oos_spec(-1, category=category) + payload = reform_validation_payload([spec], period=2024) + assert payload["out_of_sample_simulated"] is False + assert payload["reforms"][0]["microcosm"]["budget_effect"] is None + assert ( + payload["reforms"][0]["reform"]["parameter_changes"] == spec.parameter_changes + ) + assert "scoring_comparison" not in payload["reforms"][0] + assert "obbba_pre_baseline_parameter_changes" not in payload + + +def test_obbba_claim_duplicate_ids_follow_existing_score_lookup(monkeypatch): + # Duplicate OBBBA IDs already use the last stacked score. Metadata must + # describe that same comparison, even with an ordinary row sharing the ID. + first = replace(_oos_spec(-1, category="OBBBA"), id="shared") + second = replace( + first, + parameter_changes={"gov.example.other": {"2027": 0}}, + period=2027, + ) + ordinary = replace(first, category="Other") + monkeypatch.setattr(reform_validation_module, "_build_parameter_reform", deepcopy) + calls = [] + + def simulate(reform): + calls.append(deepcopy(reform)) + return _FakeSim({"income_tax": [100, 90, 100, 80, 110, 105][len(calls) - 1]}) + + payload = reform_validation_payload( + [first, ordinary, second], period=2024, simulate=simulate + ) + first_row, ordinary_row, second_row = payload["reforms"] + assert len(calls) == 6 + assert first_row["microcosm"]["budget_effect"] == -20 + assert second_row["microcosm"]["budget_effect"] == -20 + assert ordinary_row["microcosm"]["budget_effect"] == -5 + assert first_row["scoring_comparison"] == second_row["scoring_comparison"] + assert second_row["scoring_comparison"]["reform"]["parameter_changes"] == calls[3] + assert ordinary_row["scoring_comparison"]["reform"]["parameter_changes"] == calls[5] + assert ordinary_row["scoring_comparison"]["baseline"]["parameter_changes"] is None + + def _oos_spec(score: float, *, category: str = "Other") -> ReformValidationSpec: return ReformValidationSpec( id="obbba_salt", @@ -1344,3 +1517,440 @@ def __init__(self, *, file_path: str) -> None: assert reformed["reform"] == "a reform" assert baseline["dataset"].file_path == str(dataset_path) assert reformed["dataset"].file_path == str(dataset_path) + + +def _spec(**overrides): + fields = { + "id": "invented_reform", + "name": "Invented reform", + "category": "Invented category", + "in_sample": False, + "period": 2031, + "jct_score": 17.0, + "jct_window": "Invented window", + "jct_source": "Invented source mentioning JCT", + "jct_source_url": "https://example.test/invented", + "neutralized_variable": "invented_credit", + } + fields.update(overrides) + return reform_validation_module.ReformValidationSpec(**fields) + + +def _level(**overrides): + fields = { + "id": "invented_level", + "name": "Invented level", + "variable": "invented_total", + "period": 2032, + "benchmark_value": 43.0, + "benchmark_year": "Invented year", + "source": "Invented source mentioning IRS", + "source_url": "https://example.test/level", + } + fields.update(overrides) + return reform_validation_module.BaselineLevelSpec(**fields) + + +class _FakeTotal: + def __init__(self, total): + self.total = total + + def sum(self): + return self.total + + +class _FakeSimulation: + def __init__(self, total, calls): + self.total = total + self.calls = calls + + def calculate(self, measure, period): + self.calls.append((measure, period)) + return _FakeTotal(self.total) + + +@pytest.mark.parametrize("factory", [_spec, _level]) +def test_claim_benchmark_publisher_is_optional_and_explicit(factory): + assert factory().benchmark_publisher is None + assert factory(benchmark_publisher="invented_agency").benchmark_publisher == ( + "invented_agency" + ) + + +@pytest.mark.parametrize( + "loader,benchmark_key,definition", + [ + ( + reform_validation_module.out_of_sample_reform_specs, + "jct", + {"parameter_changes": {"gov.invented.amount": {"2031": 7}}}, + ), + ( + reform_validation_module.tax_expenditure_reform_specs, + "benchmark", + {"neutralized_variable": "invented_credit"}, + ), + ( + reform_validation_module.state_program_reform_specs, + "benchmark", + {"neutralized_variable": ["invented_a", "invented_b"]}, + ), + ( + reform_validation_module.state_reform_specs, + "benchmark", + {"parameter_changes": {"gov.invented.amount": {"2031": 7}}}, + ), + ], +) +@pytest.mark.parametrize( + "publisher_fields,expected", + [ + ({}, None), + ({"publisher": None}, None), + ({"publisher": ""}, None), + ({"publisher": "invented_agency"}, "invented_agency"), + ], +) +def test_claim_reform_loaders_preserve_publisher_without_source_inference( + tmp_path, loader, benchmark_key, definition, publisher_fields, expected +): + raw = { + "id": "invented_reform", + "name": "Invented reform", + "period": 2031, + **definition, + benchmark_key: { + "score": -17.0, + "window": "Invented window", + "source": "Invented source mentioning JCT and IRS", + "source_url": "https://example.test/source", + **publisher_fields, + }, + } + path = tmp_path / "invented.json" + path.write_text(json.dumps({"reforms": [raw]})) + (spec,) = loader(path, period=2099) + assert spec.benchmark_publisher == expected + assert spec.period == 2031 + assert spec.jct_score == -17.0 + assert spec.jct_window == "Invented window" + assert spec.jct_source == raw[benchmark_key]["source"] + assert spec.jct_source_url == raw[benchmark_key]["source_url"] + + +@pytest.mark.parametrize( + "loader", + [ + reform_validation_module.soi_baseline_level_specs, + reform_validation_module.state_program_level_specs, + reform_validation_module.federal_eitc_state_level_specs, + reform_validation_module.state_spm_poverty_level_specs, + ], +) +@pytest.mark.parametrize( + "publisher_fields,expected", + [ + ({}, None), + ({"publisher": None}, None), + ({"publisher": ""}, None), + ({"publisher": "invented_agency"}, "invented_agency"), + ], +) +def test_claim_baseline_loader_wrappers_preserve_explicit_publisher( + tmp_path, loader, publisher_fields, expected +): + raw = { + "id": "invented_level", + "name": "Invented level", + "variable": "invented_total", + "period": 2032, + "benchmark": { + "value": 43.0, + "year": "Invented year", + "source": "Invented source mentioning Census and IRS", + "source_url": "https://example.test/level", + **publisher_fields, + }, + } + path = tmp_path / "invented.json" + path.write_text(json.dumps({"levels": [raw]})) + (spec,) = loader(path) + assert spec.benchmark_publisher == expected + assert spec.period == 2032 + assert spec.benchmark_value == 43.0 + assert spec.benchmark_year == "Invented year" + assert spec.source == raw["benchmark"]["source"] + assert spec.source_url == raw["benchmark"]["source_url"] + + +def test_claim_in_sample_factory_identifies_jct_without_simulation(): + reform = SimpleTaxExpenditureReform( + target_name="invented_target", + neutralized_variable="invented_credit", + measure="invented_target", + period=2001, + source="Invented target source", + output_variable="income_tax", + ) + (spec,) = reform_validation_module.in_sample_reform_specs([reform], period=2031) + assert spec.benchmark_publisher == "jct" + assert spec.period == 2031 + assert spec.jct_source == "Invented target source" + + def simulate(_): + pytest.fail("A supplied calibration estimate must not run a simulation") + + payload = reform_validation_module.reform_validation_payload( + [spec], + period=2030, + simulate=simulate, + in_sample_estimates={"invented_target": 11.0}, + in_sample_targets={"invented_target": 13.0}, + ) + row = payload["reforms"][0] + assert row["jct"]["publisher"] == "jct" + assert row["jct"]["score"] == 13.0 + assert row["microcosm"]["budget_effect"] == 11.0 + assert row["microcosm"]["measure"] == "income_tax" + assert row["microcosm"]["baseline_total"] is None + assert row["microcosm"]["reform_total"] is None + assert "scoring_comparison" not in row + + +@pytest.mark.parametrize( + "direction,expected_effect", + [("reform_minus_baseline", 20.0), ("baseline_minus_reform", -20.0)], +) +def test_claim_parameter_definition_is_additive_and_preserves_actual_sign_and_year( + monkeypatch, tmp_path, direction, expected_effect +): + changes = {"gov.invented.amount": {"2031-01-01.2031-12-31": 7}} + spec = _spec( + neutralized_variable=None, + parameter_changes=changes, + effect_direction=direction, + benchmark_publisher="invented_agency", + ) + token = object() + builds = [] + + def build_reform(value): + builds.append(value) + return token + + monkeypatch.setattr( + reform_validation_module.ReformValidationSpec, "build_reform", build_reform + ) + simulations = [] + calculations = [] + + def simulate(reform): + simulations.append(reform) + assert reform is None or reform is token + return _FakeSimulation(100.0 if reform is None else 120.0, calculations) + + payload = reform_validation_module.reform_validation_payload( + [spec], period=2030, simulate=simulate, release_id="invented_release" + ) + assert simulations == [None, token] + assert builds == [spec] + assert calculations == [("income_tax", 2031), ("income_tax", 2031)] + row = payload["reforms"][0] + assert row["reform"] == { + "framework": "policyengine_us", + "parameter_changes": changes, + "neutralized_variables": None, + "effect_direction": direction, + } + assert row["jct"]["publisher"] == "invented_agency" + assert row["scoring_comparison"] == { + "baseline": { + "framework": "policyengine_us", + "parameter_changes": None, + "neutralized_variables": None, + }, + "reform": { + "framework": "policyengine_us", + "parameter_changes": changes, + "neutralized_variables": None, + }, + "effect_direction": direction, + } + legacy = deepcopy(payload) + legacy["reforms"][0].pop("reform") + legacy["reforms"][0].pop("scoring_comparison") + legacy["reforms"][0]["jct"].pop("publisher") + assert legacy == { + "schema_version": 1, + "baseline_period": 2030, + "scoring_window": "see per-reform jct.window", + "out_of_sample_simulated": True, + "release_id": "invented_release", + "reforms": [ + { + "id": "invented_reform", + "name": "Invented reform", + "category": "Invented category", + "in_sample": False, + "period": 2031, + "description": None, + "jct": { + "score": 17.0, + "score_fy2027": None, + "score_type": "conventional", + "window": "Invented window", + "source": "Invented source mentioning JCT", + "source_url": "https://example.test/invented", + }, + "microcosm": { + "budget_effect": expected_effect, + "period": 2031, + "window": "Invented window", + "measure": "income_tax", + "baseline_total": 100.0, + "reform_total": 120.0, + }, + } + ], + } + path = reform_validation_module.write_reform_validation( + payload, tmp_path / "payload.json" + ) + assert json.loads(path.read_text()) == payload + row["scoring_comparison"]["reform"]["parameter_changes"]["gov.invented.amount"][ + "2031-01-01.2031-12-31" + ] = 999 + assert changes["gov.invented.amount"]["2031-01-01.2031-12-31"] == 7 + assert row["reform"]["parameter_changes"] == changes + + +@pytest.mark.parametrize( + "neutralized,expected", + [ + ("invented_credit", ["invented_credit"]), + (["invented_a", "invented_b"], ["invented_a", "invented_b"]), + ( + ["invented_b", "invented_a", "invented_b"], + ["invented_b", "invented_a", "invented_b"], + ), + ], +) +def test_claim_neutralizations_are_normalized_and_detached_from_inputs( + neutralized, expected +): + spec = _spec(neutralized_variable=neutralized) + first = reform_validation_module.reform_validation_payload([spec], period=2031) + second = reform_validation_module.reform_validation_payload([spec], period=2031) + definition = first["reforms"][0]["reform"] + assert definition == { + "framework": "policyengine_us", + "parameter_changes": None, + "neutralized_variables": expected, + "effect_direction": "reform_minus_baseline", + } + definition["neutralized_variables"].append("mutated_output") + assert second["reforms"][0]["reform"]["neutralized_variables"] == expected + assert ( + reform_validation_module.reform_validation_payload([spec], period=2031)[ + "reforms" + ][0]["reform"]["neutralized_variables"] + == expected + ) + assert spec.neutralized_variable == neutralized + if isinstance(neutralized, list): + assert neutralized == expected + + +def test_claim_parameter_definitions_deepcopy_nested_inputs_across_payloads(): + changes = {"gov.invented.amount": {"2031": [7, {"value": 9}]}} + original = deepcopy(changes) + spec = _spec(neutralized_variable=None, parameter_changes=changes) + first = reform_validation_module.reform_validation_payload([spec], period=2031) + second = reform_validation_module.reform_validation_payload([spec], period=2031) + output_changes = first["reforms"][0]["reform"]["parameter_changes"] + output_changes["gov.invented.amount"]["2031"][1]["value"] = 999 + output_changes["gov.invented.added"] = {"2031": 2} + assert changes == original + assert second["reforms"][0]["reform"]["parameter_changes"] == original + third = reform_validation_module.reform_validation_payload([spec], period=2031) + assert third["reforms"][0]["reform"]["parameter_changes"] == original + changes["gov.invented.amount"]["2031"][1]["value"] = 123 + assert second["reforms"][0]["reform"]["parameter_changes"] == original + assert third["reforms"][0]["reform"]["parameter_changes"] == original + + +@pytest.mark.parametrize("publisher", [None, "invented_agency"]) +def test_claim_baseline_rows_publish_attribution_without_reform_definition(publisher): + level = _level(benchmark_publisher=publisher) + simulations = [] + calculations = [] + + def simulate(reform): + simulations.append(reform) + return _FakeSimulation(43.0, calculations) + + payload = reform_validation_module.reform_validation_payload( + [], period=2030, simulate=simulate, baseline_levels=[level] + ) + (row,) = payload["reforms"] + assert "reform" not in row + assert "scoring_comparison" not in row + assert row["jct"]["publisher"] == publisher + assert row["jct"]["source"] == "Invented source mentioning IRS" + assert row["jct"]["score"] == 43.0 + assert row["microcosm"]["budget_effect"] == 43.0 + assert row["microcosm"]["reform_total"] is None + assert simulations == [None] + assert calculations == [("invented_total", 2032)] + + +def test_claim_unsimulated_rows_publish_definition_and_null_publisher(): + payload = reform_validation_module.reform_validation_payload( + [_spec()], period=2030, baseline_levels=[_level()] + ) + assert payload["schema_version"] == 1 + assert payload["out_of_sample_simulated"] is False + reform, level = payload["reforms"] + assert reform["reform"]["neutralized_variables"] == ["invented_credit"] + assert "reform" not in level + for row in (reform, level): + assert "scoring_comparison" not in row + assert row["jct"]["publisher"] is None + assert row["microcosm"]["budget_effect"] is None + assert row["microcosm"]["baseline_total"] is None + assert row["microcosm"]["reform_total"] is None + assert "populace" not in row + + +@pytest.mark.parametrize( + "neutralized,changes", + [(None, None), ([], {}), ("invented_credit", {"gov.invented": {"2031": 7}})], +) +def test_claim_ambiguous_or_absent_definitions_are_rejected(neutralized, changes): + with pytest.raises(ValueError, match="provide exactly one"): + _spec(neutralized_variable=neutralized, parameter_changes=changes) + + +def test_claim_invalid_direction_is_rejected_before_reporting(): + with pytest.raises(ValueError, match="effect_direction"): + _spec(effect_direction="invented_unsupported_direction") + + +def test_claim_conflicting_obbba_patches_fail_before_callback(): + specs = [ + _spec( + id=f"invented_{value}", + category="OBBBA", + neutralized_variable=None, + parameter_changes={"gov.invented.amount": {"2031": value}}, + ) + for value in (7, 9) + ] + + def simulate(_): + pytest.fail("Conflicting patches must fail before any callback") + + with pytest.raises(ValueError, match="conflicts with"): + reform_validation_module.reform_validation_payload( + specs, period=2031, simulate=simulate + ) diff --git a/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py b/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py index c9ae76814..6c48cba44 100644 --- a/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py +++ b/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py @@ -3,17 +3,22 @@ from __future__ import annotations from collections.abc import Mapping +from copy import deepcopy from dataclasses import fields, is_dataclass +from pathlib import Path import pytest from microcosm.build.spec_engine import ResourceKind, load_bundle from microcosm.build.spec_engine.canonical import canonical_json_bytes from microcosm.build.spec_engine.imputation_semantics import ( + _project_transfer_execution_identity, derive_primary_effective_predictor_tuples, project_imputation_legacy_payloads, ) +from microcosm.build.spec_engine.loader import SpecValidationError, load_schema_registry from microcosm.build.spec_engine.seeds import LEGACY_V1_PROTOCOL +from microcosm.build.spec_engine.yaml12 import load_yaml12_file from microcosm.build.us_runtime.acs_transfer import ( DEFAULT_ACS_TRANSFER_MAX_TARGETS_PER_FIT, acs_transfer_execution_contract_identity, @@ -56,6 +61,127 @@ def _json_ready(value: object) -> object: return value +@pytest.fixture(scope="module") +def transfer_document() -> dict[str, object]: + path = ( + Path(__file__).resolve().parents[1] + / "src/microcosm/build/us/spec/imputation.yaml" + ) + return load_yaml12_file(path) + + +def test_transfer_execution_source_only_renderer_is_byte_equal( + transfer_document: dict[str, object], +) -> None: + from tools.generate_us_bundle_from_constants import DEFAULT_OUTPUT_DIR, render_yaml + from tools.us_bundle_generation.imputation import build_transfer_execution + + regenerated = deepcopy(transfer_document) + regenerated["transfer_execution"] = build_transfer_execution() + assert regenerated == transfer_document + assert ( + render_yaml("imputation.yaml", regenerated) + == (DEFAULT_OUTPUT_DIR / "imputation.yaml").read_bytes() + ) + load_schema_registry().validate(regenerated, "imputation.schema.json") + + +@pytest.mark.parametrize("profile", ["acs_transfer_default", "acs_transfer_early"]) +@pytest.mark.parametrize( + "targets", + [ + [], + ["long_term_capital_gains_before_response"], + ["non_sch_d_capital_gains"], + ["pre_subsidy_care_expenses"], + ["is_incapable_of_self_care"], + ["ssn_card_type"], + ["immigration_status_str"], + ["ssn_card_type", "immigration_status_str"], + ], +) +def test_transfer_execution_boundary_identities_match_live_bytes( + transfer_document: dict[str, object], profile: str, targets: list[str] +) -> None: + projected = _project_transfer_execution_identity( + transfer_document["transfer_execution"], + transfer_document["predictor_blocks"], + profile_id=profile, + targets=targets, + ) + live = acs_transfer_execution_contract_identity( + targets=targets, derive_schedule_d=profile == "acs_transfer_early" + ) + assert canonical_json_bytes(projected) == canonical_json_bytes(live) + assert projected["sha256"] == live["sha256"] + + +def test_transfer_execution_declared_family_identities_match_live_bytes( + transfer_document: dict[str, object], +) -> None: + cases = [ + ( + "acs_transfer_early", + [ + target + for families in direction.target_families.values() + for targets in families.values() + for target in targets + ], + ) + for direction in stacked_gap_fill_plan() + ] + cases.extend( + ("acs_transfer_default", list(group.targets)) + for group in CANONICAL_US_LATE_TRANSFER_GROUPS + ) + assert len(cases) == 21 + for profile, targets in cases: + projected = _project_transfer_execution_identity( + transfer_document["transfer_execution"], + transfer_document["predictor_blocks"], + profile_id=profile, + targets=targets, + ) + live = acs_transfer_execution_contract_identity( + targets=targets, derive_schedule_d=profile == "acs_transfer_early" + ) + assert canonical_json_bytes(projected) == canonical_json_bytes(live) + + +@pytest.mark.parametrize( + "mutation", + [ + lambda transfer: transfer.pop("immigration_evidence_features"), + lambda transfer: transfer["immigration_evidence_features"].update( + {"unreviewed": True} + ), + lambda transfer: transfer["immigration_evidence_features"].update( + {"no_arrival_evidence_value": True} + ), + lambda transfer: transfer["immigration_evidence_features"][ + "asec_arrival_year_midpoints" + ].update({"29": 2025}), + lambda transfer: transfer["post_transfer_features"].pop( + "humanitarian_immigration" + ), + lambda transfer: transfer["post_transfer_features"]["humanitarian_immigration"][ + "contract" + ].update({"unreviewed": True}), + lambda transfer: transfer["post_transfer_features"][ + "schedule_d_capital_gain_distributions" + ]["activation"].update({"any_targets": []}), + ], +) +def test_transfer_execution_schema_refuses_unrepresented_semantics( + transfer_document: dict[str, object], mutation +) -> None: + changed = deepcopy(transfer_document) + mutation(changed["transfer_execution"]) + with pytest.raises(SpecValidationError): + load_schema_registry().validate(changed, "imputation.schema.json") + + @pytest.fixture(scope="module") def us_domains() -> dict[str, dict[str, object]]: spec = load_bundle("us") diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 852543734..ba4d06d55 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -15,6 +15,7 @@ SpecValidationError, bundle_lock_bytes, bundle_lock_payload, + inventory_coverage, load_bundle, load_schema_registry, ) @@ -234,9 +235,15 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non kernel_registry=KernelRegistry.from_ids(SELECTION_KERNEL_IDS), ) # Pin the domain separator, normalization rules, schema-set receipt, and - # exact normative projection as one reviewable golden vector. + # exact normative projection as one reviewable golden vector. This includes + # the prior-year module and its extracted canonical constants in the seed + # source closure; the assertion below binds the corresponding protocol pin. + assert ( + first.seed_protocol.implementation_sha256 + == (inventory_coverage.EXPECTED_HASHES["seed_protocol"]) + ) assert first.spec_sha256 == ( - "a260da046550cac52594d8e7edeaec06422ea273031e97b40d1e98209d3348b5" + "f55229a38c8b073f79ed479c17fa1b05d8e483453dd07dc3829331b0e2dff603" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_spec_engine_seeds.py b/packages/microcosm-build/tests/test_spec_engine_seeds.py index c0e181a93..a312ca68b 100644 --- a/packages/microcosm-build/tests/test_spec_engine_seeds.py +++ b/packages/microcosm-build/tests/test_spec_engine_seeds.py @@ -252,6 +252,31 @@ def test_f0_legacy_protocol_contains_no_block_first_draw_site() -> None: assert all("block" not in site.id for site in LEGACY_V1_PROTOCOL.sites) +def test_prior_year_constants_remain_in_the_direct_kernel_source_identity( + monkeypatch, +) -> None: + from microcosm.build.us_runtime import prior_year_income_constants + + kernel = next( + row for row in LEGACY_V1_PROTOCOL.kernels if row.id == "legacy_v1_direct_draws" + ) + assert prior_year_income_constants.__name__ in kernel.source_modules + assert source_inventory_sha256(kernel.source_modules) == kernel.source_sha256 + constant_path = Path(prior_year_income_constants.__file__).resolve() + read_bytes = Path.read_bytes + + def changed_constant_source(path): + data = read_bytes(path) + if path.resolve() == constant_path: + return data.replace(b'"previous_year_income_available"', b'"changed_input"') + return data + + # No source file is changed: the exact canonical module must be among the + # bytes consumed by the real source-identity function, not just imported. + monkeypatch.setattr(Path, "read_bytes", changed_constant_source) + assert source_inventory_sha256(kernel.source_modules) != kernel.source_sha256 + + def test_audited_source_manifest_is_independent_and_total() -> None: assert set(AUDITED_SOURCE_BY_SITE) == EXPECTED_LEGACY_V1_SITES assert set(AUDITED_SOURCE_BY_SITE) == {site.id for site in LEGACY_V1_PROTOCOL.sites} diff --git a/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py b/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py new file mode 100644 index 000000000..2b6477087 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py @@ -0,0 +1,151 @@ +"""Source-only context formatting: no audit install or denied file operation.""" + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + root = Path(__file__).resolve().parents[3] + spec = importlib.util.spec_from_file_location( + "spec_seed_diagnostic_context_test", + root / "tools/spec_seed_identity_diagnostics.py", + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_diagnostic_source_roster_retains_extracted_seed_constants(diagnostic): + from microcosm.build.spec_engine.seeds import LEGACY_V1_PROTOCOL + + modules = tuple( + sorted( + { + name + for kernel in LEGACY_V1_PROTOCOL.kernels + for name in kernel.source_modules + } + ) + ) + assert diagnostic.SEED_MODULES == modules + module = "microcosm.build.us_runtime.prior_year_income_constants" + assert module in modules + assert ( + "packages/microcosm-build/src/" + module.replace(".", "/") + ".py" + in diagnostic.SOURCE_PATHS + ) + + +@pytest.mark.parametrize( + ("value", "scope", "shown"), + ( + ("/repo/tools/example.py", "repository", "tools/example.py"), + ( + "/repo/packages/example/schema.json", + "repository", + "packages/example/schema.json", + ), + ("/repo/data/invented.csv", "repository", None), + ("/repo/.env", "repository", None), + ("/repo/credentials.json", "repository", None), + ("/repo/example.err.log", "repository", None), + ("/unlisted/identity/example.py", "unlisted", "example.py"), + ("/opt/python/lib/example.py", "system_opt", "python/lib/example.py"), + ), +) +def test_path_classification_is_bounded_and_redacts_data_and_credentials( + diagnostic, value, scope, shown +): + result = diagnostic.refusal_path_context( + value, (("repository", Path("/repo")), ("system_opt", Path("/opt"))) + ) + assert result["scope"] == scope + assert result["path"] == shown + assert result["name_redacted"] is (shown is None) + + +def test_requested_and_resolved_code_scopes_remain_distinct_without_io(diagnostic): + # Literal examples only: no symlink or path is created, resolved or opened. + payload = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/opt/python/lib/loader.py"), + "/environment/lib/loader.py", + (("environment", Path("/environment")), ("system_opt", Path("/opt"))), + ) + value = json.loads(payload) + assert value["requested"]["scope"] == "environment" + assert value["resolved"]["scope"] == "system_opt" + assert value["paths_are_metadata_only"] is True + assert len(payload) <= 4096 and len(value["frames"]) <= 8 + + +def test_context_never_serializes_locals_or_exception_text(diagnostic): + private_local = "invented-secret-value-that-must-never-be-exported" + error = RuntimeError(private_local) + payload = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/repo/credentials.json"), + "/repo/credentials.json", + (("repository", Path("/repo")),), + ) + assert str(error).encode() not in payload + value = json.loads(payload) + assert value["requested"]["path"] is None + assert value["resolved"]["path"] is None + assert value["code"] == "READ_SCOPE" and value["event"] == "open" + assert all(set(frame) == {"file", "function", "line"} for frame in value["frames"]) + assert len(payload) <= 4096 + + +@pytest.mark.parametrize( + ("path", "label"), + ( + ("/etc/os-release", "etc_os_release"), + ("/usr/lib/os-release", "usr_lib_os_release"), + ("/lib/os-release", "lib_os_release"), + ("/etc/localtime", "etc_localtime"), + ( + f"/runtime/lib/python{sys.version_info.major}{sys.version_info.minor}.zip", + "python_stdlib_zip_candidate", + ), + ("/runtime/lib/invented-data.zip", None), + ), +) +def test_fixed_os_metadata_labels_do_not_expose_data_path_text(diagnostic, path, label): + value = diagnostic.refusal_path_context(path, (("runtime", Path("/runtime")),)) + assert value["known_path_label"] == label + assert value["path"] is None and value["name_redacted"] is True + + +def test_terminal_refusal_retains_own_context_after_an_earlier_probe(diagnostic): + first = diagnostic.encoded({"code": "DATA_FILE", "event": "open"}) + last = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/usr/lib/os-release"), + "/etc/os-release", + (("system_usr", Path("/usr")), ("system_etc", Path("/etc"))), + ) + swallowed = diagnostic.RefusalError("DATA_FILE", boundary_context=first) + terminal = diagnostic.RefusalError("READ_SCOPE", boundary_context=last) + assert str(swallowed) == "DATA_FILE" and str(terminal) == "READ_SCOPE" + assert type(terminal.boundary_context) is bytes + assert terminal.boundary_context == last and swallowed.boundary_context == first + context = json.loads(terminal.boundary_context) + assert context["requested"]["known_path_label"] == "etc_os_release" + assert context["resolved"]["known_path_label"] == "usr_lib_os_release" + assert context["code"] != json.loads(swallowed.boundary_context)["code"] + assert diagnostic.RefusalError("WALL").boundary_context is None + + +@pytest.mark.parametrize("value", (bytearray(b"{}"), b"x" * 4097)) +def test_exception_context_requires_immutable_bounded_bytes(diagnostic, value): + with pytest.raises(TypeError, match="REFUSAL_CONTEXT"): + diagnostic.RefusalError("READ_SCOPE", boundary_context=value) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py b/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py new file mode 100644 index 000000000..9c31d579c --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py @@ -0,0 +1,226 @@ +"""Invented bootstrap controls; no Torch, Microcosm, engines or data imported.""" + +from __future__ import annotations + +import importlib +import importlib.machinery +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + # Normal stdlib-only module loading; main/audit/derive are never called. + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("cpu_bootstrap_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def fresh_import_state(monkeypatch): + # Keep the process import roster isolated without importing any dependency. + for name in tuple(sys.modules): + if ( + name == "torch" + or name == "cuda.bindings" + or name.startswith("cuda.bindings.") + ): + monkeypatch.delitem(sys.modules, name) + monkeypatch.setattr(sys, "meta_path", list(sys.meta_path)) + + +@pytest.mark.parametrize( + "name", ["json", "torch", "cuda", "cuda.pathfinder", "cuda.bindings_other"] +) +def test_non_cuda_bindings_imports_pass_through(diagnostic, name): + finder = diagnostic._CpuOnlyCudaBindingsFinder() + assert finder.find_spec(name, ["invented"], object()) is None + assert finder.attempts == 0 + + +def test_only_exact_optional_bindings_are_excluded(diagnostic): + finder = diagnostic._CpuOnlyCudaBindingsFinder() + with pytest.raises(ModuleNotFoundError) as error: + finder.find_spec("cuda.bindings") + assert error.value.name == "cuda.bindings" + assert finder.attempts == 1 + + +@pytest.mark.parametrize("fails", [False, True]) +def test_finder_removed_on_success_and_failure(diagnostic, fresh_import_state, fails): + before = tuple(sys.meta_path) + other = object() + evidence = {} + + class InventedFailureError(Exception): + pass + + try: + with diagnostic.cpu_only_torch_import(evidence): + finder = sys.meta_path[0] + assert type(finder) is diagnostic._CpuOnlyCudaBindingsFinder + assert tuple(sys.meta_path[1:]) == before + sys.meta_path.append(other) + with pytest.raises(ModuleNotFoundError): + finder.find_spec("cuda.bindings") + if fails: + raise InventedFailureError + except InventedFailureError: + assert fails + assert tuple(sys.meta_path) == (*before, other) + assert evidence == { + "cuda_bindings_import_attempts": 1, + "temporary_finder_removed": True, + } + + +@pytest.mark.parametrize("name", ["torch", "cuda.bindings", "cuda.bindings.runtime"]) +def test_preloaded_optional_path_refuses_before_finder( + diagnostic, fresh_import_state, monkeypatch, name +): + monkeypatch.setitem(sys.modules, name, SimpleNamespace()) + before = tuple(sys.meta_path) + with pytest.raises(diagnostic.RefusalError, match="^TORCH_BOOTSTRAP_PRELOADED$"): + with diagnostic.cpu_only_torch_import({}): + pytest.fail("preloaded dependency was accepted") + assert tuple(sys.meta_path) == before + + +def test_actual_fallback_contract_is_required( + diagnostic, fresh_import_state, monkeypatch +): + evidence = { + "temporary_finder_removed": True, + "cuda_bindings_import_attempts": 1, + } + monkeypatch.setitem( + sys.modules, + "torch.cuda._utils", + SimpleNamespace(_HAS_CUDA_BINDINGS=False, _cuda_bindings_runtime=None), + ) + diagnostic.verify_cpu_only_torch_import(evidence) + assert evidence["torch_optional_bindings_fallback_verified"] is True + + +@pytest.mark.parametrize("bindings_present", [False, True]) +def test_unconfirmed_or_loaded_bindings_refuse( + diagnostic, fresh_import_state, monkeypatch, bindings_present +): + evidence = { + "temporary_finder_removed": True, + "cuda_bindings_import_attempts": 1 if bindings_present else 0, + } + monkeypatch.setitem( + sys.modules, + "torch.cuda._utils", + SimpleNamespace(_HAS_CUDA_BINDINGS=False, _cuda_bindings_runtime=None), + ) + if bindings_present: + monkeypatch.setitem(sys.modules, "cuda.bindings", SimpleNamespace()) + with pytest.raises(diagnostic.RefusalError, match="^TORCH_CPU_FALLBACK$"): + diagnostic.verify_cpu_only_torch_import(evidence) + + +def _candidate(monkeypatch): + monkeypatch.setattr(sys, "base_prefix", "/invented-bootstrap-prefix") + return str( + Path(sys.base_prefix) + / "lib" + / f"python{sys.version_info.major}{sys.version_info.minor}.zip" + ) + + +def test_only_exact_confirmed_absent_zip_entry_is_removed(diagnostic, monkeypatch): + candidate = _candidate(monkeypatch) + others = ["", "/invented/code", "/invented/unrelated.zip", candidate + "-other"] + monkeypatch.setattr(sys, "path", [candidate, *others, candidate]) + checked = [] + + def absent(path): + checked.append(path) + raise FileNotFoundError(path) + + monkeypatch.setattr(diagnostic.os, "lstat", absent) + assert diagnostic.omit_absent_stdlib_zip() is True + assert checked == [candidate] + assert sys.path == others + + +@pytest.mark.parametrize("kind", ["regular", "broken_symlink"]) +def test_existing_zip_or_symlink_remains_subject_to_original_guard( + diagnostic, monkeypatch, kind +): + candidate = _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", [candidate, "other"]) + # Any successful lstat, including a broken symlink, prevents omission. + monkeypatch.setattr(diagnostic.os, "lstat", lambda path: SimpleNamespace(kind=kind)) + assert diagnostic.omit_absent_stdlib_zip() is False + assert sys.path == [candidate, "other"] + + +def test_other_stat_errors_do_not_justify_omission(diagnostic, monkeypatch): + candidate = _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", [candidate]) + + def denied(path): + raise PermissionError(path) + + monkeypatch.setattr(diagnostic.os, "lstat", denied) + with pytest.raises(PermissionError): + diagnostic.omit_absent_stdlib_zip() + assert sys.path == [candidate] + + +def test_unlisted_stdlib_candidate_is_not_probed(diagnostic, monkeypatch): + _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", ["other"]) + monkeypatch.setattr( + diagnostic.os, "lstat", lambda path: pytest.fail("unexpected metadata probe") + ) + assert diagnostic.omit_absent_stdlib_zip() is False + assert sys.path == ["other"] + + +def test_real_non_cuda_import_keeps_original_loader( + diagnostic, fresh_import_state, monkeypatch +): + name = "invented_non_cuda_bootstrap_control" + monkeypatch.delitem(sys.modules, name, raising=False) + loaded = [] + + class Loader: + def create_module(self, spec): + return None + + def exec_module(self, module): + loaded.append(module.__name__) + module.marker = "ordinary_loader" + + loader = Loader() + + class Finder: + def find_spec(self, fullname, path=None, target=None): + if fullname == name: + return importlib.machinery.ModuleSpec(fullname, loader) + return None + + finder = Finder() + sys.meta_path.insert(0, finder) + before = tuple(sys.meta_path) + try: + with diagnostic.cpu_only_torch_import({}): + module = importlib.import_module(name) + assert module.marker == "ordinary_loader" + assert module.__loader__ is loader + assert loaded == [name] + assert tuple(sys.meta_path) == before + finally: + sys.modules.pop(name, None) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py b/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py new file mode 100644 index 000000000..2771d8988 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py @@ -0,0 +1,186 @@ +"""Pinned engine parameter files and the urllib3 probe: invented hooks only. + +The derive phase imports policyengine-us, which reads eight public parameter +CSVs at module import. The boundary may open exactly those files, only when +their bytes hash to the pins verified before the hook is armed; every other +data-suffixed open still refuses. urllib3's import-time IPv6 probe creates one +socket and binds it to loopback from ``_has_ipv6``; nothing else on a socket is +accepted. +""" + +import base64 +import hashlib +import importlib.util +import os +import types +from importlib import metadata +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("engine_parameter_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _boundary(diagnostic, monkeypatch, allowed): + hooks, first, distinct, contexts = [], [], [], {} + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + monkeypatch.setattr(Path, "resolve", lambda path: path) + monkeypatch.setattr(Path, "cwd", classmethod(lambda cls: Path("/invented/repo"))) + diagnostic.install_boundary( + Path("/invented/repo"), + Path("/invented/owned"), + Path("/invented/output"), + first_refusal=first, + distinct_refusals=distinct, + code_contexts=contexts, + allowed_data_files=allowed, + ) + assert len(hooks) == 1 + return hooks[0], distinct + + +@pytest.mark.requires_us +def test_pins_match_the_installed_distribution_record(diagnostic): + """Every pin is a regular file in the distribution whose RECORD digest agrees.""" + + dist = metadata.distribution(diagnostic.ENGINE_PUBLIC_PARAMETER_DISTRIBUTION) + root = Path(dist.locate_file("")).resolve() + record = {} + for entry in dist.files or (): + if entry.hash is not None and entry.hash.mode == "sha256": + padded = entry.hash.value + "=" * (-len(entry.hash.value) % 4) + record[str(entry)] = base64.urlsafe_b64decode(padded).hex() + assert len(diagnostic.ENGINE_PUBLIC_PARAMETER_FILES) == 8 + for relative, expected, size in diagnostic.ENGINE_PUBLIC_PARAMETER_FILES: + path = root / relative + assert path.is_file() and not path.is_symlink(), relative + raw = path.read_bytes() + assert len(raw) == size and hashlib.sha256(raw).hexdigest() == expected + assert record[relative] == expected + report, allowed = diagnostic.pinned_engine_parameter_files() + assert report["policy"] == "pinned_engine_public_parameter_files_v1" + assert report["count"] == 8 and report["version"] == dist.version + assert allowed == frozenset( + (root / relative).resolve() + for relative, _digest, _size in diagnostic.ENGINE_PUBLIC_PARAMETER_FILES + ) + + +def test_a_pin_whose_bytes_differ_refuses_before_arming(diagnostic, tmp_path): + """Verification fails closed on bytes, size, RECORD, or a missing file.""" + + relative = "invented_engine/parameters/table.csv" + root = tmp_path / "site-packages" + (root / "invented_engine/parameters").mkdir(parents=True) + raw = b"invented,public,parameter\n1,2,3\n" + (root / relative).write_bytes(raw) + good = hashlib.sha256(raw).hexdigest() + encoded = base64.urlsafe_b64encode(hashlib.sha256(raw).digest()).rstrip(b"=") + + class _Hash: + mode = "sha256" + + def __init__(self, value: str) -> None: + self.value = value + + class _Entry(str): + hash = _Hash(encoded.decode()) + + class _Distribution: + version = "0.0.0-invented" + files = [_Entry(relative)] + + def locate_file(self, _name): + return root + + def distribution(name): + assert name == "invented-engine" + return _Distribution() + + real = metadata.distribution + try: + metadata.distribution = distribution + report, allowed = diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=((relative, good, len(raw)),) + ) + assert report["count"] == 1 and allowed == {(root / relative).resolve()} + for pins in ( + ((relative, "0" * 64, len(raw)),), + ((relative, good, len(raw) + 1),), + (("invented_engine/parameters/absent.csv", good, len(raw)),), + (("../outside.csv", good, len(raw)),), + ): + with pytest.raises(diagnostic.RefusalError, match="^ENGINE_PARAMETER_PIN$"): + diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=pins + ) + _Entry.hash = _Hash("AAAA") # RECORD disagrees with the file bytes. + with pytest.raises(diagnostic.RefusalError, match="^ENGINE_PARAMETER_PIN$"): + diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=((relative, good, len(raw)),) + ) + finally: + metadata.distribution = real + + +def test_hook_opens_only_the_allowed_data_files(diagnostic, monkeypatch): + allowed = frozenset({Path("/invented/site-packages/engine/parameters/pinned.csv")}) + hook, distinct = _boundary(diagnostic, monkeypatch, allowed) + hook("open", (str(next(iter(allowed))), "r", os.O_RDONLY)) + assert distinct == [] + for path in ( + "/invented/site-packages/engine/parameters/other.csv", + "/invented/site-packages/engine/parameters/pinned.csv.bak.csv", + "/invented/repo/pinned.csv", + ): + with pytest.raises(diagnostic.RefusalError, match="^DATA_FILE$"): + hook("open", (path, "r", os.O_RDONLY)) + assert distinct == ["DATA_FILE"] + + +def test_proc_stat_is_readable_metadata(diagnostic, monkeypatch): + hook, distinct = _boundary(diagnostic, monkeypatch, frozenset()) + hook("open", ("/proc/stat", "r", os.O_RDONLY)) + assert distinct == [] + with pytest.raises(diagnostic.RefusalError, match="^READ_SCOPE$"): + hook("open", ("/proc/version", "r", os.O_RDONLY)) + + +def _call_as_has_ipv6(diagnostic, event, args): + """Invoke the hook from a frame named like urllib3's probe function.""" + + code = compile( + "def _has_ipv6(hook, event, args):\n return hook(event, args)\n", + "/invented/site-packages/urllib3/util/connection.py", + "exec", + ) + namespace: dict[str, object] = {} + exec(code, namespace) + return namespace["_has_ipv6"](diagnostic, event, args) + + +def test_urllib3_ipv6_probe_is_the_only_accepted_socket_use(diagnostic, monkeypatch): + hook, distinct = _boundary(diagnostic, monkeypatch, frozenset()) + sock = types.SimpleNamespace() + _call_as_has_ipv6(hook, "socket.__new__", (sock, 10, 1, 0)) + _call_as_has_ipv6(hook, "socket.bind", (sock, ("::1", 0))) + _call_as_has_ipv6(hook, "socket.bind", (sock, ("127.0.0.1", 0))) + assert distinct == [] + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + _call_as_has_ipv6(hook, "socket.bind", (sock, ("0.0.0.0", 0))) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + _call_as_has_ipv6(hook, "socket.connect", (sock, ("::1", 80))) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook("socket.__new__", (sock, 10, 1, 0)) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook("socket.bind", (sock, ("::1", 0))) + assert distinct == ["NETWORK_OR_CHILD"] diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_lock.py b/packages/microcosm-build/tests/test_spec_seed_identity_lock.py new file mode 100644 index 000000000..07764e2f8 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_lock.py @@ -0,0 +1,40 @@ +"""Checked-in source hashing only; no diagnostic bootstrap or runtime execution.""" + +import hashlib +import importlib.util +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("seed_lock_controls", source) + module = importlib.util.module_from_spec(spec) + # Read the checked-in source directly; do not consume a cached bytecode file. + exec(compile(source.read_bytes(), str(source), "exec"), module.__dict__) + return module + + +def test_checked_in_lock_passes_actual_source_stamps(diagnostic): + root = Path(__file__).resolve().parents[3] + stamps = diagnostic.source_stamps(root) + assert ( + stamps["uv.lock"] == hashlib.sha256((root / "uv.lock").read_bytes()).hexdigest() + ) + assert set(stamps) == { + *diagnostic.SOURCE_PATHS, + "tools/spec_seed_identity_diagnostics.py", + "uv.lock", + ".github/workflows/test.yml", + } + + +def test_actual_source_stamps_refuses_an_incorrect_lock_pin(diagnostic, monkeypatch): + root = Path(__file__).resolve().parents[3] + monkeypatch.setattr(diagnostic, "LOCK_SHA256", "0" * 64) + with pytest.raises(diagnostic.RefusalError, match="^LOCK$"): + diagnostic.source_stamps(root) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py b/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py new file mode 100644 index 000000000..a24788b54 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py @@ -0,0 +1,107 @@ +"""Formatting-only outer-failure controls; no main/audit/dependency execution.""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("outer_failure_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("kind", [TypeError, PermissionError]) +def test_outer_error_keeps_fixed_type_and_code_coordinates_without_message( + diagnostic, kind +): + try: + raise kind("secret-password.csv user-input-payload") + except kind as error: + payload = diagnostic.outer_failure_context(error) + value = json.loads(payload) + assert value["exception_type"] == kind.__name__ + assert value["code"] == "OUTER_FAILURE" + assert value["completed"] is value["coverage_pass"] is False + assert value["artifact_status_not_modified"] is True + assert b"secret-password" not in payload + assert b"user-input-payload" not in payload + assert len(payload) <= 8192 + + +def test_actual_short_named_traceback_frame_is_retained(diagnostic): + def cleanup_probe(): + raise TypeError("not emitted") + + try: + cleanup_probe() + except TypeError as error: + result = json.loads(diagnostic.outer_failure_context(error)) + assert result["traceback_code_frames"][-1]["function"] == "cleanup_probe" + assert result["traceback_code_frames"][-1]["line"] > 0 + + +def test_fixed_refusal_retains_attached_sanitized_context(diagnostic): + context = {"code": "READ_SCOPE", "event": "open", "requested": {"path": None}} + error = diagnostic.RefusalError( + "READ_SCOPE", boundary_context=diagnostic.encoded(context) + ) + result = json.loads(diagnostic.outer_failure_context(error)) + assert result["exception_type"] == "RefusalError" + assert result["code"] == "READ_SCOPE" + assert result["boundary_refusal"] == context + + +def test_unknown_refusal_and_exception_names_are_fixed_labels(diagnostic): + class SecretNamedError(Exception): + pass + + errors = ( + diagnostic.RefusalError("arbitrary-secret-code"), + SecretNamedError("secret-message"), + ) + for error in errors: + payload = diagnostic.outer_failure_context(error) + assert b"secret" not in payload.lower() + assert b"SecretNamedError" not in payload + result = json.loads(payload) + assert result["code"] in {"OUTER_FAILURE", "OTHER_FIXED_REFUSAL"} + + +def test_traceback_scan_and_output_are_bounded(diagnostic): + def recurse(depth): + if depth: + recurse(depth - 1) + else: + raise ValueError("unprinted") + + try: + recurse(40) + except ValueError as error: + payload = diagnostic.outer_failure_context(error) + result = json.loads(payload) + assert result["traceback_truncated"] is True + assert len(result["traceback_code_frames"]) <= 8 + assert len(payload) <= 8192 + + +def test_exception_only_metadata_does_not_open_or_resolve_files( + diagnostic, monkeypatch +): + def forbidden(*args, **kwargs): + pytest.fail("outer report attempted file I/O or resolution") + + monkeypatch.setattr(Path, "read_bytes", forbidden) + monkeypatch.setattr(Path, "read_text", forbidden) + monkeypatch.setattr(Path, "resolve", forbidden) + monkeypatch.setattr(Path, "stat", forbidden) + monkeypatch.setattr(Path, "open", forbidden) + payload = diagnostic.outer_failure_context(TypeError("no traceback")) + assert json.loads(payload)["traceback_code_frames"] == [] diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py b/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py new file mode 100644 index 000000000..2fe9d1739 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py @@ -0,0 +1,60 @@ +"""Owned temporary-directory controls; no audit, main, Torch or Microcosm run.""" + +import importlib.util +import stat +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("owned_temp_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("refuses", [False, True]) +def test_owned_fixture_survives_success_and_exact_refusal( + diagnostic, tmp_path, refuses +): + context = b'{"code":"READ_SCOPE","event":"open"}\n' + error = diagnostic.RefusalError("READ_SCOPE", boundary_context=context) + caught = None + try: + with diagnostic.retained_owned_directory(tmp_path) as owned: + assert owned.parent == tmp_path + assert owned.name.startswith("spec-seed-owned-") + assert stat.S_IMODE(owned.stat().st_mode) == 0o700 + fixture = owned / "invented-spec.txt" + fixture.write_text("invented fixture only") + if refuses: + raise error + except diagnostic.RefusalError as actual: + caught = actual + assert caught is (error if refuses else None) + if caught is not None: + assert caught.boundary_context is context + assert fixture.read_text() == "invented fixture only" + assert owned.is_dir() + + +def test_each_workspace_is_distinct_and_has_no_implicit_cleanup( + diagnostic, tmp_path, monkeypatch +): + def forbidden(*args, **kwargs): + pytest.fail("implicit TemporaryDirectory cleanup was registered") + + monkeypatch.setattr(diagnostic.tempfile, "TemporaryDirectory", forbidden) + with diagnostic.retained_owned_directory(tmp_path) as first: + (first / "invented.txt").write_text("first") + with diagnostic.retained_owned_directory(tmp_path) as second: + (second / "invented.txt").write_text("second") + assert first != second + assert first.parent == second.parent == tmp_path + assert (first / "invented.txt").read_text() == "first" + assert (second / "invented.txt").read_text() == "second" diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py b/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py new file mode 100644 index 000000000..0ee55f9a9 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py @@ -0,0 +1,228 @@ +"""Invented audit calls only; no real hook, source read, child or derivation.""" + +import importlib.util +import json +import os +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("per_code_context_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def audited(diagnostic, monkeypatch): + hooks, first, distinct, contexts = [], [], [], {} + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + monkeypatch.setattr(Path, "resolve", lambda path: path) + monkeypatch.setattr(Path, "cwd", classmethod(lambda cls: Path("/invented/repo"))) + refusals = diagnostic.install_boundary( + Path("/invented/repo"), + Path("/invented/owned"), + Path("/invented/output"), + first_refusal=first, + distinct_refusals=distinct, + code_contexts=contexts, + ) + assert len(hooks) == 1 + return hooks[0], refusals, first, distinct, contexts + + +def test_caught_middle_child_denial_keeps_own_context(diagnostic, audited): + hook, refusals, first, distinct, contexts = audited + terminal = None + secret = "invented-secret-never-retain" + for event, args, code in ( + ("open", ("/proc/version", "r", os.O_RDONLY), "READ_SCOPE"), + ( + "subprocess.Popen", + (secret, [secret], secret, {secret: secret}), + "NETWORK_OR_CHILD", + ), + ("open", ("/invented/repo/never-open.csv", "r", os.O_RDONLY), "DATA_FILE"), + ): + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$") as error: + hook(event, args) + terminal = error.value + # The historical refusal counter remains first-code-only. Every operation + # still raises; retaining context must not turn a swallowed denial into success. + assert refusals == ["READ_SCOPE"] + assert distinct == ["READ_SCOPE", "NETWORK_OR_CHILD", "DATA_FILE"] + assert list(contexts) == distinct + assert first == [contexts["READ_SCOPE"]] + assert terminal.boundary_context == contexts["DATA_FILE"] + child = json.loads(contexts["NETWORK_OR_CHILD"]) + assert child["code"] == "NETWORK_OR_CHILD" and child["event"] == "subprocess.Popen" + assert child["frames"] and len(child["frames"]) <= 8 + assert child["requested"]["path"] is None and child["resolved"]["path"] is None + for payload in contexts.values(): + assert type(payload) is bytes and len(payload) <= 4096 + assert secret.encode() not in payload and b"never-open.csv" not in payload + + +def test_repeated_code_retains_first_record_but_exception_has_fresh_context( + diagnostic, audited +): + hook, refusals, first, distinct, contexts = audited + + def original_caller(): + hook("subprocess.Popen", ("invented", [], None, None)) + + def later_caller(): + hook("subprocess.Popen", ("invented", [], None, None)) + + with pytest.raises(diagnostic.RefusalError) as original: + original_caller() + retained = contexts["NETWORK_OR_CHILD"] + with pytest.raises(diagnostic.RefusalError) as later: + later_caller() + assert contexts == {"NETWORK_OR_CHILD": retained} + assert first == [retained] and refusals == distinct == ["NETWORK_OR_CHILD"] + assert original.value.boundary_context == retained + assert later.value.boundary_context != retained + assert "original_caller" in [f["function"] for f in json.loads(retained)["frames"]] + assert "later_caller" in [ + f["function"] for f in json.loads(later.value.boundary_context)["frames"] + ] + + +def test_eight_record_capacity_never_discards_or_grows(diagnostic, audited): + hook, refusals, first, distinct, contexts = audited + # A full invented collector exercises the bound even though the current + # boundary emits only seven different fixed codes. + codes = ( + "WRITE_SCOPE", + "DATA_FILE", + "FILESYSTEM_PATH", + "FILESYSTEM_DESCRIPTOR", + "FILESYSTEM_LINK", + "NETWORK_OR_CHILD", + "WALL", + "OUTPUT_CHANGED", + ) + contexts.update({code: diagnostic.encoded({"code": code}) for code in codes}) + retained = tuple(contexts.items()) + with pytest.raises(diagnostic.RefusalError, match="^READ_SCOPE$") as error: + hook("open", ("/proc/version", "r", os.O_RDONLY)) + assert tuple(contexts.items()) == retained and len(contexts) == 8 + assert refusals == distinct == ["READ_SCOPE"] + assert first == [error.value.boundary_context] + + +@pytest.mark.parametrize( + ("event", "label"), + ( + ("os.posix_spawn", "os.posix_spawn"), + ("socket.connect", "network_or_child_event"), + ("subprocess.invented-secret-event", "network_or_child_event"), + ), +) +def test_network_context_event_is_fixed_and_never_serializes_arguments( + diagnostic, audited, event, label +): + hook, _, _, _, contexts = audited + secret = "invented-secret-arguments" + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook(event, (secret, {secret: secret})) + payload = contexts["NETWORK_OR_CHILD"] + assert json.loads(payload)["event"] == label + assert secret.encode() not in payload and b"invented-secret-event" not in payload + + +def test_context_capture_reads_no_source_files_or_locals( + diagnostic, audited, monkeypatch +): + hook, _, _, _, contexts = audited + private_local = "invented-private-local-not-evidence" + + def forbidden(*args, **kwargs): + pytest.fail("context formatting attempted a source or filesystem read") + + with monkeypatch.context() as patch: + for name in ("open", "read_bytes", "read_text", "stat", "lstat"): + patch.setattr(Path, name, forbidden) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook("subprocess.Popen", (private_local,)) + payload = contexts["NETWORK_OR_CHILD"] + assert private_local.encode() not in payload + assert all( + set(frame) == {"file", "function", "line"} + for frame in json.loads(payload)["frames"] + ) + + +def test_context_capture_failure_does_not_replace_denial_with_exception_text( + diagnostic, audited, monkeypatch +): + hook, refusals, first, _, contexts = audited + + def unavailable(*args): + raise RuntimeError("invented-sensitive-exception-text") + + monkeypatch.setattr(diagnostic, "read_refusal_context", unavailable) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$") as error: + hook("subprocess.Popen", ("invented",)) + expected = diagnostic.encoded( + {"code": "NETWORK_OR_CHILD", "context": "unavailable"} + ) + assert refusals == ["NETWORK_OR_CHILD"] + assert first == [expected] and contexts == {"NETWORK_OR_CHILD": expected} + assert error.value.boundary_context == expected + + +def test_all_maximum_records_fit_status_cap_and_preserve_aggregate_cap(diagnostic): + base = diagnostic.encoded({"code": "READ_SCOPE", "context": ""}) + maximum = diagnostic.encoded( + {"code": "READ_SCOPE", "context": "x" * (4096 - len(base))} + ) + assert len(maximum) == 4096 + status = diagnostic.encoded( + { + "completed": False, + "coverage_pass": False, + "first_boundary_refusal": json.loads(maximum), + "terminal_boundary_refusal": json.loads(maximum), + "boundary_refusal_contexts": [json.loads(maximum)] * 8, + "boundary_refusal_context_limit": {"codes": 8, "bytes_per_code": 4096}, + } + ) + assert 16 * 1024 < len(status) < 64 * 1024 + assert diagnostic.CAPS == { + "candidate-digests.json": 64 * 1024, + "seed-protocol.json": 256 * 1024, + "seed-map.json": 256 * 1024, + "seed-bindings.json": 128 * 1024, + "environment-and-source.json": 128 * 1024, + "diagnostic-status.json": 64 * 1024, + } + assert diagnostic.MAX_TOTAL == 1024 * 1024 + payloads = {name: b"{}" for name in diagnostic.CAPS} + payloads["diagnostic-status.json"] = status + diagnostic.validate_payloads(payloads) + + +@pytest.mark.parametrize( + "name", + ( + "candidate-digests.json", + "seed-protocol.json", + "seed-map.json", + "seed-bindings.json", + "environment-and-source.json", + "diagnostic-status.json", + ), +) +def test_each_existing_output_cap_still_refuses_before_publication(diagnostic, name): + payloads = {key: b"{}" for key in diagnostic.CAPS} + payloads[name] = b"x" * (diagnostic.CAPS[name] + 1) + with pytest.raises(diagnostic.RefusalError, match="^OUTPUT_CAP$"): + diagnostic.validate_payloads(payloads) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py new file mode 100644 index 000000000..8b4c89b98 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py @@ -0,0 +1,353 @@ +"""Invented metadata controls; no main, engine, data, child or real audit hook.""" + +import importlib.util +import os +import platform +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("system_metadata_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def invented_uname(diagnostic, monkeypatch): + observed = SimpleNamespace( + sysname="Linux", + nodename="invented-host", + release="invented-release", + version="invented-version", + machine="invented_arch", + ) + info = platform.uname_result( + observed.sysname, + observed.nodename, + observed.release, + observed.version, + observed.machine, + ) + monkeypatch.setattr(diagnostic.sys, "platform", "linux") + monkeypatch.setattr(diagnostic.os, "uname", lambda: observed) + monkeypatch.setattr(diagnostic.platform, "uname", lambda: info) + + def forbidden(*args, **kwargs): + pytest.fail("processor metadata attempted a subprocess") + + monkeypatch.setattr(subprocess, "check_output", forbidden) + return info, observed + + +def test_real_cached_property_uses_machine_without_replacing_stdlib( + diagnostic, invented_uname +): + info, observed = invented_uname + before = ( + platform.uname, + platform.processor, + platform.uname_result, + vars(type(info))["processor"], + ) + bootstrap = {} + state = diagnostic.prime_processor_metadata(bootstrap) + assert platform.processor() == observed.machine + assert tuple(info) == ( + observed.sysname, + observed.nodename, + observed.release, + observed.version, + observed.machine, + observed.machine, + ) + assert bootstrap["system_metadata"]["uname_p_parity_claimed"] is False + assert bootstrap["system_metadata"]["source"] == "os.uname.machine" + assert before == ( + platform.uname, + platform.processor, + platform.uname_result, + vars(type(info))["processor"], + ) + diagnostic.verify_processor_metadata(state, bootstrap) + # An already-cached identical machine value is not silently altered. + again = diagnostic.prime_processor_metadata(bootstrap) + assert again == state + + +def test_conflicting_observed_processor_cache_is_not_overwritten( + diagnostic, invented_uname +): + info, _ = invented_uname + info.processor = "different_observed_processor" + bootstrap = {} + with pytest.raises(diagnostic.RefusalError, match="^PROCESSOR_CACHE_CONFLICT$"): + diagnostic.prime_processor_metadata(bootstrap) + assert info.processor == "different_observed_processor" + assert bootstrap == {} + + +@pytest.mark.parametrize("change", ("cached_value", "os_machine", "evidence")) +def test_metadata_change_refuses(diagnostic, invented_uname, change): + info, observed = invented_uname + bootstrap = {} + state = diagnostic.prime_processor_metadata(bootstrap) + if change == "cached_value": + info.processor = "changed" + elif change == "os_machine": + observed.machine = "changed" + else: + bootstrap["system_metadata"]["uname_p_parity_claimed"] = True + with pytest.raises(diagnostic.RefusalError, match="^PROCESSOR_METADATA_CHANGED$"): + diagnostic.verify_processor_metadata(state, bootstrap) + + +@pytest.fixture +def audited(diagnostic, monkeypatch): + hooks = [] + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + monkeypatch.setattr(diagnostic.os, "getpid", lambda: 321) + monkeypatch.setattr(diagnostic.os, "readlink", lambda _: "pipe:[invented]") + monkeypatch.setattr(Path, "cwd", classmethod(lambda cls: Path("/invented/repo"))) + + def resolve(path): + return Path("/proc/321/maps") if path == Path("/proc/self/maps") else path + + monkeypatch.setattr(Path, "resolve", resolve) + refusals = diagnostic.install_boundary( + Path("/invented/repo"), Path("/invented/owned"), Path("/invented/output") + ) + assert len(hooks) == 1 + return hooks[0], refusals + + +@pytest.mark.parametrize("path", ("/proc/self/maps", "/proc/321/maps")) +def test_only_logical_and_resolved_own_maps_are_admitted(audited, path): + hook, refusals = audited + hook("open", (path, "r", os.O_RDONLY)) + assert refusals == [] + + +@pytest.mark.parametrize( + "kind", ("other_process", "other_file", "relative", "descriptor", "child") +) +def test_maps_alias_does_not_admit_other_paths_descriptors_or_children( + diagnostic, audited, kind +): + hook, refusals = audited + path = { + "other_process": "/proc/322/maps", + "other_file": "/proc/321/mem", + "relative": "maps", + "descriptor": 55, + "child": None, + }[kind] + code = "NETWORK_OR_CHILD" if kind == "child" else "READ_SCOPE" + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$"): + if kind == "child": + hook("subprocess.Popen", ("uname", ["uname", "-p"], None, None)) + else: + hook("open", (path, "r", os.O_RDONLY)) + assert refusals == [code] + + +@pytest.fixture +def coordinate_environment(diagnostic, monkeypatch): + """Exercise coordinate validation against an isolated invented environment.""" + values = { + "DIAG_CHECKOUT_SHA": "1" * 40, + "GITHUB_SHA": "1" * 40, + "DIAG_WORKFLOW_SHA": "2" * 40, + "DIAG_PR_HEAD_SHA": "3" * 40, + "DIAG_PR_BASE_SHA": "4" * 40, + "DIAG_MERGE_SHA": "5" * 40, + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_RUN_ID": "123456", + "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_REPOSITORY": "PolicyEngine/microcosm", + } + # coordinates() consumes only environ. Keep the real process environment, + # filesystem, dependency imports and bootstrap paths outside these cases. + monkeypatch.setattr(diagnostic, "os", SimpleNamespace(environ=values)) + return values + + +@pytest.mark.parametrize("event", ("pull_request", "push")) +def test_coordinates_preserve_all_distinct_supplied_identities( + diagnostic, coordinate_environment, event +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + original = dict(values) + assert diagnostic.coordinates() == { + "checkout_sha": "1" * 40, + "event_sha": "1" * 40, + "workflow_sha": "2" * 40, + "pr_head_sha": "3" * 40, + "pr_base_sha": "4" * 40, + "event_merge_sha": "5" * 40, + "github_run_id": "123456", + "github_run_attempt": "2", + "event": event, + "repository": "PolicyEngine/microcosm", + } + assert values == original + + +@pytest.mark.parametrize("missing", (False, True)) +def test_pull_request_coordinates_keep_absent_merge_metadata_empty( + diagnostic, coordinate_environment, missing +): + values = coordinate_environment + if missing: + del values["DIAG_MERGE_SHA"] + else: + values["DIAG_MERGE_SHA"] = "" + result = diagnostic.coordinates() + assert result["event"] == "pull_request" + assert result["event_merge_sha"] == "" + assert result["checkout_sha"] == result["event_sha"] == "1" * 40 + assert result["workflow_sha"] == "2" * 40 + assert result["pr_head_sha"] == "3" * 40 + assert result["pr_base_sha"] == "4" * 40 + assert values.get("DIAG_MERGE_SHA", "") == "" + + +@pytest.mark.parametrize("missing", (False, True)) +def test_push_coordinates_allow_absent_pull_request_metadata( + diagnostic, coordinate_environment, missing +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = "push" + for name in ("DIAG_PR_HEAD_SHA", "DIAG_PR_BASE_SHA", "DIAG_MERGE_SHA"): + if missing: + del values[name] + else: + values[name] = "" + result = diagnostic.coordinates() + assert result["event"] == "push" + assert result["pr_head_sha"] == result["pr_base_sha"] == "" + assert result["event_merge_sha"] == "" + assert result["checkout_sha"] == result["event_sha"] == "1" * 40 + assert result["workflow_sha"] == "2" * 40 + + +@pytest.mark.parametrize( + ("event", "name"), + ( + ("pull_request", "DIAG_CHECKOUT_SHA"), + ("pull_request", "GITHUB_SHA"), + ("pull_request", "DIAG_WORKFLOW_SHA"), + ("pull_request", "DIAG_PR_HEAD_SHA"), + ("pull_request", "DIAG_PR_BASE_SHA"), + ("push", "DIAG_CHECKOUT_SHA"), + ("push", "GITHUB_SHA"), + ("push", "DIAG_WORKFLOW_SHA"), + ), +) +def test_required_coordinate_hashes_still_refuse_when_absent( + diagnostic, coordinate_environment, event, name +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + del values[name] + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("event", "name"), + ( + ("pull_request", "DIAG_CHECKOUT_SHA"), + ("pull_request", "GITHUB_SHA"), + ("pull_request", "DIAG_WORKFLOW_SHA"), + ("pull_request", "DIAG_PR_HEAD_SHA"), + ("pull_request", "DIAG_PR_BASE_SHA"), + ("pull_request", "DIAG_MERGE_SHA"), + ("push", "DIAG_PR_HEAD_SHA"), + ("push", "DIAG_PR_BASE_SHA"), + ("push", "DIAG_MERGE_SHA"), + ), +) +def test_nonempty_coordinate_hashes_still_require_valid_format( + diagnostic, coordinate_environment, event, name +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + values[name] = "not-a-sha" + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + "value", ("null", "A" * 40, "f" * 39, "f" * 41, "f" * 40 + "\n") +) +def test_missing_merge_exception_does_not_accept_malformed_present_hashes( + diagnostic, coordinate_environment, value +): + coordinate_environment["DIAG_MERGE_SHA"] = value + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize("event", ("pull_request", "push")) +def test_checkout_must_match_event_even_without_merge_metadata( + diagnostic, coordinate_environment, event +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + values["DIAG_MERGE_SHA"] = "" + values["DIAG_CHECKOUT_SHA"] = "6" * 40 + with pytest.raises(diagnostic.RefusalError, match="^CHECKOUT$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("name", "value"), + ( + ("GITHUB_RUN_ID", None), + ("GITHUB_RUN_ID", ""), + ("GITHUB_RUN_ID", "not-a-run"), + ("GITHUB_RUN_ATTEMPT", None), + ("GITHUB_RUN_ATTEMPT", ""), + ("GITHUB_RUN_ATTEMPT", "not-an-attempt"), + ), +) +def test_coordinate_run_identity_remains_required( + diagnostic, coordinate_environment, name, value +): + if value is None: + del coordinate_environment[name] + else: + coordinate_environment[name] = value + with pytest.raises(diagnostic.RefusalError, match="^RUN_COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("name", "value", "code"), + ( + ("GITHUB_EVENT_NAME", None, "EVENT"), + ("GITHUB_EVENT_NAME", "", "EVENT"), + ("GITHUB_EVENT_NAME", "workflow_dispatch", "EVENT"), + ("GITHUB_REPOSITORY", None, "REPOSITORY"), + ("GITHUB_REPOSITORY", "invented/other", "REPOSITORY"), + ), +) +def test_coordinate_event_and_repository_boundaries_remain_required( + diagnostic, coordinate_environment, name, value, code +): + if value is None: + del coordinate_environment[name] + else: + coordinate_environment[name] = value + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$"): + diagnostic.coordinates() diff --git a/packages/microcosm-build/tests/test_spec_worker_identity_lock.py b/packages/microcosm-build/tests/test_spec_worker_identity_lock.py new file mode 100644 index 000000000..516f5398e --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_worker_identity_lock.py @@ -0,0 +1,83 @@ +"""Actual lock validation without worker, engine or identity-probe execution.""" + +import hashlib +import importlib.util +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture +def worker_identity(monkeypatch): + root = Path(__file__).resolve().parents[3] + source = root / ( + "packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py" + ) + spec = importlib.util.spec_from_file_location("worker_lock_controls", source) + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, module) + exec(compile(source.read_bytes(), str(source), "exec"), module.__dict__) + # Locate the real checked-in lock without importing the PUF worker. + monkeypatch.setattr(module, "_repository_root", lambda: root) + return module + + +def test_checked_in_lock_passes_actual_worker_validation(worker_identity): + root = Path(__file__).resolve().parents[3] + assert ( + worker_identity._approved_uv_lock_sha256() + == hashlib.sha256((root / "uv.lock").read_bytes()).hexdigest() + ) + + +def test_actual_worker_validation_refuses_an_incorrect_pin( + worker_identity, monkeypatch +): + monkeypatch.setattr(worker_identity, "APPROVED_UV_LOCK_SHA256", "0" * 64) + with pytest.raises(RuntimeError, match="unapproved uv.lock digest"): + worker_identity._approved_uv_lock_sha256() + + +@pytest.mark.parametrize( + "digest", + [ + # The superseded current lock must not become an implicit legacy alias. + "751d5ef5d25406bbae1798667f0e29890d4aad933d323c12912c7d45d8809bb9", + "0" * 64, + ], +) +def test_unapproved_explicit_lock_refuses_before_worker_probe( + worker_identity, monkeypatch, digest +): + def unexpected_probe(): + raise AssertionError("An unapproved lock reached the worker source probe") + + monkeypatch.setattr(worker_identity, "_worker_source_identity", unexpected_probe) + with pytest.raises(ValueError, match="worker lock is not approved"): + worker_identity._uncached_primary_qrf_worker_semantic_identity( + uv_lock_sha256=digest + ) + + +def test_wheel_without_checkout_uses_approved_lock(worker_identity, monkeypatch): + root = Path(__file__).resolve().parents[3] + expected = hashlib.sha256((root / "uv.lock").read_bytes()).hexdigest() + monkeypatch.setattr(worker_identity, "_repository_root", lambda: None) + assert worker_identity._approved_uv_lock_sha256() == expected + + +def test_explicit_legacy_campaign_remains_accepted_before_probe( + worker_identity, monkeypatch +): + class StopBeforeProbeError(Exception): + pass + + def stop_before_probe(): + raise StopBeforeProbeError + + monkeypatch.setattr(worker_identity, "_worker_source_identity", stop_before_probe) + with pytest.raises(StopBeforeProbeError): + worker_identity._uncached_primary_qrf_worker_semantic_identity( + uv_lock_sha256=worker_identity.LEGACY_CAMPAIGN_UV_LOCK_SHA256 + ) diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index f730d2d10..7c2fa104b 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -160,6 +160,31 @@ def test_prepared_column_path_materializes_filtered_values(): ] +@pytest.mark.parametrize( + "left,right,expected", + [ + ([True, False, True], [True, False, False], [2.0, 0.0, 1.0]), + ([True, False, True], [2, 3, 4], [3.0, 3.0, 5.0]), + ([2, 3, 4], [True, False, True], [3.0, 3.0, 5.0]), + ([1, 2, 3], [4, 5, 6], [5.0, 7.0, 9.0]), + ([0.25, -0.5, 2.0], [0.5, 1.5, -1.0], [0.75, 1.0, 1.0]), + ], +) +def test_shared_expression_uses_arithmetic_for_boolean_terms(left, right, expected): + adapter = StubAdapter() + adapter.tables["person"]["left"] = np.asarray(left) + adapter.tables["person"]["right"] = np.asarray(right) + registry = _resolution_registry("sum") + result = materialize_target_bindings( + adapter, + registry, + {"sum": {"bindings": {"policyengine": {"value_expression": "left + right"}}}}, + period=2025, + ) + assert result.skipped == () + np.testing.assert_array_equal(adapter.tables["person"]["sum_measure"], expected) + + @pytest.mark.parametrize("fact_period", [2024, 2025]) @pytest.mark.parametrize("require_matching_fact_period", [None, False, True]) def test_existing_measure_respects_fact_guard_at_default_measurement_period( diff --git a/packages/microcosm-build/tests/test_uk_atomic_area_support.py b/packages/microcosm-build/tests/test_uk_atomic_area_support.py new file mode 100644 index 000000000..b25887a87 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_atomic_area_support.py @@ -0,0 +1,482 @@ +"""Invented array/identity controls only; no publisher files or country engine.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import atomic_geography as geo +from microcosm.build.uk_runtime.atomic_area_support import ( + IDENTITY_COLUMN, + SOURCES, + SYSTEMS, + assemble_uk_atomic_area_support, + uk_atomic_assignment_definition, +) +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE + + +def parts(system): + index = SYSTEMS.index(system) + regions = ( + [ + r + for r in FRS_REGION_TO_REGION_CODE + if r not in {"SCOTLAND", "NORTHERN_IRELAND"} + ] + if index == 0 + else ["SCOTLAND"] + if index == 1 + else ["NORTHERN_IRELAND"] + ) + rows = [] + region_order = list(FRS_REGION_TO_REGION_CODE) + for region in regions: + prefix = FRS_REGION_TO_REGION_CODE[region][0] + for offset in range(3): + serial = region_order.index(region) * 3 + offset + 1 + area = prefix + ("20" if index == 2 else "00") + f"{serial:06d}" + rows.append( + { + "oa_code": area, + "population": [2.0, 6.0, 1000.0][offset], + "households": [0.0, 30.0, 10.0][offset], + "constituency_code": prefix + + "14" + + f"{serial - offset + (offset == 2):06d}", + "region_code": FRS_REGION_TO_REGION_CODE[region], + "lsoa_code": area + if index == 2 + else prefix + "01" + f"{serial:06d}", + "msoa_code": prefix + "02" + f"{serial:06d}", + "local_authority_code": prefix + "06" + f"{serial:06d}", + "ward_code": prefix + "05" + f"{serial:06d}", + "itl3_code": "TL" + + "CDEFGHIJKLMN"[region_order.index(region)] + + "01", + } + ) + arrays = {column: np.asarray([r[column] for r in rows]) for column in rows[0]} + descriptions = {} + for column in arrays: + if column in {"population", "households"}: + descriptions[column] = { + "kind": "weight", + "source": "invented-" + system + "-" + column, + "basis": "invented persons" + if column == "population" + else "invented occupied households", + } + else: + descriptions[column] = { + "kind": "code", + "source": "invented-" + system + "-" + column, + "vintage": "2022_census" if index == 1 else "2021_census", + "relation": "exact" + if column in {"oa_code", "lsoa_code", "msoa_code", "region_code"} + else "best_fit", + } + descriptions["constituency_code"]["vintage"] = "2024_pcon" + if index == 2: + descriptions["constituency_code"]["relation"] = "official_tabulation" + return arrays, descriptions + + +def payloads(): + return { + system: assemble_uk_atomic_area_support( + system=system, arrays=arrays, column_metadata=metadata + ) + for system in SYSTEMS + for arrays, metadata in [parts(system)] + } + + +def key(source_id=101, path=()): + return household_draw_key( + source="invented-frs", + source_vintage="invented-2024-25", + source_household_id=source_id, + clone_path=path, + ) + + +def complete(households, payload): + spec = uk_atomic_assignment_definition(payload, seed=43) + supports = { + system: geo.decode_atomic_support(data) for system, data in payload.items() + } + result = pd.concat( + [households, geo.assign_atomic(households, spec, supports)], axis=1 + ) + result = pd.concat([result, geo.derive_geography(result, spec, supports)], axis=1) + return result, spec, supports + + +def test_normalization_preserves_all_rows_counts_mappings_and_source_labels(): + for system in SYSTEMS: + arrays, metadata = parts(system) + original = {k: v.copy() for k, v in arrays.items()} + data = assemble_uk_atomic_area_support( + system=system, arrays=arrays, column_metadata=metadata + ) + support = geo.decode_atomic_support(data) + order = np.argsort(arrays["oa_code"], kind="stable") + for column in arrays: + normalized = "area" if column == "oa_code" else column + np.testing.assert_array_equal( + support.arrays[normalized], arrays[column][order] + ) + assert dict(support.metadata["columns"][normalized]) == metadata[column] + np.testing.assert_array_equal(arrays[column], original[column]) + assert support.arrays["population"].dtype == np.dtype("int64") + assert support.arrays["households"].dtype == np.dtype("int64") + assert support.arrays["area"].flags.writeable is False + assert support.sha256 == geo.decode_atomic_support(data).sha256 + + +def test_normalized_bytes_are_stable_under_input_row_order_and_integer_count_storage(): + arrays, metadata = parts(SYSTEMS[0]) + expected = assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + reversed_arrays = {k: v[::-1].copy() for k, v in arrays.items()} + for column in ("population", "households"): + reversed_arrays[column] = reversed_arrays[column].astype(np.int64) + assert ( + assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=reversed_arrays, column_metadata=metadata + ) + == expected + ) + + +@pytest.mark.parametrize( + "defect", + [ + "nonintegral", + "float32", + "bool", + "negative", + "nan", + "overflow_total", + "zero_population", + "zero_region_households", + "duplicate_area", + "length", + "missing_column", + "object_code", + "foreign_nation", + "cross_ew_nation", + "blank_code", + "missing_region", + "bad_atomic_vintage", + "missing_source", + ], +) +def test_converter_refuses_bad_counts_codes_coverage_and_metadata(defect): + arrays, metadata = parts(SYSTEMS[0]) + if defect == "nonintegral": + arrays["households"][0] = 0.5 + elif defect == "float32": + arrays["households"] = arrays["households"].astype(np.float32) + elif defect == "bool": + arrays["households"] = arrays["households"] > 0 + elif defect == "negative": + arrays["households"][0] = -1 + elif defect == "nan": + arrays["population"][0] = np.nan + elif defect == "overflow_total": + arrays["population"][0] = float(2**53) + elif defect == "zero_population": + arrays["population"][0] = 0 + elif defect == "zero_region_households": + arrays["households"][:3] = 0 + elif defect == "duplicate_area": + arrays["oa_code"][1] = arrays["oa_code"][0] + elif defect == "length": + arrays["msoa_code"] = arrays["msoa_code"][:-1] + elif defect == "missing_column": + del arrays["population"] + elif defect == "object_code": + arrays["oa_code"] = arrays["oa_code"].astype(object) + elif defect == "foreign_nation": + arrays["ward_code"][0] = "S05000001" + elif defect == "cross_ew_nation": + arrays["ward_code"][0] = "W05000001" + elif defect == "blank_code": + arrays["ward_code"][0] = "" + elif defect == "missing_region": + arrays = {k: v[3:].copy() for k, v in arrays.items()} + elif defect == "bad_atomic_vintage": + metadata["oa_code"]["vintage"] = "2011_census" + elif defect == "missing_source": + metadata["population"]["source"] = "" + with pytest.raises(ValueError, match="UK atomic support:"): + assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + + +@pytest.mark.parametrize("defect", ["inferred_ni_constituency", "ni_lsoa_not_dz"]) +def test_ni_source_convention_refusals(defect): + arrays, metadata = parts(SYSTEMS[2]) + if defect == "inferred_ni_constituency": + metadata["constituency_code"]["relation"] = "inferred_modal" + else: + arrays["lsoa_code"][0] = "N01000001" + with pytest.raises(ValueError, match="UK atomic support: NI"): + assemble_uk_atomic_area_support( + system=SYSTEMS[2], arrays=arrays, column_metadata=metadata + ) + + +def test_three_system_assignment_keeps_observed_region_and_native_alias_roles(): + households = pd.DataFrame( + { + "household_id": np.arange(1, 5, dtype=np.int64), + "region": pd.array( + ["LONDON", "WALES", "SCOTLAND", "NORTHERN_IRELAND"], dtype="string" + ), + IDENTITY_COLUMN: pd.array( + [key(i) for i in (101, 102, 103, 104)], dtype="string" + ), + } + ) + result, spec, supports = complete(households, payloads()) + pd.testing.assert_series_equal(result["region"], households["region"]) + assert result["atomic_area_system"].tolist() == [ + SYSTEMS[0], + SYSTEMS[0], + SYSTEMS[1], + SYSTEMS[2], + ] + assert result["atomic_area_basis"].tolist() == ["assigned"] * 4 + assert result.loc[:2, "output_area_code"].equals(result.loc[:2, "atomic_area_code"]) + assert pd.isna(result.loc[3, "output_area_code"]) + assert result.loc[2, "data_zone_code"] == result.loc[2, "lsoa_code"] + assert result.loc[2, "intermediate_zone_code"] == result.loc[2, "msoa_code"] + assert result.loc[3, "data_zone_code"] == result.loc[3, "atomic_area_code"] + assert result.loc[3, "super_data_zone_code"] == result.loc[3, "msoa_code"] + assert result.loc[3, "district_electoral_area_code"] == result.loc[3, "ward_code"] + assert pd.isna(result.loc[0, "super_data_zone_code"]) + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + assert {s["source"] for s in spec["systems"]} == set(SOURCES.values()) + + +def test_source_metadata_changes_are_identified_and_declaration_refuses_forged_derived_field(): + payload = payloads() + original = payload[SYSTEMS[0]] + arrays, metadata = parts(SYSTEMS[0]) + metadata["population"]["source"] += "-revision-2" + changed = assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + assert changed != original + support = geo.decode_atomic_support(original) + forged_arrays = {k: v.copy() for k, v in support.arrays.items()} + forged_arrays["itl1_code"][0] = "TLN" + forged = geo.encode_atomic_support( + {k: dict(v) if k == "columns" else v for k, v in support.metadata.items()}, + forged_arrays, + ) + payload[SYSTEMS[0]] = forged + with pytest.raises(ValueError, match="canonical UK adapter output"): + uk_atomic_assignment_definition(payload, seed=43) + + +@pytest.mark.parametrize("defect", ["missing_system", "wrong_system", "bad_seed"]) +def test_declaration_refuses_mismatched_inputs(defect): + payload = payloads() + options = {"seed": 43} + if defect == "missing_system": + del payload[SYSTEMS[2]] + elif defect == "wrong_system": + payload[SYSTEMS[2]] = payload[SYSTEMS[1]] + elif defect == "bad_seed": + options["seed"] = True + with pytest.raises(ValueError): + uk_atomic_assignment_definition(payload, **options) + + +def test_declaration_cannot_substitute_an_income_or_counter_identity_column(): + with pytest.raises(TypeError, match="identity_column"): + uk_atomic_assignment_definition( + payloads(), seed=43, identity_column="household_weight" + ) + + +def test_distinct_post_clone_keys_assign_then_remain_stable_under_order_subset_and_growth(): + paths = [ + (("spi_support_channel", 0), ("cgt_incidence_clone", 0)), + (("spi_support_channel", 1), ("cgt_incidence_clone", 0)), + (("spi_support_channel", 1), ("cgt_incidence_clone", 1)), + ( + ("spi_support_channel", 1), + ("cgt_incidence_clone", 1), + ("cgt_band_donors", 1), + ), + ] + original = pd.DataFrame( + { + "household_id": np.asarray([11, 22, 33, 44], dtype=np.int64), + "region": pd.array(["LONDON"] * 4, dtype="string"), + IDENTITY_COLUMN: pd.array( + [key(101, path) for path in paths], dtype="string" + ), + } + ) + assert original[IDENTITY_COLUMN].nunique() == 4 + payload = payloads() + expected, spec, supports = complete(original, payload) + changed = original.iloc[[3, 1, 0]].copy() + changed["household_weight"] = [0.0, 2.0, 8.0] + extra = pd.DataFrame( + { + "household_id": [55], + "region": pd.array(["LONDON"], dtype="string"), + IDENTITY_COLUMN: pd.array( + [key(202, (("geographic_support", 1),))], dtype="string" + ), + "household_weight": [1.0], + }, + index=[8], + ) + actual, _, _ = complete(pd.concat([changed, extra]), payload) + for column in expected: + if column != "region": + pd.testing.assert_series_equal( + actual.loc[changed.index, column], expected.loc[changed.index, column] + ) + assert geo.validate_geography(actual, spec, supports)["outcome"] == "pass" + assigned_columns = ["atomic_area_code", "oa_code", "constituency_code"] + pruned = expected.iloc[[3, 1]].copy() + assert geo.validate_geography(pruned, spec, supports)["outcome"] == "pass" + pd.testing.assert_frame_equal( + pruned[assigned_columns], expected.loc[pruned.index, assigned_columns] + ) + with pytest.raises(ValueError, match="assignment output already exists"): + geo.assign_atomic(expected, spec, supports) + + +def test_country_declaration_retains_the_two_stage_sampling_law(): + households = pd.DataFrame( + { + "household_id": np.arange(1, 2001, dtype=np.int64), + "region": pd.array(["LONDON"] * 2000, dtype="string"), + IDENTITY_COLUMN: pd.array([key(i) for i in range(1, 2001)], dtype="string"), + } + ) + result, spec, _ = complete(households, payloads()) + assert all( + s["stages"] + == [ + {"level": "constituency_code", "weight": "households"}, + {"level": "area", "weight": "population"}, + ] + for s in spec["systems"] + ) + arrays, _ = parts(SYSTEMS[0]) + london = arrays["region_code"] == FRS_REGION_TO_REGION_CODE["LONDON"] + areas = arrays["oa_code"][london] + shares = result["atomic_area_code"].value_counts(normalize=True) + # Constituent household mass 30:10, then within-first-area population 2:6; + # the third area's population 1000 must not dominate the first-stage draw. + for area, expected in zip(areas, (0.1875, 0.5625, 0.25), strict=True): + assert abs(shares.get(area, 0) - expected) < 0.04 + + +def test_exact_source_ids_and_ordered_branches_are_preserved(): + first = key(2**53 + 1, (("spi_support_channel", 1), ("cgt_incidence_clone", 0))) + second = key(2**53 + 2, (("spi_support_channel", 1), ("cgt_incidence_clone", 0))) + assert first != second + assert str(2**53 + 1) in first + assert first == key( + np.int64(2**53 + 1), + (("spi_support_channel", np.int64(1)), ("cgt_incidence_clone", 0)), + ) + grown = [key(101, (("geographic_support", i),)) for i in range(6)] + assert grown[:3] == [key(101, (("geographic_support", i),)) for i in range(3)] + + +@pytest.mark.parametrize( + "defect", + [ + "float_id", + "bool_id", + "zero_id", + "oversize_id", + "no_vintage", + "mutable_path", + "counter_branch", + "income_branch", + "max_id_branch", + "out_of_order", + "duplicate_branch", + "float_ordinal", + "negative_ordinal", + "oversize_ordinal", + ], +) +def test_identity_refuses_inferred_or_inexact_conventions(defect): + args = { + "source": "invented-frs", + "source_vintage": "invented-2024-25", + "source_household_id": 101, + "clone_path": (), + } + if defect == "float_id": + args["source_household_id"] = 101.0 + elif defect == "bool_id": + args["source_household_id"] = True + elif defect == "zero_id": + args["source_household_id"] = 0 + elif defect == "oversize_id": + args["source_household_id"] = 2**63 + elif defect == "no_vintage": + args["source_vintage"] = "" + elif defect == "mutable_path": + args["clone_path"] = [] + elif defect in {"counter_branch", "income_branch", "max_id_branch"}: + args["clone_path"] = ( + ( + { + "counter_branch": "row_counter", + "income_branch": "income", + "max_id_branch": "sample_max_id", + }[defect], + 1, + ), + ) + elif defect == "out_of_order": + args["clone_path"] = (("cgt_incidence_clone", 1), ("spi_support_channel", 1)) + elif defect == "duplicate_branch": + args["clone_path"] = (("spi_support_channel", 0), ("spi_support_channel", 1)) + else: + ordinal = { + "float_ordinal": 1.5, + "negative_ordinal": -1, + "oversize_ordinal": 2**32, + }[defect] + args["clone_path"] = (("spi_support_channel", ordinal),) + with pytest.raises(ValueError, match="UK geography identity:"): + household_draw_key(**args) + + +def test_duplicate_or_unknown_household_identity_is_refused_by_shared_assignment(): + payload = payloads() + spec = uk_atomic_assignment_definition(payload, seed=43) + supports = {s: geo.decode_atomic_support(p) for s, p in payload.items()} + households = pd.DataFrame( + { + "household_id": [11, 22], + "region": ["LONDON", "LONDON"], + IDENTITY_COLUMN: [key(101), key(101)], + } + ) + with pytest.raises(ValueError, match="duplicate draw identity"): + geo.assign_atomic(households, spec, supports) + households.loc[1, IDENTITY_COLUMN] = key(102) + households.loc[1, "region"] = "UNKNOWN" + with pytest.raises(ValueError, match="exactly one"): + geo.assign_atomic(households, spec, supports) diff --git a/packages/microcosm-build/tests/test_uk_atomic_household_lineage.py b/packages/microcosm-build/tests/test_uk_atomic_household_lineage.py new file mode 100644 index 000000000..ace6106f1 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_atomic_household_lineage.py @@ -0,0 +1,588 @@ +"""Supplied lineage and real invented Graph/Frame controls; no FRS or engine.""" + +import json +from dataclasses import FrozenInstanceError, replace + +import numpy as np +import pandas as pd +import pytest +from test_uk_atomic_area_support import payloads + +from microcosm.build import atomic_geography +from microcosm.build.graph_atomic_geography import ( + atomic_geography_nodes, + register_atomic_geography_kernels, +) +from microcosm.build.uk_runtime.atomic_area_support import ( + IDENTITY_COLUMN, + SOURCES, + uk_atomic_assignment_definition, +) +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.atomic_household_lineage import ( + HouseholdExpansion, + HouseholdSelection, + project_atomic_household_keys, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) + + +def roots(ids=(1, 2)): + return pd.DataFrame( + { + "household_id": np.asarray(ids, dtype=np.int64), + "source": pd.array(["invented-frs"] * len(ids), dtype="string"), + "source_vintage": pd.array(["invented-2024-25"] * len(ids), dtype="string"), + "source_household_id": np.asarray(ids, dtype=np.int64), + } + ) + + +def expansion(branch, before, pairs, ordinals=None): + return HouseholdExpansion( + branch=branch, + before_ids=tuple(before), + after_ids=(*before, *(c for c, _ in pairs)), + parent_pairs=tuple(pairs), + child_ordinals=tuple((c, 1) for c, _ in pairs) + if ordinals is None + else ordinals, + ) + + +def chain(): + a = expansion("spi_support_channel", (1, 2), ((11, 1), (12, 2))) + b = expansion("cgt_incidence_clone", a.after_ids, ((21, 1), (31, 11))) + c = expansion("cgt_band_donors", b.after_ids, ((41, 31),)) + s = HouseholdSelection(c.after_ids, (1, 11, 21, 31, 41)) + d = expansion("geographic_support", s.after_ids, ((101, 1), (141, 41))) + return a, b, c, s, d + + +def project(source=None, steps=None, ids=None): + source = roots() if source is None else source + steps = chain() if steps is None else steps + ids = steps[-1].after_ids if ids is None else ids + return project_atomic_household_keys( + source, steps=steps, final_ids=np.asarray(ids, dtype=np.int64) + ) + + +def by_id(table): + return table.set_index("household_id")[IDENTITY_COLUMN].sort_index() + + +def test_explicit_full_chain_selection_and_clone_keys(): + result = project() + assert tuple(result.household_id) == chain()[-1].after_ids + assert list(result) == ["household_id", IDENTITY_COLUMN] + assert result.household_id.dtype == np.dtype("int64") + assert result[IDENTITY_COLUMN].dtype == pd.StringDtype(storage="python") + expected_paths = { + 1: (), + 11: (("spi_support_channel", 1),), + 21: (("cgt_incidence_clone", 1),), + 31: (("spi_support_channel", 1), ("cgt_incidence_clone", 1)), + 41: ( + ("spi_support_channel", 1), + ("cgt_incidence_clone", 1), + ("cgt_band_donors", 1), + ), + 101: (("geographic_support", 1),), + 141: ( + ("spi_support_channel", 1), + ("cgt_incidence_clone", 1), + ("cgt_band_donors", 1), + ("geographic_support", 1), + ), + } + actual = by_id(result) + for target, path in expected_paths.items(): + assert actual[target] == household_draw_key( + source="invented-frs", + source_vintage="invented-2024-25", + source_household_id=1, + clone_path=path, + ) + assert actual.is_unique + + +def test_axes_and_pair_order_do_not_supply_identity(): + steps = tuple( + replace( + x, + before_ids=x.before_ids[::-1], + after_ids=x.after_ids[::-1], + **( + { + "parent_pairs": x.parent_pairs[::-1], + "child_ordinals": x.child_ordinals[::-1], + } + if isinstance(x, HouseholdExpansion) + else {} + ), + ) + for x in chain() + ) + result = project(roots().iloc[::-1], steps, steps[-1].after_ids) + pd.testing.assert_series_equal(by_id(result), by_id(project())) + + +def test_exact_large_ids_and_growth_preserve_existing_keys(): + big = 2**53 + 7 + first = expansion("geographic_support", (big,), ((big + 2, big),)) + more = expansion( + "geographic_support", + (big, big + 1), + ((big + 2, big), (big + 3, big + 1)), + ((big + 2, 1), (big + 3, 9)), + ) + small = project(roots((big,)), (first,)) + large = project(roots((big, big + 1)), (more,)) + pd.testing.assert_series_equal(by_id(small), by_id(large).loc[by_id(small).index]) + assert by_id(large).is_unique + assert str(big) in by_id(small).iloc[0] + + +def test_source_and_explicit_ordinal_change_keys_without_numeric_inference(): + expected = by_id(project()) + source = roots() + source.loc[0, "source_vintage"] = "different-vintage" + changed = by_id(project(source)) + assert (changed != expected).all() + steps = chain() + changed = by_id( + project( + steps=(*steps[:-1], replace(steps[-1], child_ordinals=((101, 2), (141, 8)))) + ) + ) + assert (changed.loc[[101, 141]] != expected.loc[[101, 141]]).all() + pd.testing.assert_series_equal(changed.drop([101, 141]), expected.drop([101, 141])) + + +def test_noop_and_empty_selection_are_explicit_and_detached(): + original = roots() + saved = original.copy(deep=True) + no_steps = project(original, (), (2, 1)) + assert no_steps.household_id.tolist() == [2, 1] + empty = project(original, (HouseholdSelection((1, 2), ()),), ()) + assert empty.empty and empty.household_id.dtype == np.dtype("int64") + no_steps.loc[0, IDENTITY_COLUMN] = "changed descriptive output" + pd.testing.assert_frame_equal(original, saved) + with pytest.raises(FrozenInstanceError): + chain()[0].branch = "changed" + + +@pytest.mark.parametrize( + "defect", + [ + "duplicate_id", + "float_id", + "bool_id", + "zero_id", + "duplicate_root_identity", + "missing_source", + "blank_source", + "extra_column", + "duplicate_column", + "invalid_original", + "empty_roots", + ], +) +def test_roots_refuse_unqualified_or_ambiguous_descriptions(defect): + source = roots() + if defect == "duplicate_id": + source.loc[1, "household_id"] = 1 + elif defect == "float_id": + source["household_id"] = source.household_id.astype(float) + elif defect == "bool_id": + source["household_id"] = [True, False] + elif defect == "zero_id": + source.loc[0, "household_id"] = 0 + elif defect == "duplicate_root_identity": + source.loc[1, "source_household_id"] = 1 + elif defect == "missing_source": + source.loc[0, "source"] = pd.NA + elif defect == "blank_source": + source.loc[0, "source_vintage"] = " " + elif defect == "extra_column": + source["guess"] = 1 + elif defect == "duplicate_column": + source.columns = ["household_id", "source", "source", "source_household_id"] + elif defect == "invalid_original": + source["source_household_id"] = [1.0, 2.0] + elif defect == "empty_roots": + source = source.iloc[:0] + with pytest.raises((TypeError, ValueError)): + project(source) + + +@pytest.mark.parametrize( + "defect", + [ + "unknown_branch", + "reordered_branch", + "repeated_branch", + "mutable_steps", + "mutable_axis", + "duplicate_before", + "duplicate_after", + "unexplained_before", + "deleted_incumbent", + "unknown_parent", + "null_parent", + "same_step_parent", + "missing_parent", + "duplicate_parent_target", + "unexplained_child", + "missing_ordinal", + "extra_ordinal", + "duplicate_ordinal", + "negative_ordinal", + "bool_ordinal", + "float_child", + "overflow_child", + "selection_arrival", + "implicit_selection", + "final_extra", + "final_duplicate", + ], +) +def test_invalid_steps_and_final_axis_refuse(defect): + steps = list(chain()) + first = steps[0] + final = steps[-1].after_ids + if defect == "unknown_branch": + steps[0] = replace(first, branch="income_guess") + elif defect == "reordered_branch": + steps[0] = replace(first, branch="geographic_support") + elif defect == "repeated_branch": + steps[1] = replace(steps[1], branch=first.branch) + elif defect == "mutable_steps": + with pytest.raises((TypeError, ValueError)): + project(steps=steps) + return + elif defect == "mutable_axis": + steps[0] = replace(first, before_ids=[1, 2]) + elif defect == "duplicate_before": + steps[0] = replace(first, before_ids=(1, 1, 2)) + elif defect == "duplicate_after": + steps[0] = replace(first, after_ids=(*first.after_ids, 11)) + elif defect == "unexplained_before": + steps[0] = replace(first, before_ids=(1, 3)) + elif defect == "deleted_incumbent": + steps[0] = replace(first, after_ids=(1, 11, 12)) + elif defect == "unknown_parent": + steps[0] = replace(first, parent_pairs=((11, 3), (12, 2))) + elif defect == "null_parent": + steps[0] = replace(first, parent_pairs=((11, None), (12, 2))) + elif defect == "same_step_parent": + steps[0] = replace(first, parent_pairs=((11, 1), (12, 11))) + elif defect == "missing_parent": + steps[0] = replace(first, parent_pairs=((11, 1),)) + elif defect == "duplicate_parent_target": + steps[0] = replace(first, parent_pairs=((11, 1), (11, 2))) + elif defect == "unexplained_child": + steps[0] = replace(first, after_ids=(*first.after_ids, 13)) + elif defect == "missing_ordinal": + steps[0] = replace(first, child_ordinals=((11, 1),)) + elif defect == "extra_ordinal": + steps[0] = replace(first, child_ordinals=(*first.child_ordinals, (99, 1))) + elif defect == "duplicate_ordinal": + steps[0] = replace(first, parent_pairs=((11, 1), (12, 1))) + elif defect == "negative_ordinal": + steps[0] = replace(first, child_ordinals=((11, -1), (12, 1))) + elif defect == "bool_ordinal": + steps[0] = replace(first, child_ordinals=((11, True), (12, 1))) + elif defect == "float_child": + steps[0] = replace(first, parent_pairs=((11.0, 1), (12, 2))) + elif defect == "overflow_child": + steps[0] = replace( + first, + after_ids=(1, 2, 11, 2**63), + parent_pairs=((11, 1), (2**63, 2)), + child_ordinals=((11, 1), (2**63, 1)), + ) + elif defect == "selection_arrival": + steps[3] = replace(steps[3], after_ids=(*steps[3].after_ids, 99)) + elif defect == "implicit_selection": + del steps[3] + elif defect == "final_extra": + final = (*final, 99) + elif defect == "final_duplicate": + final = (*final, final[0]) + with pytest.raises((TypeError, ValueError)): + project(steps=tuple(steps), ids=final) + + +class _FrameKernel(KernelBase): + ref = "invented.uk.lineage.frame@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def __init__(self, frame): + self.frame = frame + + def run(self, context): + return KernelResult(frame=self.frame) + + +class _ExpandKernel(KernelBase): + ref = "invented.uk.lineage.expand@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND + ) + + def run(self, context): + return KernelResult( + expand={ + e: pd.Series( + [1, 2, 3], + index=pd.Index([11, 12, 13], name=e + "_id"), + dtype="int64", + ) + for e in ("person", "benunit", "household") + }, + weights=Weights(np.full(6, 0.5), WeightKind.IMPORTANCE), + ) + + +class _SelectKernel(KernelBase): + ref = "invented.uk.lineage.select@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.FILTER + ) + + def run(self, context): + ids = context.tables["person"].person_id + return KernelResult( + keep=pd.Series( + ids.isin([1, 3, 11, 13]).to_numpy(), + index=pd.Index(ids, name="person_id"), + ) + ) + + +def test_actual_expand_filter_receipts_feed_shared_atomic_graph_and_replay(tmp_path): + ids = np.arange(1, 4, dtype=np.int64) + person = pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": [21, 33, 44], + } + ) + benunit = pd.DataFrame({"benunit_id": ids, "family_marker": [4, 5, 6]}) + household = pd.DataFrame( + { + "household_id": ids, + "region": pd.array( + ["LONDON", "SCOTLAND", "NORTHERN_IRELAND"], dtype="string" + ), + } + ) + frame = Frame( + {"person": person, "benunit": benunit, "household": household}, + EntitySchema(group_entities=("benunit", "household")), + {"household": Weights(np.ones(3), WeightKind.DESIGN)}, + pd.Series(["invented"] * 3, name="stratum"), + metadata={"note": "invented UK lineage"}, + ) + columns = ( + Owned("person", "age", "int64"), + Owned("benunit", "family_marker", "int64"), + Owned("household", "region", "string"), + ) + graph = compile_graph( + Graph( + "uk", + (SourceRef("invented", "raw-bytes-v1"),), + ( + Node( + "root", + _FrameKernel.ref, + structural=StructuralDelta.CREATE, + sources=("invented",), + outputs=columns, + ), + Node( + "expand", + _ExpandKernel.ref, + base="root", + structural=StructuralDelta.EXPAND, + mass="conserve", + params={ + "expand_cells": (), + "expand_weight_entity": "household", + "expand_weight_kind": "importance", + }, + ), + Node( + "select", + _SelectKernel.ref, + base="expand", + structural=StructuralDelta.FILTER, + mass="free", + inputs=(Slice("person", ("age",)),), + ), + ), + ) + ) + registry = KernelRegistry() + for k in (_FrameKernel(frame), _ExpandKernel(), _SelectKernel()): + registry.register(k) + source = tmp_path / "invented.txt" + source.write_text("invented rows; no external survey") + store = ContentStore(tmp_path / "lineage-store") + cold = run_graph(graph, sources={"invented": source}, store=store, kernels=registry) + replay = run_graph( + graph, + sources={"invented": source}, + store=store, + kernels=registry, + resume="require", + ) + assert all(n.hit for n in replay.nodes.values()) + pair_rows = cold.nodes["expand"].receipt["expand"]["household"] + assert pair_rows == replay.nodes["expand"].receipt["expand"]["household"] + expanded = cold.population("expand") + selected = cold.population("select") + steps = ( + HouseholdExpansion( + "spi_support_channel", + tuple(ids), + tuple(expanded.table("household").household_id), + tuple(tuple(p) for p in pair_rows), + ((11, 1), (12, 1), (13, 1)), + ), + HouseholdSelection( + tuple(expanded.table("household").household_id), + tuple(selected.table("household").household_id), + ), + ) + keys = project( + roots((1, 2, 3)), steps, tuple(selected.table("household").household_id) + ) + selected_tables = {e: selected.table(e).copy(deep=True) for e in selected.entities} + selected_tables["household"][IDENTITY_COLUMN] = keys[IDENTITY_COLUMN].array.copy() + keyed = Frame( + selected_tables, + selected.schema, + {"household": selected.weights_for("household")}, + selected.strata, + metadata=selected.metadata, + mass_log=selected.mass_log, + ) + support_payloads = payloads() + definition = uk_atomic_assignment_definition(support_payloads, seed=43) + all_columns = (*columns, Owned("household", IDENTITY_COLUMN, "string")) + atomic_nodes = atomic_geography_nodes( + definition, all_columns, base="keyed", emit_validation_artifact=True + ) + assignment_graph = compile_graph( + Graph( + "uk", + ( + SourceRef("invented", "raw-bytes-v1"), + *(SourceRef(SOURCES[s], "raw-bytes-v1") for s in support_payloads), + ), + ( + Node( + "keyed", + _FrameKernel.ref, + structural=StructuralDelta.CREATE, + sources=("invented",), + outputs=all_columns, + ), + *atomic_nodes, + ), + ) + ) + source_paths = {"invented": source} + for system, payload in support_payloads.items(): + path = tmp_path / (system + ".npz") + path.write_bytes(payload) + source_paths[SOURCES[system]] = path + atomic_registry = KernelRegistry() + atomic_registry.register(_FrameKernel(keyed)) + register_atomic_geography_kernels(atomic_registry) + atomic_store = ContentStore(tmp_path / "atomic-store") + assigned = run_graph( + assignment_graph, + sources=source_paths, + store=atomic_store, + kernels=atomic_registry, + ) + required = run_graph( + assignment_graph, + sources=source_paths, + store=atomic_store, + kernels=atomic_registry, + resume="require", + ) + assert len(assigned.nodes) == 7 and all(n.hit for n in required.nodes.values()) + actual = assigned.population("keyed") + for entity in keyed.entities: + pd.testing.assert_frame_equal( + actual.table(entity)[list(keyed.table(entity))], + keyed.table(entity), + check_index_type=False, + ) + np.testing.assert_array_equal( + actual.weights_for("household").values, keyed.weights_for("household").values + ) + pd.testing.assert_series_equal(actual.strata, keyed.strata, check_index_type=False) + assert actual.metadata == keyed.metadata and actual.mass_log == keyed.mass_log + assert assigned.nodes["geography.gate"].receipt["outcome"] == "pass" + supports = { + k: atomic_geography.decode_atomic_support(v) + for k, v in support_payloads.items() + } + reordered = keyed.table("household").iloc[::-1] + expected = atomic_geography.assign_atomic(reordered, definition, supports) + for c in expected: + pd.testing.assert_series_equal( + expected[c].astype(pd.StringDtype(storage="python")).reset_index(drop=True), + actual.table("household")[c] + .iloc[::-1] + .astype(pd.StringDtype(storage="python")) + .reset_index(drop=True), + check_names=False, + ) + + (tmp_path / "acceptance.json").write_text( + json.dumps( + { + "scope": "invented actual shared CREATE/EXPAND/FILTER and atomic graph; pure descriptive lineage helper", + "lineage_nodes": len(cold.nodes), + "lineage_required_hits": sum(n.hit for n in replay.nodes.values()), + "atomic_nodes": len(assigned.nodes), + "atomic_required_hits": sum(n.hit for n in required.nodes.values()), + "root_households": frame.n("household"), + "expanded_households": expanded.n("household"), + "selected_households": selected.n("household"), + "source_admission": False, + "native_inputs": False, + }, + indent=2, + ) + + "\n" + ) diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py index e8e59ae96..fabd98a1f 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -57,6 +57,19 @@ def _signing_key(monkeypatch): monkeypatch.setenv("MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY", SIGNING_KEY) +@pytest.fixture +def invented_code_pin(monkeypatch): + """Supply fixture provenance without claiming the wheel venv is a checkout.""" + pin = "1" * 40 + + def fixture_pin(repository): + assert repository == calibration_run._REPOSITORY + return pin + + monkeypatch.setattr(calibration_run, "git_code_pin", fixture_pin) + return pin + + def _frame(): ids = np.arange(4, dtype="int64") return uk_national_frame( @@ -235,7 +248,9 @@ def test_import_hygiene_does_not_load_national_build_in_fresh_subprocess(): assert " ".join(("from", legacy_module, "import")) not in source -def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: Path): +def test_run_uk_calibration_writes_cross_pinned_outputs( + monkeypatch, tmp_path: Path, invented_code_pin +): pytest.importorskip("tables") # pandas HDF backend monkeypatch.setattr( calibration_run, @@ -371,10 +386,12 @@ def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: P == signature ) assert result.logbook_spool.exists() + row = json.loads(result.logbook_spool.read_text()) + assert row["code_pin"] == invented_code_pin def test_readback_failure_updates_build_record_and_failed_attempt( - monkeypatch, tmp_path: Path + monkeypatch, tmp_path: Path, invented_code_pin ): pytest.importorskip("tables") monkeypatch.setattr( @@ -441,7 +458,7 @@ def fail_readback() -> None: def test_run_uk_calibration_requires_the_band_edge_register( - tmp_path: Path, + tmp_path: Path, invented_code_pin ): # Required, never defaulted: an empty receipt is a claim that nothing was # pruned, not permission to skip the reconciliation, so the seam takes no @@ -470,7 +487,7 @@ def test_run_uk_calibration_requires_the_band_edge_register( def test_run_uk_calibration_reconciles_an_empty_receipt_as_no_prunes( - tmp_path: Path, + tmp_path: Path, invented_code_pin ): # A pruned register handed in with an empty receipt must refuse: with # nothing declared excluded, the two rosters have to be name-identical. @@ -500,7 +517,9 @@ def test_run_uk_calibration_reconciles_an_empty_receipt_as_no_prunes( assert not paths.build_record_json.exists() -def test_run_uk_calibration_refuses_incoherent_band_edge_register(tmp_path: Path): +def test_run_uk_calibration_refuses_incoherent_band_edge_register( + tmp_path: Path, invented_code_pin +): paths = _paths(tmp_path) edge_registry = TargetRegistry( [ @@ -540,7 +559,7 @@ def test_run_uk_calibration_refuses_incoherent_band_edge_register(tmp_path: Path def test_run_uk_calibration_records_band_edge_register_sha256( - monkeypatch, tmp_path: Path + monkeypatch, tmp_path: Path, invented_code_pin ): pytest.importorskip("tables") # pandas HDF backend monkeypatch.setattr( @@ -597,7 +616,9 @@ def test_run_uk_calibration_records_band_edge_register_sha256( ) -def test_run_uk_calibration_refuses_input_sha_before_outputs(tmp_path: Path): +def test_run_uk_calibration_refuses_input_sha_before_outputs( + tmp_path: Path, invented_code_pin +): pytest.importorskip("tables") # pandas HDF backend input_h5 = tmp_path / "input.h5" write_uk_national_frame(_frame(), input_h5) @@ -633,7 +654,9 @@ def test_run_uk_calibration_refuses_input_sha_before_outputs(tmp_path: Path): assert not paths.diagnostics_json.exists() -def test_run_uk_calibration_refuses_absent_input_sidecar(tmp_path: Path): +def test_run_uk_calibration_refuses_absent_input_sidecar( + tmp_path: Path, invented_code_pin +): pytest.importorskip("tables") # pandas HDF backend input_h5 = tmp_path / "input.h5" write_uk_national_frame(_frame(), input_h5) @@ -683,7 +706,7 @@ def test_run_uk_calibration_refuses_absent_input_sidecar(tmp_path: Path): ], ) def test_run_uk_calibration_refuses_unbound_input_sidecar( - override, message, tmp_path: Path + override, message, tmp_path: Path, invented_code_pin ): pytest.importorskip("tables") # pandas HDF backend frame = _frame() @@ -725,7 +748,9 @@ def test_run_uk_calibration_refuses_unbound_input_sidecar( assert not paths.terminal_gate_json.exists() -def test_seam_never_modifies_data_variables(monkeypatch, tmp_path: Path): +def test_seam_never_modifies_data_variables( + monkeypatch, tmp_path: Path, invented_code_pin +): """The seam's defining invariant: weights move, data never does. Every data column of every entity table in the staged H5 must be @@ -925,7 +950,9 @@ def _load_logbook_tool(): return module -def test_refusal_records_a_failed_attempt_and_stages_nothing(tmp_path: Path): +def test_refusal_records_a_failed_attempt_and_stages_nothing( + tmp_path: Path, invented_code_pin +): pytest.importorskip("tables") # pandas HDF backend input_h5 = tmp_path / "input.h5" write_uk_national_frame(_frame(), input_h5) @@ -975,7 +1002,7 @@ def test_refusal_records_a_failed_attempt_and_stages_nothing(tmp_path: Path): def test_attempt_ids_are_unique_across_reruns_of_one_release( - monkeypatch, tmp_path: Path + monkeypatch, tmp_path: Path, invented_code_pin ): pytest.importorskip("tables") # pandas HDF backend monkeypatch.setattr( @@ -1023,7 +1050,9 @@ def test_attempt_ids_are_unique_across_reruns_of_one_release( assert all(value.startswith("uk-frs-calibration-attempt-") for value in build_ids) -def test_verified_ledger_identity_reaches_the_run_evidence(monkeypatch, tmp_path: Path): +def test_verified_ledger_identity_reaches_the_run_evidence( + monkeypatch, tmp_path: Path, invented_code_pin +): pytest.importorskip("tables") # pandas HDF backend monkeypatch.setattr( calibration_run, @@ -1159,7 +1188,9 @@ def _mixed_epoch_artifact_dir(tmp_path: Path) -> Path: return artifact_dir -def test_mixed_epoch_feed_epochs_reach_the_uk_release_evidence(monkeypatch, tmp_path): +def test_mixed_epoch_feed_epochs_reach_the_uk_release_evidence( + monkeypatch, tmp_path, invented_code_pin +): """A UK run says which Chronicle era resolved its targets. The run's own provenance block used to be assembled field by field from @@ -1251,3 +1282,41 @@ def test_the_uk_block_delegates_rather_than_reassembling_the_shared_one(tmp_path continue assert block[field] == value, field assert block["manifest"]["schema_version"] == shared["schema_version"] + + +def test_run_uk_calibration_refuses_unresolved_code_pin_before_io( + monkeypatch, tmp_path +): + paths = _paths(tmp_path) + probes = [] + + def unavailable_pin(repository): + probes.append(repository) + raise RuntimeError("fixture code pin unavailable") + + def forbidden_io(*args, **kwargs): + pytest.fail("unresolved code identity must refuse before input or attempt I/O") + + monkeypatch.setattr(calibration_run, "git_code_pin", unavailable_pin) + monkeypatch.setattr(calibration_run, "_sha256_file", forbidden_io) + monkeypatch.setattr(calibration_run, "resolve_predecessor", forbidden_io) + monkeypatch.setattr(calibration_run, "_run_uk_calibration_attempt", forbidden_io) + with pytest.raises(RuntimeError, match="^fixture code pin unavailable$"): + run_uk_calibration( + paths=paths, + input_sha256="a" * 64, + ledger_artifact=object(), + register_registry=_registry(), + band_edge_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=1), + doctrine_overrides={}, + measure_resolver=None, + source_pins={}, + run_config_extra={}, + release_id="unresolved-fixture-code-pin", + ) + + assert probes == [calibration_run._REPOSITORY] + assert list(tmp_path.iterdir()) == [] diff --git a/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py b/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py new file mode 100644 index 000000000..9fac189b9 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py @@ -0,0 +1,140 @@ +"""Normal-import contracts for the real UK runtime public package. + +A fresh no-country-resource guard supplies the cold-import check. In a larger +suite, already imported country modules are allowed; these tests never remove, +stub or replace modules to manufacture an isolated import environment. +""" + +import hashlib +import importlib +import json +import sys + +import pytest + +_PACKAGE = "microcosm.build.uk_runtime" +_UNRELATED_MODULES = ( + f"{_PACKAGE}.calibration_run", + f"{_PACKAGE}.national_calibration", + f"{_PACKAGE}.firm_generation", +) +# SHA256 of compact JSON for all 458 ordered entries (457 unique) from +# ec3b306f3f9742ff..., recomputed unchanged against the merge of origin/main on +# 17 September 2026: the UK runtime's exports moved on neither side. +# This includes the existing duplicate ladder_vs_chronicle_household_dispersion. +_ORDERED_ALL_SHA256 = "b1b27eaa0204f49402b2883b80c996ea460cbec388ebe64644a688aca430552d" + + +def _unrelated_modules() -> set[str]: + return { + name + for name in sys.modules + if any( + name == prefix or name.startswith(prefix + ".") + for prefix in _UNRELATED_MODULES + ) + } + + +def test_normal_package_and_pure_helper_imports_do_not_load_unrelated_modules(): + before = _unrelated_modules() + package = importlib.import_module(_PACKAGE) + helper = importlib.import_module(f"{_PACKAGE}.rowwise_geography") + + assert sys.modules[_PACKAGE] is package + assert package.rowwise_geography is helper + assert callable(helper.assign_household_geography) + assert _unrelated_modules() == before + + +def test_public_from_import_returns_and_caches_the_real_defining_object(): + from microcosm.build.uk_runtime import assign_household_geography + from microcosm.build.uk_runtime.rowwise_geography import ( + assign_household_geography as direct, + ) + + package = importlib.import_module(_PACKAGE) + assert assign_household_geography is direct + assert package.assign_household_geography is direct + assert vars(package)["assign_household_geography"] is direct + assert package.assign_household_geography is assign_household_geography + + +def test_direct_submodule_import_and_from_import_preserve_module_identity(): + # In the fresh guard this exercises Python's fallback before any direct + # content_identity import, without removing an already loaded module in CI. + from microcosm.build.uk_runtime import content_identity + + assert content_identity is sys.modules[f"{_PACKAGE}.content_identity"] + + import microcosm.build.uk_runtime.content_identity as content_direct + import microcosm.build.uk_runtime.rowwise_geography as direct + from microcosm.build.uk_runtime import rowwise_geography + + package = importlib.import_module(_PACKAGE) + assert content_identity is content_direct + assert content_direct is sys.modules[f"{_PACKAGE}.content_identity"] + assert package.content_identity is content_direct + assert "content_identity" not in package.__all__ + assert rowwise_geography is direct + assert direct is sys.modules[f"{_PACKAGE}.rowwise_geography"] + assert package.rowwise_geography is direct + assert "rowwise_geography" not in package.__all__ + + +def test_discovery_preserves_the_frozen_ordered_export_contract_without_resolution(): + package = importlib.import_module(_PACKAGE) + imports_before = _unrelated_modules() + aliases_before = { + name: vars(package)[name] for name in package.__all__ if name in vars(package) + } + + discovered = dir(package) + + assert len(package.__all__) == 458 + assert len(set(package.__all__)) == 457 + assert package.__all__.count("ladder_vs_chronicle_household_dispersion") == 2 + encoded = json.dumps(package.__all__, separators=(",", ":")).encode() + assert hashlib.sha256(encoded).hexdigest() == _ORDERED_ALL_SHA256 + assert set(package.__all__).issubset(discovered) + assert discovered == sorted(set(discovered)) + aliases_after = { + name: vars(package)[name] for name in package.__all__ if name in vars(package) + } + assert aliases_after.keys() == aliases_before.keys() + assert all(aliases_after[name] is value for name, value in aliases_before.items()) + assert _unrelated_modules() == imports_before + with pytest.raises(TypeError): + package._EXPORTS["invented_missing_export"] = ("invented", "missing") + + +def test_unknown_attribute_and_unknown_from_import_keep_normal_failures(): + package = importlib.import_module(_PACKAGE) + with pytest.raises( + AttributeError, match="has no attribute 'invented_missing_export'" + ): + _ = package.invented_missing_export + with pytest.raises( + ImportError, match="cannot import name 'invented_missing_export'" + ): + from microcosm.build.uk_runtime import invented_missing_export # noqa: F401 + + assert "invented_missing_export" not in vars(package) + + +def test_reload_clears_public_aliases_and_rebinds_the_same_defining_objects(): + package = importlib.import_module(_PACKAGE) + helper = importlib.import_module(f"{_PACKAGE}.rowwise_geography") + real_function = helper.assign_household_geography + assert package.assign_household_geography is real_function + assert "assign_household_geography" in vars(package) + imports_before = _unrelated_modules() + + assert importlib.reload(package) is package + + assert "assign_household_geography" not in vars(package) + assert package.rowwise_geography is helper + assert sys.modules[f"{_PACKAGE}.rowwise_geography"] is helper + assert package.assign_household_geography is real_function + assert vars(package)["assign_household_geography"] is real_function + assert _unrelated_modules() == imports_before diff --git a/packages/microcosm-build/tests/test_us_acs_housing_source.py b/packages/microcosm-build/tests/test_us_acs_housing_source.py new file mode 100644 index 000000000..eb5a461c9 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_housing_source.py @@ -0,0 +1,2634 @@ +"""Independent invented-source tests for the approved ACS housing-unit source bridge. + +Written against the approved v2 plan contract, not against the production +module: no byte of ``acs_housing_universe_source.py`` or of the packaged +``acs_2024_housing_universe.json`` was read while authoring this file, so a +failure here is evidence about the contract rather than a restatement of the +implementation. + +Every archive, member, header, identifier and row below is wholly invented. No +genuine ACS/Census archive, candidate, target or dictionary file is opened, +statted or referenced. Nothing here authenticates real data, and nothing here +makes a scientific claim: the fixtures exercise the closed source rules only. + +Scope is the source producer/readback pair. ``prepare_acs_housing_population`` +builds a Frame and is deliberately left to the production author's own +Frame/graph tests. + +Two reconciliation seams are collected in one place each, so integrating this +file with the finished module is a rename exercise and never a semantic one: + +* ``NAMES`` - column, constant, filename and definition-name spellings. +* ``EXPECTED_REASONS`` - exact refusal reason codes, asserted only once + ``REASON_CODES_AGREED`` is flipped to True after agreeing them with the + production author. Until then every refusal test still asserts the exception + type, message sanitisation, and that semantically distinct causes carry + distinct codes, which is the part that does not need agreement. +""" + +import csv +import hashlib +import io +import json +import os +import re +import signal +import stat +import struct +import tempfile +import zipfile +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +# The production import must surface as a loud collection error naming the +# module while it is absent. It must never skip: no engine is involved here. +from microcosm.build.us_runtime import acs_housing_universe_source as source +from microcosm.build.us_runtime.acs_housing_universe import ( + classify_acs_housing_universe, +) + +# -------------------------------------------------------------------------- +# Reconciliation seam 1: identifier spellings. +# -------------------------------------------------------------------------- + +NAMES = { + # Frame columns carrying the archive-member/row lineage of each row. + "member_column": "source_member", + "ordinal_column": "source_row_ordinal", + # The normalised two-digit state column exposed on the household frame. + # ``_state_series`` falls back to the raw ST/STATE header aliases, so only + # the dedicated naming test depends on this spelling. + "state_column": "ST", + # Files written into the source artifact directory. + "projection_filename": "projection.json", + "receipt_filename": "receipt.json", + # Named source definitions bound by the artifact receipt. + "vacancy_definition": "acs_2024_source_vacancy_v1", + "gq_definition": "acs_2024_gq_person_placeholder_design_v1", + # Module byte caps, monkeypatched tiny so no test allocates real GiBs. + "source_max_bytes": "ACS_HU_SOURCE_MAX_BYTES", + "receipt_max_bytes": "ACS_HU_RECEIPT_MAX_BYTES", + # The private archive pin tuple; entries are (role, filename, sha256, size). + "archive_pins": "_ARCHIVE_PINS", + "household_role": "household", + "person_role": "person", +} + +HOUSEHOLD_ARCHIVE = "csv_hus.zip" +PERSON_ARCHIVE = "csv_pus.zip" + +# -------------------------------------------------------------------------- +# Reconciliation seam 2: refusal reason codes. +# -------------------------------------------------------------------------- + +REASON_CODES_AGREED = True + +EXPECTED_REASONS: dict[str, str] = { + "artifact_forgery": "RECONSTRUCTION_MISMATCH", + "capture_digest": "SOURCE_SHA256", + "csv_record": "CSV_ROW_WIDTH", + "header_contract": "CSV_STATE_REQUIRED", + "join_integrity": "ORPHAN_PERSON", + "output_precondition": "OUTPUT_EXISTS", + "selection": "SELECTION_UNKNOWN", + "value_domain": "TEN", + "zip_preflight": "ZIP_DUPLICATE_MEMBER", +} + +# Coarse refusal families that must not collapse onto one reason code. +DISTINCT_FAMILIES = ( + "capture_digest", + "zip_preflight", + "csv_record", + "header_contract", + "value_domain", + "join_integrity", + "selection", + "artifact_forgery", + "output_precondition", +) + +# -------------------------------------------------------------------------- +# Invented source bytes. +# -------------------------------------------------------------------------- + +HOUSEHOLD_HEADER = ( + "RT", + "SERIALNO", + "DIVISION", + "PUMA", + "REGION", + "ST", + "ADJHSG", + "ADJINC", + "WGTP", + "NP", + "TYPEHUGQ", + "ACR", + "BDSP", + "TEN", + "VALP", + "HINCP", +) + +PERSON_HEADER = ( + "RT", + "SERIALNO", + "SPORDER", + "PUMA", + "ST", + "ADJINC", + "PWGTP", + "AGEP", + "SEX", + "WAGP", +) + +HOUSEHOLD_DEFAULTS = { + "RT": "H", + "DIVISION": "9", + "PUMA": "00100", + "REGION": "4", + "ST": "06", + "ADJHSG": "1000000", + "ADJINC": "1010000", + "ACR": "1", + "BDSP": "3", + "VALP": "410000", + "HINCP": "82000", +} + +PERSON_DEFAULTS = { + "RT": "P", + "PUMA": "00100", + "ST": "06", + "ADJINC": "1010000", + "AGEP": "41", + "SEX": "1", + "WAGP": "52000", +} + +# Projected observation rosters, per the closed plan. +HOUSEHOLD_ROSTER = ("SERIALNO", "TYPEHUGQ", "NP", "TEN", "WGTP", "PUMA") +PERSON_ROSTER = ("SERIALNO", "SPORDER", "PWGTP") + +# Fixed classifier output order, read from the pinned pure module. +CODE_COLUMNS = ( + "interview_scope", + "physical_unit", + "household_kind", + "tenure_subtype", + "occupied_hu", + "hu_tenure_class", + "unresolved_reasons", + "TEN_valid", +) + + +def household_row(serialno, typehugq, persons, ten, wgtp, **overrides): + """One invented household record as an ordered tuple of lexical tokens.""" + values = dict(HOUSEHOLD_DEFAULTS) + values.update(SERIALNO=serialno, TYPEHUGQ=typehugq, NP=persons, TEN=ten, WGTP=wgtp) + values.update(overrides) + return tuple(values[name] for name in HOUSEHOLD_HEADER) + + +def person_row(serialno, sporder, pwgtp, **overrides): + """One invented person record as an ordered tuple of lexical tokens.""" + values = dict(PERSON_DEFAULTS) + values.update(SERIALNO=serialno, SPORDER=sporder, PWGTP=pwgtp) + values.update(overrides) + return tuple(values[name] for name in PERSON_HEADER) + + +# Eight invented households covering every accepted source class exactly once, +# plus both WGTP boundaries and the whole observed TEN domain. +BASE_HOUSEHOLDS = ( + # occupied owner with a mortgage, two people + household_row("2024HU0000001", "1", "2", "1", "118", PUMA="00100", ST="06"), + # occupied renter paying rent, one person + household_row("2024HU0000002", "1", "1", "3", "76", PUMA="03701", ST="06"), + # occupied housing unit whose tenure is not available, three people + household_row("2024HU0000003", "1", "3", "", "1", PUMA="81003", ST="36"), + # vacant housing unit: no people, blank tenure, positive weight + household_row("2024HU0000004", "1", "0", "", "55", PUMA="00100", ST="11"), + # institutional group quarters placeholder + household_row("2024GQ0000005", "2", "1", "", "0", PUMA="03701", ST="36"), + # noninstitutional group quarters placeholder + household_row("2024GQ0000006", "3", "1", "", "0", PUMA="00100", ST="01"), + # occupied owner outright + household_row("2024HU0000007", "1", "1", "2", "200", PUMA="00100", ST="01"), + # occupied renter paying no rent, maximum housing weight + household_row("2024HU0000008", "1", "1", "4", "9999", PUMA="81003", ST="11"), +) + +BASE_PERSONS = ( + person_row("2024HU0000001", "01", "121", WAGP="52000"), + person_row("2024HU0000001", "02", "97", WAGP="0"), + person_row("2024HU0000002", "1", "80", WAGP="31000"), + person_row("2024HU0000003", "1", "12", WAGP="17000"), + person_row("2024HU0000003", "2", "11", WAGP="0"), + person_row("2024HU0000003", "3", "9", WAGP="0"), + person_row("2024GQ0000005", "01", "64", WAGP="0"), + person_row("2024GQ0000006", "1", "9999", WAGP="7000"), + person_row("2024HU0000007", "01", "205", WAGP="99000"), + person_row("2024HU0000008", "1", "1", WAGP="4000"), +) + +# The base population is split across two members per role so that member and +# ordinal lineage is exercised by every ordinary test rather than by one. +BASE_HOUSEHOLD_MEMBERS = ( + ("psam_husa.csv", BASE_HOUSEHOLDS[:5]), + ("psam_husb.csv", BASE_HOUSEHOLDS[5:]), +) +BASE_PERSON_MEMBERS = ( + ("psam_pusa.csv", BASE_PERSONS[:6]), + ("psam_pusb.csv", BASE_PERSONS[6:]), +) + +BASE_SERIALNOS = tuple( + row[HOUSEHOLD_HEADER.index("SERIALNO")] for row in BASE_HOUSEHOLDS +) + +# The plan fixes what each row contains, never the order rows are emitted in. +# Content assertions below are therefore keyed by identifier, and the ordering +# decision lives in exactly one place: this constant and the two tests that +# read it. "sorted_serialno" is the order the production module emits; +# "source" would be member-then-row-ordinal order. See the author report. +CANONICAL_ROW_ORDER = "sorted_serialno" + + +def expected_row_order(serialnos): + if CANONICAL_ROW_ORDER == "sorted_serialno": + return sorted(serialnos) + if CANONICAL_ROW_ORDER == "source": + return list(serialnos) + raise AssertionError(f"unknown row-order contract {CANONICAL_ROW_ORDER!r}") + + +# (occupied_hu, hu_tenure_class, tenure_subtype, unresolved_reasons, +# physical_unit, TEN_valid) recomputed by hand from the pinned classifier. +BASE_EXPECTED_CODES = { + "interview_scope": [0, 0, 0, 0, 0, 0, 0, 0], + "physical_unit": [1, 1, 1, 1, 2, 2, 1, 1], + "household_kind": [0, 0, 0, 0, 0, 0, 0, 0], + "tenure_subtype": [1, 3, 0, 0, 0, 0, 2, 4], + "occupied_hu": [1, 1, 1, 0, 2, 2, 1, 1], + "hu_tenure_class": [1, 2, 3, 0, 0, 0, 1, 2], + "unresolved_reasons": [0, 0, 2, 1, 0, 0, 0, 0], + "TEN_valid": [1, 1, 0, 0, 0, 0, 1, 1], +} + + +# -------------------------------------------------------------------------- +# Building invented archives. +# -------------------------------------------------------------------------- + + +EXPECTED_HOUSEHOLDS = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: { + name: row[HOUSEHOLD_HEADER.index(name)] for name in HOUSEHOLD_ROSTER + ("ST",) + } + for row in BASE_HOUSEHOLDS +} + +EXPECTED_PERSONS = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): row[PERSON_HEADER.index("PWGTP")] + for row in BASE_PERSONS +} + +EXPECTED_CODES = { + serialno: tuple(BASE_EXPECTED_CODES[name][index] for name in CODE_COLUMNS) + for index, serialno in enumerate(BASE_SERIALNOS) +} + +EXPECTED_HOUSEHOLD_MEMBER = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: member + for member, rows in BASE_HOUSEHOLD_MEMBERS + for row in rows +} + +EXPECTED_PERSON_MEMBER = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): member + for member, rows in BASE_PERSON_MEMBERS + for row in rows +} + + +def csv_bytes(header, rows, *, newline="\n", trailing_newline=True): + """Serialise invented tokens without any quoting the fixtures do not ask for.""" + buffer = io.StringIO(newline="") + writer = csv.writer(buffer, lineterminator=newline, quoting=csv.QUOTE_MINIMAL) + writer.writerow(header) + writer.writerows(rows) + text = buffer.getvalue() + if not trailing_newline and text.endswith(newline): + text = text[: -len(newline)] + return text.encode("utf-8") + + +@dataclass(frozen=True) +class Member: + """One planned archive entry, data or auxiliary.""" + + name: str + data: bytes + compress_type: int = zipfile.ZIP_DEFLATED + external_attr: int | None = None + create_system: int = 3 + + +def data_members(header, members, *, newline="\n", **kwargs): + return tuple( + Member(name, csv_bytes(header, rows, newline=newline, **kwargs)) + for name, rows in members + ) + + +def write_archive(path, members): + """Write invented members exactly as planned, preserving order and duplicates.""" + with zipfile.ZipFile(path, "w") as archive: + for member in members: + info = zipfile.ZipInfo(member.name, date_time=(2026, 1, 1, 0, 0, 0)) + info.compress_type = member.compress_type + info.create_system = member.create_system + info.external_attr = ( + member.external_attr + if member.external_attr is not None + else (0o644 << 16) + ) + archive.writestr(info, member.data) + return path + + +def flip_archive_byte(path, offset=None): + """Change one compressed byte in place: same size, different digest. + + A deflated archive holds no plaintext, so a token replacement here would + silently do nothing and the test would stop testing what it claims. + """ + payload = Path(path).read_bytes() + index = len(payload) // 2 if offset is None else offset + mutated = payload[:index] + bytes([payload[index] ^ 0xFF]) + payload[index + 1 :] + assert len(mutated) == len(payload) and mutated != payload + Path(path).write_bytes(mutated) + return payload + + +def digest_and_size(path): + payload = Path(path).read_bytes() + return hashlib.sha256(payload).hexdigest(), len(payload) + + +@dataclass +class Fixture: + """An invented source directory plus everything a test needs to drive it.""" + + tmp_path: Path + source_dir: Path + snapshot_root: Path + monkeypatch: pytest.MonkeyPatch + households: tuple = () + persons: tuple = () + secrets: list = field(default_factory=list) + _output_index: int = 0 + + @property + def household_zip(self): + return self.source_dir / HOUSEHOLD_ARCHIVE + + @property + def person_zip(self): + return self.source_dir / PERSON_ARCHIVE + + def pin(self): + """Point the private pin tuple at the invented archives as they are now.""" + pins = [] + for role, name in ( + (NAMES["household_role"], HOUSEHOLD_ARCHIVE), + (NAMES["person_role"], PERSON_ARCHIVE), + ): + path = self.source_dir / name + sha, size = digest_and_size(path) + pins.append((role, name, sha, size)) + self.monkeypatch.setattr(source, NAMES["archive_pins"], tuple(pins)) + return tuple(pins) + + def output_dir(self, label="artifact"): + self._output_index += 1 + return self.tmp_path / f"out-{self._output_index}-{label}" + + def produce(self, *, output_dir=None, serialnos=None, source_dir=None): + return source.produce_acs_housing_source( + self.source_dir if source_dir is None else source_dir, + snapshot_root=self.snapshot_root, + output_dir=self.output_dir() if output_dir is None else output_dir, + serialnos=serialnos, + ) + + def load(self, output_dir, *, serialnos=None, source_dir=None): + return source.load_acs_housing_source( + self.source_dir if source_dir is None else source_dir, + output_dir, + snapshot_root=self.snapshot_root, + serialnos=serialnos, + ) + + def produce_and_load(self, *, serialnos=None): + output_dir = self.output_dir() + produced = self.produce(output_dir=output_dir, serialnos=serialnos) + return produced, self.load(output_dir, serialnos=serialnos), output_dir + + +def build_fixture( + tmp_path, + monkeypatch, + *, + households=BASE_HOUSEHOLD_MEMBERS, + persons=BASE_PERSON_MEMBERS, + household_members=None, + person_members=None, + household_header=HOUSEHOLD_HEADER, + person_header=PERSON_HEADER, + newline="\n", + pin=True, +): + """Assemble an invented source directory; explicit members override the rows.""" + # Resolve first: on macOS pytest's tmp_path sits under /var, a symlink to + # /private/var, and the capture refuses symlinked path components. The + # deliberate symlink refusals below build their own links explicitly. + tmp_path = Path(tmp_path).resolve() + source_dir = Path(tmp_path) / "acs-source" + source_dir.mkdir(parents=True, exist_ok=True) + snapshot_root = Path(tmp_path) / "snapshots" + snapshot_root.mkdir(parents=True, exist_ok=True) + + if household_members is None: + household_members = data_members(household_header, households, newline=newline) + if person_members is None: + person_members = data_members(person_header, persons, newline=newline) + + write_archive(source_dir / HOUSEHOLD_ARCHIVE, household_members) + write_archive(source_dir / PERSON_ARCHIVE, person_members) + + fixture = Fixture( + tmp_path=Path(tmp_path), + source_dir=source_dir, + snapshot_root=snapshot_root, + monkeypatch=monkeypatch, + households=households, + persons=persons, + ) + fixture.secrets = sanitisation_secrets(fixture) + if pin: + fixture.pin() + return fixture + + +def sanitisation_secrets(fixture): + """Strings a sanitised refusal must never echo back to the caller.""" + secrets = [ + str(fixture.tmp_path), + str(fixture.source_dir), + str(fixture.snapshot_root), + ] + secrets.extend(BASE_SERIALNOS) + # Unprojected invented money tokens: no legitimate reason code names these, + # unlike the domain bounds (1..9999, 100..81003) a code may legitimately cite. + secrets.extend(["82000", "52000", "410000", "31000", "99000", "17000"]) + return secrets + + +@pytest.fixture(autouse=True) +def _invented_disk_budget(monkeypatch): + """Eight invented rows must not require a full-source disk reserve in CI.""" + monkeypatch.setattr( + source.shutil, "disk_usage", lambda _path: SimpleNamespace(free=64 * 1024**3) + ) + + +@pytest.fixture +def acs(tmp_path, monkeypatch): + """The plain accepted invented population, pinned and ready to produce.""" + return build_fixture(tmp_path, monkeypatch) + + +# -------------------------------------------------------------------------- +# Refusal helpers. +# -------------------------------------------------------------------------- + + +def refuses(fixture, call, *args, **kwargs): + """Assert the exact refusal type, capture the reason, and check sanitisation.""" + with pytest.raises(source.ACSHousingSourceError) as caught: + call(*args, **kwargs) + reason = str(caught.value) + assert type(caught.value) is source.ACSHousingSourceError + assert reason, "a refusal must carry a reason" + assert len(reason) <= 120, "a reason code must not be prose" + assert "\n" not in reason + for secret in fixture.secrets: + assert secret not in reason, "refusal leaked invented source content" + return reason + + +def flat_scalars(payload): + """Every scalar reachable in a decoded receipt, for shape-tolerant assertions.""" + found = [] + stack = [payload] + while stack: + item = stack.pop() + if isinstance(item, dict): + stack.extend(item.keys()) + stack.extend(item.values()) + elif isinstance(item, (list, tuple)): + stack.extend(item) + else: + found.append(item) + return found + + +def flat_strings(payload): + return {item for item in flat_scalars(payload) if isinstance(item, str)} + + +def flat_numbers(payload): + return { + item + for item in flat_scalars(payload) + if isinstance(item, int) and not isinstance(item, bool) + } + + +def state_series(households): + """Read the state column under whichever spelling the module exposes.""" + for name in (NAMES["state_column"], "ST", "STATE"): + if name in households.columns: + return households[name] + raise AssertionError("household frame exposes no state column") + + +def household_keys(households): + keys = lexical(households["SERIALNO"]) + assert len(set(keys)) == len(keys), "household identifiers must be unique" + return keys + + +def household_map(households, names=HOUSEHOLD_ROSTER + ("ST",)): + """Every projected household token, keyed by identifier rather than by row.""" + keys = household_keys(households) + columns = { + name: lexical(state_series(households) if name == "ST" else households[name]) + for name in names + } + return { + key: {name: columns[name][row] for name in names} + for row, key in enumerate(keys) + } + + +def person_map(persons): + """Person weight keyed by identifier and the raw SPORDER token.""" + entries = list( + zip( + lexical(persons["SERIALNO"]), + lexical(persons["SPORDER"]), + lexical(persons["PWGTP"]), + strict=True, + ) + ) + keyed = {(serialno, sporder): weight for serialno, sporder, weight in entries} + assert len(keyed) == len(entries), "person keys must be unique" + return keyed + + +def codes_map(result): + """The eight native codes as a tuple per household identifier.""" + keys = household_keys(result.households) + codes = result.codes + return { + key: tuple(int(codes[name].iloc[row]) for name in CODE_COLUMNS) + for row, key in enumerate(keys) + } + + +def lexical(series): + """A projected column must be lexical text with no NA conversion.""" + values = series.tolist() + assert all(isinstance(value, str) for value in values), ( + f"{series.name} is not lexical: {[type(v).__name__ for v in values]}" + ) + return values + + +def strict_arrays(households): + """Rebuild the classifier inputs from the projected lexical household tokens.""" + tenure_tokens = lexical(households["TEN"]) + tenure_valid = np.asarray([token != "" for token in tenure_tokens], dtype=bool) + frame = pd.DataFrame( + { + "TYPEHUGQ": np.asarray( + [int(token) for token in lexical(households["TYPEHUGQ"])], dtype="int64" + ), + "NP": np.asarray( + [int(token) for token in lexical(households["NP"])], dtype="int64" + ), + "TEN": np.asarray( + [int(token) if token != "" else 0 for token in tenure_tokens], + dtype="int64", + ), + }, + index=households.index, + ) + return frame, tenure_valid + + +def assert_lineage(frame, keys, expected_member): + """Find member/ordinal lineage without depending on either column spelling. + + Order-independent by construction: the member column must agree row by row + with the invented layout, and each member's ordinals must form one + consecutive run, whatever order the rows are emitted in. + """ + members = [expected_member[key] for key in keys] + member_column = next( + ( + name + for name in frame.columns + if [value if isinstance(value, str) else None for value in frame[name]] + == members + ), + None, + ) + assert member_column is not None, ( + "no column carries the archive member of each row; expected " + f"{sorted(set(members))}" + ) + + for name in frame.columns: + if name == member_column: + continue + try: + values = [int(value) for value in frame[name].tolist()] + except (TypeError, ValueError): + continue + if len(values) != len(members): + continue + runs = {} + for member, value in zip(members, values, strict=True): + runs.setdefault(member, []).append(value) + bases = {min(run) for run in runs.values()} + if bases <= {0} or bases <= {1}: + if all( + sorted(run) == list(range(min(run), min(run) + len(run))) + for run in runs.values() + ): + return member_column, name + raise AssertionError( + "no column carries a within-member row ordinal restarting at each member" + ) + + +# -------------------------------------------------------------------------- +# Accepted population: projection, classification and roundtrip. +# -------------------------------------------------------------------------- + + +def test_projected_households_are_lexical_and_complete(acs): + result = acs.produce() + households = result.households + assert len(households) == len(BASE_HOUSEHOLDS) + assert households.columns.is_unique + assert set(HOUSEHOLD_ROSTER) <= set(households.columns) + for name in HOUSEHOLD_ROSTER: + lexical(households[name]) + # Every projected token survives byte for byte: blank tenure stays an empty + # string rather than NaN, None or the dictionary's display character, and + # leading zeros on PUMA and ST are preserved. + assert household_map(households) == EXPECTED_HOUSEHOLDS + + +def test_projected_persons_keep_raw_sporder_tokens(acs): + persons = acs.produce().persons + assert len(persons) == len(BASE_PERSONS) + assert set(PERSON_ROSTER) <= set(persons.columns) + # "01" and "1" are both accepted and both retained exactly as written. + assert person_map(persons) == EXPECTED_PERSONS + + +def test_native_codes_are_uint8_in_the_pinned_order(acs): + result = acs.produce() + codes = result.codes + assert list(codes.columns) == list(CODE_COLUMNS) + for name in CODE_COLUMNS: + assert codes[name].dtype == np.dtype("uint8") + assert codes.index.equals(result.households.index) + assert codes_map(result) == EXPECTED_CODES + + +def test_codes_equal_the_pinned_pure_classifier_on_the_strict_arrays(acs): + """The source result must not invent its own semantics for the same rows.""" + result = acs.produce() + frame, tenure_valid = strict_arrays(result.households) + expected, _header = classify_acs_housing_universe(frame, tenure_valid=tenure_valid) + pd.testing.assert_frame_equal(result.codes, expected, check_like=False) + + +def test_np0_housing_unit_stays_occupancy_unknown_in_the_native_codes(acs): + """Source vacancy evidence must not be back-written into the pure classes.""" + codes = dict( + zip(CODE_COLUMNS, codes_map(acs.produce())["2024HU0000004"], strict=True) + ) + assert codes["occupied_hu"] == 0 + assert codes["hu_tenure_class"] == 0 + assert codes["unresolved_reasons"] == 1 + assert codes["physical_unit"] == 1 + assert codes["TEN_valid"] == 0 + + +def test_occupied_unit_with_unavailable_tenure_is_retained_not_recoded(acs): + codes = dict( + zip(CODE_COLUMNS, codes_map(acs.produce())["2024HU0000003"], strict=True) + ) + assert codes["occupied_hu"] == 1 + assert codes["hu_tenure_class"] == 3 + assert codes["tenure_subtype"] == 0 + assert codes["unresolved_reasons"] == 2 + + +def test_member_and_row_ordinal_lineage_is_accessible(acs): + result = acs.produce() + assert_lineage( + result.households, + household_keys(result.households), + EXPECTED_HOUSEHOLD_MEMBER, + ) + assert_lineage( + result.persons, + list(person_map(result.persons)), + EXPECTED_PERSON_MEMBER, + ) + + +def test_rows_are_emitted_in_the_agreed_canonical_order(acs): + """The plan leaves row order open; this is the single place it is decided.""" + assert household_keys(acs.produce().households) == expected_row_order( + BASE_SERIALNOS + ) + + +def test_row_order_does_not_depend_on_the_member_split(tmp_path, monkeypatch): + """Splitting the same invented rows differently must not reorder them.""" + one = single_member(tmp_path / "one", monkeypatch).produce() + three = build_fixture( + tmp_path / "three", + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[6:])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:3])), + Member("psam_husc.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:6])), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ).produce() + assert household_keys(three.households) == household_keys(one.households) + assert household_map(three.households) == household_map(one.households) + + +def test_source_artifact_holds_exactly_the_projection_and_receipt(acs): + output_dir = acs.output_dir() + result = acs.produce(output_dir=output_dir) + assert sorted(path.name for path in output_dir.iterdir()) == sorted( + (NAMES["projection_filename"], NAMES["receipt_filename"]) + ) + # The house convention appends a trailing newline to published JSON. + assert (output_dir / NAMES["projection_filename"]).read_bytes().rstrip( + b"\n" + ) == result.projection_json.rstrip(b"\n") + assert (output_dir / NAMES["receipt_filename"]).read_bytes().rstrip( + b"\n" + ) == result.receipt_json.rstrip(b"\n") + json.loads(result.projection_json) + json.loads(result.receipt_json) + + +def test_readback_returns_the_same_bytes_and_the_same_population(acs): + produced, loaded, _output_dir = acs.produce_and_load() + assert loaded.projection_json == produced.projection_json + assert loaded.receipt_json == produced.receipt_json + pd.testing.assert_frame_equal(loaded.households, produced.households) + pd.testing.assert_frame_equal(loaded.persons, produced.persons) + pd.testing.assert_frame_equal(loaded.codes, produced.codes) + + +def test_two_produces_from_identical_inputs_are_byte_identical(acs): + """Readback compares exact bytes, so both artifacts must be deterministic.""" + first = acs.produce(output_dir=acs.output_dir("first")) + second = acs.produce(output_dir=acs.output_dir("second")) + assert first.projection_json == second.projection_json + assert first.receipt_json == second.receipt_json + + +@pytest.mark.parametrize("attribute", ["households", "persons", "codes"]) +def test_returned_frames_are_owned_copies(acs, attribute): + result = acs.produce() + first = getattr(result, attribute) + assert getattr(result, attribute) is not first + before = first.copy(deep=True) + projection = result.projection_json + first.iloc[0, 0] = first.iloc[-1, 0] + first.rename(columns={first.columns[0]: "clobbered"}, inplace=True) + first.index = pd.RangeIndex(1000, 1000 + len(first)) + after = getattr(result, attribute) + pd.testing.assert_frame_equal(after, before) + assert result.projection_json == projection + + +def test_receipt_records_auditable_member_and_definition_evidence(acs): + result = acs.produce() + receipt = json.loads(result.receipt_json) + strings = flat_strings(receipt) + numbers = flat_numbers(receipt) + + assert NAMES["vacancy_definition"] in strings + assert NAMES["gq_definition"] in strings + assert {HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE} <= strings + assert {"psam_husa.csv", "psam_husb.csv", "psam_pusa.csv", "psam_pusb.csv"} <= ( + strings + ) + + for archive, names in ( + (acs.household_zip, ("psam_husa.csv", "psam_husb.csv")), + (acs.person_zip, ("psam_pusa.csv", "psam_pusb.csv")), + ): + with zipfile.ZipFile(archive) as opened: + for name in names: + info = opened.getinfo(name) + payload = opened.read(name) + assert hashlib.sha256(payload).hexdigest() in strings, ( + f"receipt omits the decompressed digest of {name}" + ) + assert info.CRC in numbers, f"receipt omits the CRC of {name}" + assert info.file_size in numbers + + # Counts an auditor needs: the whole population and the vacancy evidence. + assert len(BASE_HOUSEHOLDS) in numbers + assert len(BASE_PERSONS) in numbers + + +def test_receipt_binds_the_pinned_archive_digests(acs): + pins = acs.pin() + strings = flat_strings(json.loads(acs.produce().receipt_json)) + for _role, name, sha, _size in pins: + assert sha in strings, f"receipt omits the pinned digest of {name}" + + +# -------------------------------------------------------------------------- +# Population variants: one deliberate change against the accepted invented rows. +# -------------------------------------------------------------------------- + + +def with_household(index, **fields): + rows = list(BASE_HOUSEHOLDS) + values = dict(zip(HOUSEHOLD_HEADER, rows[index], strict=True)) + values.update(fields) + rows[index] = tuple(values[name] for name in HOUSEHOLD_HEADER) + return tuple(rows) + + +def with_person(index, **fields): + rows = list(BASE_PERSONS) + values = dict(zip(PERSON_HEADER, rows[index], strict=True)) + values.update(fields) + rows[index] = tuple(values[name] for name in PERSON_HEADER) + return tuple(rows) + + +def rename_serialno(old, new): + """Rename a key on both sides so the only change is the identifier token.""" + key = HOUSEHOLD_HEADER.index("SERIALNO") + households = tuple( + row[:key] + (new,) + row[key + 1 :] if row[key] == old else row + for row in BASE_HOUSEHOLDS + ) + key = PERSON_HEADER.index("SERIALNO") + persons = tuple( + row[:key] + (new,) + row[key + 1 :] if row[key] == old else row + for row in BASE_PERSONS + ) + return households, persons + + +def single_member(tmp_path, monkeypatch, *, households=None, persons=None, **kwargs): + """One member per role, so a variant's only difference is the row content.""" + return build_fixture( + tmp_path, + monkeypatch, + households=( + ("psam_husa.csv", BASE_HOUSEHOLDS if households is None else households), + ), + persons=(("psam_pusa.csv", BASE_PERSONS if persons is None else persons),), + **kwargs, + ) + + +# -------------------------------------------------------------------------- +# Key selection. +# -------------------------------------------------------------------------- + + +def test_selection_none_retains_the_whole_invented_source(acs): + result = acs.produce(serialnos=None) + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert len(result.persons) == len(BASE_PERSONS) + + +def test_selection_keeps_whole_households_and_no_other_person(acs): + result = acs.produce(serialnos=("2024HU0000001", "2024GQ0000005")) + assert set(household_keys(result.households)) == { + "2024HU0000001", + "2024GQ0000005", + } + assert person_map(result.persons) == { + ("2024HU0000001", "01"): "121", + ("2024HU0000001", "02"): "97", + ("2024GQ0000005", "01"): "64", + } + assert codes_map(result) == { + key: EXPECTED_CODES[key] for key in ("2024HU0000001", "2024GQ0000005") + } + + +def test_a_vacancy_only_selection_is_a_valid_source_projection(acs): + """Source-only preparation may validly retain no people at all.""" + result = acs.produce(serialnos=("2024HU0000004",)) + assert household_keys(result.households) == ["2024HU0000004"] + assert len(result.persons) == 0 + assert result.codes["occupied_hu"].tolist() == [0] + assert result.codes["unresolved_reasons"].tolist() == [1] + + +def test_selected_rows_ignore_the_caller_tuple_order(acs): + """The request is a set of keys; canonical order is the module's own.""" + forward = acs.produce( + output_dir=acs.output_dir("forward"), + serialnos=("2024HU0000002", "2024HU0000008"), + ) + reversed_request = acs.produce( + output_dir=acs.output_dir("reversed"), + serialnos=("2024HU0000008", "2024HU0000002"), + ) + assert forward.projection_json == reversed_request.projection_json + assert household_keys(forward.households) == expected_row_order( + ("2024HU0000002", "2024HU0000008") + ) + + +def test_selection_changes_the_projection_bytes(acs): + whole = acs.produce() + selected = acs.produce(serialnos=("2024HU0000001",)) + assert selected.projection_json != whole.projection_json + + +def test_readback_under_a_different_selection_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir, serialnos=("2024HU0000001",)) + refuses(acs, acs.load, output_dir, serialnos=("2024HU0000002",)) + refuses(acs, acs.load, output_dir, serialnos=None) + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param(("2024HU0000009",), id="unknown_key"), + pytest.param(("2024HU0000001", "2024HU0000009"), id="one_unknown_key"), + pytest.param(("2024HU0000001", "2024HU0000001"), id="duplicate_key"), + pytest.param((), id="empty_selection"), + pytest.param(("",), id="blank_key"), + pytest.param((" 2024HU0000001",), id="whitespace_padded_key"), + pytest.param(("2024hu0000001",), id="case_folded_key"), + ], +) +def test_invalid_selection_refuses(acs, serialnos): + refuses(acs, acs.produce, serialnos=serialnos) + + +# -------------------------------------------------------------------------- +# Complete-join integrity, checked before any exclusion or selection. +# -------------------------------------------------------------------------- + +JOIN_VARIANTS = [ + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000004", "1", "40"),), + id="person_attached_to_a_vacant_household", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + id="orphan_person_without_a_household", + ), + pytest.param( + BASE_HOUSEHOLDS + (BASE_HOUSEHOLDS[0],), + BASE_PERSONS, + id="duplicate_household_key", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000002", "1", "44"),), + id="duplicate_person_key", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000002", "01", "44"),), + id="duplicate_person_key_via_leading_zero_alias", + ), + pytest.param( + with_household(0, NP="3"), + BASE_PERSONS, + id="household_np_above_linked_person_count", + ), + pytest.param( + with_household(0, NP="1"), + BASE_PERSONS, + id="household_np_below_linked_person_count", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS[:6] + BASE_PERSONS[7:], + id="group_quarters_placeholder_without_its_person", + ), + pytest.param( + with_household(4, NP="2"), + BASE_PERSONS + (person_row("2024GQ0000005", "02", "51"),), + id="group_quarters_placeholder_with_two_people", + ), + pytest.param( + with_household(4, NP="0"), + BASE_PERSONS[:6] + BASE_PERSONS[7:], + id="group_quarters_placeholder_with_no_people", + ), +] + + +@pytest.mark.parametrize("households,persons", JOIN_VARIANTS) +def test_join_integrity_refusals(tmp_path, monkeypatch, households, persons): + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + refuses(fixture, fixture.produce) + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param(None, id="whole_source"), + pytest.param(("2024HU0000001",), id="unrelated_single_key"), + ], +) +def test_vacant_household_person_refuses_before_exclusion_or_selection( + tmp_path, monkeypatch, serialnos +): + """An NP0 household with a person must refuse even when nothing selects it. + + An implementation that dropped NP0 rows, or applied the key selection, + before validating would silently accept this invented contradiction. + """ + fixture = single_member( + tmp_path, + monkeypatch, + persons=BASE_PERSONS + (person_row("2024HU0000004", "1", "40"),), + ) + refuses(fixture, fixture.produce, serialnos=serialnos) + + +def test_orphan_person_refuses_even_when_selection_excludes_it(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, + monkeypatch, + persons=BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + ) + refuses(fixture, fixture.produce, serialnos=("2024HU0000002",)) + + +# -------------------------------------------------------------------------- +# Closed source-value domains. +# -------------------------------------------------------------------------- + +DOMAIN_VARIANTS = [ + # SERIALNO: exactly 13 ASCII characters, 2024HU/2024GQ plus seven digits, + # numeric suffix 1..9999999, prefix agreeing with TYPEHUGQ, no whitespace, + # no empty identifier and no normalisation of the accepted bytes. + pytest.param( + *rename_serialno("2024HU0000001", " 2024HU000001"), id="serialno_leading_space" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000001 "), id="serialno_trailing_space" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU 000001"), id="serialno_interior_space" + ), + pytest.param(*rename_serialno("2024HU0000001", ""), id="serialno_empty"), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000001"), id="serialno_too_short" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU00000010"), id="serialno_too_long" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2023HU0000001"), id="serialno_wrong_vintage" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024XX0000001"), + id="serialno_unknown_type_token", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024hu0000001"), + id="serialno_lowercase_type_token", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU0000000"), id="serialno_zero_suffix" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000000A"), + id="serialno_nondigit_suffix", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024GQ0000001"), + id="serialno_gq_prefix_on_housing_unit", + ), + pytest.param( + *rename_serialno("2024GQ0000005", "2024HU0000005"), + id="serialno_hu_prefix_on_group_quarters", + ), + # SPORDER: one or two ASCII digits, value 1..20, no sign/exponent/point. + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="0"), id="sporder_zero"), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="21"), id="sporder_above_domain" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="001"), id="sporder_three_digits" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER=" 1"), id="sporder_leading_space" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="1 "), id="sporder_trailing_space" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="+1"), id="sporder_signed"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="-1"), id="sporder_negative"), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="1.0"), id="sporder_decimal_point" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="1e0"), id="sporder_exponent"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER=""), id="sporder_empty"), + # TYPEHUGQ: 1, 2 or 3 only. + pytest.param(with_household(0, TYPEHUGQ="0"), BASE_PERSONS, id="typehugq_zero"), + pytest.param( + with_household(0, TYPEHUGQ="4"), BASE_PERSONS, id="typehugq_above_domain" + ), + pytest.param(with_household(0, TYPEHUGQ=""), BASE_PERSONS, id="typehugq_empty"), + pytest.param( + with_household(0, TYPEHUGQ="H"), BASE_PERSONS, id="typehugq_nonnumeric" + ), + # NP: published integer domain 0..20. + pytest.param(with_household(0, NP="21"), BASE_PERSONS, id="np_above_domain"), + pytest.param(with_household(0, NP="-1"), BASE_PERSONS, id="np_negative"), + pytest.param(with_household(0, NP=""), BASE_PERSONS, id="np_empty"), + pytest.param(with_household(0, NP=" 2"), BASE_PERSONS, id="np_leading_space"), + pytest.param(with_household(0, NP="2.0"), BASE_PERSONS, id="np_decimal_point"), + # WGTP: 1..9999 for housing units including vacant ones; zero only for the + # group-quarters placeholders. + pytest.param( + with_household(0, WGTP="0"), BASE_PERSONS, id="wgtp_zero_on_occupied_unit" + ), + pytest.param( + with_household(3, WGTP="0"), BASE_PERSONS, id="wgtp_zero_on_vacant_unit" + ), + pytest.param(with_household(0, WGTP="10000"), BASE_PERSONS, id="wgtp_above_domain"), + pytest.param(with_household(0, WGTP="-5"), BASE_PERSONS, id="wgtp_negative"), + pytest.param(with_household(0, WGTP=""), BASE_PERSONS, id="wgtp_empty"), + # PWGTP: 1..9999 for every person, group quarters included. + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP="0"), id="pwgtp_zero"), + pytest.param( + BASE_HOUSEHOLDS, + with_person(6, PWGTP="0"), + id="pwgtp_zero_on_group_quarters_person", + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, PWGTP="10000"), id="pwgtp_above_domain" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP="-1"), id="pwgtp_negative"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP=""), id="pwgtp_empty"), + # TEN: blank means missing; observed values are 1..4; group quarters and + # vacancy require the blank token. + pytest.param(with_household(0, TEN="5"), BASE_PERSONS, id="ten_above_domain"), + pytest.param(with_household(0, TEN="0"), BASE_PERSONS, id="ten_zero"), + pytest.param(with_household(0, TEN="b"), BASE_PERSONS, id="ten_display_character"), + pytest.param(with_household(0, TEN=" "), BASE_PERSONS, id="ten_single_space"), + pytest.param( + with_household(3, TEN="1"), BASE_PERSONS, id="ten_present_on_vacant_unit" + ), + pytest.param( + with_household(4, TEN="1"), BASE_PERSONS, id="ten_present_on_group_quarters" + ), + # PUMA: exactly five ASCII digits with integer value 100..81003. + pytest.param(with_household(0, PUMA="0010"), BASE_PERSONS, id="puma_four_digits"), + pytest.param(with_household(0, PUMA="000100"), BASE_PERSONS, id="puma_six_digits"), + pytest.param(with_household(0, PUMA="00099"), BASE_PERSONS, id="puma_below_domain"), + pytest.param(with_household(0, PUMA="81004"), BASE_PERSONS, id="puma_above_domain"), + pytest.param(with_household(0, PUMA="0010A"), BASE_PERSONS, id="puma_nondigit"), + pytest.param(with_household(0, PUMA=""), BASE_PERSONS, id="puma_empty"), + # ST: exactly two ASCII digits from the 50-state and DC subset. + pytest.param(with_household(0, ST="72"), BASE_PERSONS, id="state_puerto_rico"), + pytest.param(with_household(0, ST="99"), BASE_PERSONS, id="state_unassigned_code"), + pytest.param(with_household(0, ST="6"), BASE_PERSONS, id="state_single_digit"), + pytest.param(with_household(0, ST="006"), BASE_PERSONS, id="state_three_digits"), + pytest.param( + with_household(0, ST="CA"), BASE_PERSONS, id="state_postal_abbreviation" + ), + pytest.param(with_household(0, ST=""), BASE_PERSONS, id="state_empty"), +] + + +@pytest.mark.parametrize("households,persons", DOMAIN_VARIANTS) +def test_source_value_domain_refusals(tmp_path, monkeypatch, households, persons): + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + refuses(fixture, fixture.produce) + + +def test_sporder_leading_zero_is_accepted_and_the_raw_token_is_retained( + tmp_path, monkeypatch +): + """Leading-zero SPORDER normalises for the join without losing its bytes.""" + fixture = single_member(tmp_path, monkeypatch, persons=with_person(2, SPORDER="01")) + assert ("2024HU0000002", "01") in person_map(fixture.produce().persons) + + +def test_maximum_and_minimum_accepted_weights_are_retained(acs): + weights = household_map(acs.produce().households) + assert weights["2024HU0000003"]["WGTP"] == "1" + assert weights["2024HU0000008"]["WGTP"] == "9999" + + +# -------------------------------------------------------------------------- +# Strict lexical parsing and the ordered-header contract. +# -------------------------------------------------------------------------- + +BASE_HOUSEHOLD_CSV = csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS) +BASE_PERSON_CSV = csv_bytes(PERSON_HEADER, BASE_PERSONS) + + +def household_bytes_fixture(tmp_path, monkeypatch, payload, *, name="psam_husa.csv"): + return build_fixture( + tmp_path, + monkeypatch, + household_members=(Member(name, payload),), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + + +def person_bytes_fixture(tmp_path, monkeypatch, payload, *, name="psam_pusa.csv"): + return build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + person_members=(Member(name, payload),), + ) + + +def splice_line(payload, index, line): + """Insert an invented physical line at a record boundary.""" + lines = payload.split(b"\n") + lines.insert(index, line) + return b"\n".join(lines) + + +def drop_column(header, rows, name): + position = header.index(name) + trimmed = header[:position] + header[position + 1 :] + return trimmed, tuple(row[:position] + row[position + 1 :] for row in rows) + + +def rename_column(header, old, new): + return tuple(new if name == old else name for name in header) + + +WIDE_SUFFIX = tuple(f"AUX{index:02d}" for index in range(24)) +WIDE_HEADER = HOUSEHOLD_HEADER + WIDE_SUFFIX +WIDE_ROWS = tuple(row + ("0",) * len(WIDE_SUFFIX) for row in BASE_HOUSEHOLDS) + + +def oversized_record_csv(): + """One physical record above 1 MiB whose every field stays under 64 KiB.""" + padded = ("4" * 50_000,) * len(WIDE_SUFFIX) + rows = (WIDE_ROWS[0][: len(HOUSEHOLD_HEADER)] + padded,) + WIDE_ROWS[1:] + return csv_bytes(WIDE_HEADER, rows) + + +def oversized_field_csv(): + """One field token above 64 KiB in a column that is never projected.""" + return csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="7" * 65_537)) + + +HOUSEHOLD_PARSE_VARIANTS = [ + pytest.param(b"\xef\xbb\xbf" + BASE_HOUSEHOLD_CSV, id="utf8_byte_order_mark"), + pytest.param( + BASE_HOUSEHOLD_CSV.replace(b"2024HU0000002", b"2024HU\xff000002"), + id="invalid_utf8_byte", + ), + pytest.param( + BASE_HOUSEHOLD_CSV.replace(b"82000", b"820\x0000"), id="embedded_nul_in_field" + ), + pytest.param(splice_line(BASE_HOUSEHOLD_CSV, 3, b""), id="interior_blank_record"), + pytest.param(BASE_HOUSEHOLD_CSV + b"\n", id="trailing_blank_record"), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0][:-1]).encode() + + b"\n", + id="short_record", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0] + ("extra",)).encode() + + b"\n", + id="long_record", + ), + pytest.param( + csv_bytes(rename_column(HOUSEHOLD_HEADER, "REGION", "ST"), BASE_HOUSEHOLDS), + id="duplicate_header_name", + ), + pytest.param( + csv_bytes(rename_column(HOUSEHOLD_HEADER, "REGION", ""), BASE_HOUSEHOLDS), + id="blank_header_name", + ), + pytest.param( + csv_bytes( + rename_column(HOUSEHOLD_HEADER, "SERIALNO", " SERIALNO"), BASE_HOUSEHOLDS + ), + id="whitespace_padded_header_name", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "NP")), + id="missing_np_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "SERIALNO")), + id="missing_serialno_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "TEN")), + id="missing_ten_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "WGTP")), + id="missing_wgtp_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "PUMA")), + id="missing_puma_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "ST")), + id="neither_st_nor_state_column", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="82000\r82000")), + id="embedded_carriage_return_in_field", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="82000\n82000")), + id="embedded_line_feed_in_field", + ), + pytest.param(oversized_field_csv(), id="field_token_above_64_kib"), + pytest.param(oversized_record_csv(), id="physical_record_above_1_mib"), + pytest.param(BASE_HOUSEHOLD_CSV.replace(b",", b";"), id="semicolon_delimiter"), + pytest.param(b"", id="empty_member_without_a_header"), +] + + +@pytest.mark.parametrize("payload", HOUSEHOLD_PARSE_VARIANTS) +def test_household_parser_refusals(tmp_path, monkeypatch, payload): + fixture = household_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +PERSON_PARSE_VARIANTS = [ + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "SPORDER")), + id="missing_sporder_column", + ), + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "PWGTP")), + id="missing_pwgtp_column", + ), + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "SERIALNO")), + id="missing_person_serialno_column", + ), + pytest.param(splice_line(BASE_PERSON_CSV, 2, b""), id="person_blank_record"), + pytest.param( + csv_bytes(PERSON_HEADER, BASE_PERSONS[1:]) + + ",".join(BASE_PERSONS[0][:-2]).encode() + + b"\n", + id="person_short_record", + ), +] + + +@pytest.mark.parametrize("payload", PERSON_PARSE_VARIANTS) +def test_person_parser_refusals(tmp_path, monkeypatch, payload): + fixture = person_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +def test_cross_member_reordered_header_refuses(tmp_path, monkeypatch): + """Identical column sets in a different order are still a header mismatch.""" + reordered = list(HOUSEHOLD_HEADER) + first, second = reordered.index("REGION"), reordered.index("ADJHSG") + reordered[first], reordered[second] = reordered[second], reordered[first] + rows = tuple( + tuple(dict(zip(HOUSEHOLD_HEADER, row, strict=True))[name] for name in reordered) + for row in BASE_HOUSEHOLDS[5:] + ) + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member("psam_husb.csv", csv_bytes(tuple(reordered), rows)), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_cross_member_extra_column_refuses(tmp_path, monkeypatch): + extended = HOUSEHOLD_HEADER + ("FINCP",) + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member( + "psam_husb.csv", + csv_bytes(extended, tuple(row + ("0",) for row in BASE_HOUSEHOLDS[5:])), + ), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_header_only_member_is_allowed_and_contributes_no_rows(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, ())), + ), + person_members=( + Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())), + Member("psam_pusb.csv", BASE_PERSON_CSV), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert person_map(result.persons) == EXPECTED_PERSONS + assert_lineage( + result.households, + household_keys(result.households), + dict.fromkeys(BASE_SERIALNOS, "psam_husa.csv"), + ) + assert_lineage( + result.persons, + list(person_map(result.persons)), + dict.fromkeys(EXPECTED_PERSONS, "psam_pusb.csv"), + ) + # The empty members are still inventoried even though they carry no rows. + strings = flat_strings(json.loads(result.receipt_json)) + assert {"psam_husb.csv", "psam_pusa.csv"} <= strings + + +def test_an_entirely_header_only_source_is_not_a_population(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, ())),), + person_members=(Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())),), + ) + refuses(fixture, fixture.produce) + + +def test_state_only_household_header_is_supported(tmp_path, monkeypatch): + header = rename_column(HOUSEHOLD_HEADER, "ST", "STATE") + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(header, BASE_HOUSEHOLDS) + ) + result = fixture.produce() + assert household_map(result.households)["2024HU0000001"]["ST"] == "06" + # The header alias itself is preserved in the source evidence. + assert "STATE" in flat_strings(json.loads(result.receipt_json)) + + +def test_agreeing_st_and_state_columns_are_accepted(tmp_path, monkeypatch): + header = HOUSEHOLD_HEADER + ("STATE",) + state = HOUSEHOLD_HEADER.index("ST") + rows = tuple(row + (row[state],) for row in BASE_HOUSEHOLDS) + fixture = household_bytes_fixture(tmp_path, monkeypatch, csv_bytes(header, rows)) + result = fixture.produce() + assert household_map(result.households)["2024HU0000003"]["ST"] == "36" + + +def test_disagreeing_st_and_state_columns_refuse(tmp_path, monkeypatch): + header = HOUSEHOLD_HEADER + ("STATE",) + state = HOUSEHOLD_HEADER.index("ST") + rows = tuple( + row + ("36" if index == 0 else row[state],) + for index, row in enumerate(BASE_HOUSEHOLDS) + ) + fixture = household_bytes_fixture(tmp_path, monkeypatch, csv_bytes(header, rows)) + refuses(fixture, fixture.produce) + + +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_both_physical_line_terminators_project_identically( + tmp_path, monkeypatch, newline +): + """A genuine archive's terminator choice must not change the projection.""" + fixture = build_fixture(tmp_path, monkeypatch, newline=newline) + result = fixture.produce() + assert household_map(result.households) == EXPECTED_HOUSEHOLDS + assert person_map(result.persons) == EXPECTED_PERSONS + + +# -------------------------------------------------------------------------- +# Archive preflight, run on the central directory before any by-name lookup. +# -------------------------------------------------------------------------- + +SYMLINK_ATTR = (stat.S_IFLNK | 0o777) << 16 +DIRECTORY_ATTR = (stat.S_IFDIR | 0o755) << 16 | 0x10 + + +def household_archive_fixture(tmp_path, monkeypatch, members): + return build_fixture( + tmp_path, + monkeypatch, + household_members=members, + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + + +def aux_members(count, *, prefix="ACS2024_note"): + return tuple( + Member(f"{prefix}{index:03d}.txt", b"invented auxiliary text\n") + for index in range(count) + ) + + +ARCHIVE_VARIANTS = [ + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + ), + id="duplicate_member_name", + ), + pytest.param( + ( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member("PSAM_HUSA.CSV", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[5:])), + ), + id="case_folded_member_alias", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_hus/", b"", external_attr=DIRECTORY_ATTR), + ), + id="directory_entry", + ), + pytest.param( + (Member("pums/psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="nested_path_member", + ), + pytest.param( + (Member("/psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="absolute_path_member", + ), + pytest.param( + (Member("../psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="parent_traversal_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.txt", BASE_HOUSEHOLD_CSV), + ), + id="prefix_matching_non_csv_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member( + "psam_husb.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_BZIP2 + ), + ), + id="bzip2_compressed_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_link.csv", b"psam_husa.csv", external_attr=SYMLINK_ATTR), + ), + id="symlink_member", + ), + pytest.param( + (Member("psam_pusa.csv", BASE_PERSON_CSV),), + id="wrong_role_members_only", + ), + pytest.param( + (Member("ACS2024_readme.txt", b"invented auxiliary text\n"),), + id="no_applicable_csv_member", + ), + pytest.param((), id="empty_archive"), + pytest.param( + (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + aux_members(64), + id="member_count_above_cap", + ), +] + + +@pytest.mark.parametrize("members", ARCHIVE_VARIANTS) +def test_archive_preflight_refusals(tmp_path, monkeypatch, members): + fixture = household_archive_fixture(tmp_path, monkeypatch, members) + refuses(fixture, fixture.produce) + + +def test_member_count_at_the_cap_is_accepted(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + aux_members(63), + ) + assert len(fixture.produce().households) == len(BASE_HOUSEHOLDS) + + +def test_ordinary_auxiliary_members_are_inventory_only(tmp_path, monkeypatch): + """A non-matching ordinary file is recorded but never read as data.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("ACS2024_PUMS_README.txt", b"invented auxiliary text\n"), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert_lineage( + result.households, + household_keys(result.households), + dict.fromkeys(BASE_SERIALNOS, "psam_husa.csv"), + ) + assert "ACS2024_PUMS_README.txt" in flat_strings(json.loads(result.receipt_json)) + + +def test_a_non_matching_csv_member_is_auxiliary_not_data(tmp_path, monkeypatch): + """The role prefix, not the extension, decides what counts as data.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("ACS2024_shells.csv", csv_bytes(("A", "B"), (("1", "2"),))), + ), + ) + result = fixture.produce() + assert len(result.households) == len(BASE_HOUSEHOLDS) + assert "ACS2024_shells.csv" in flat_strings(json.loads(result.receipt_json)) + + +def test_stored_members_are_accepted(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member( + "psam_husa.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_STORED + ), + ), + ) + assert len(fixture.produce().households) == len(BASE_HOUSEHOLDS) + + +def test_several_members_per_role_join_in_name_order(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:3])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:6])), + Member("psam_husc.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[6:])), + ), + person_members=( + Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, BASE_PERSONS[:4])), + Member("psam_pusb.csv", csv_bytes(PERSON_HEADER, BASE_PERSONS[4:])), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + members = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: name + for name, rows in ( + ("psam_husa.csv", BASE_HOUSEHOLDS[:3]), + ("psam_husb.csv", BASE_HOUSEHOLDS[3:6]), + ("psam_husc.csv", BASE_HOUSEHOLDS[6:]), + ) + for row in rows + } + assert_lineage(result.households, household_keys(result.households), members) + # 2024HU0000003's three people straddle the two person members. + assert lexical(result.persons["SERIALNO"]).count("2024HU0000003") == 3 + assert person_map(result.persons) == EXPECTED_PERSONS + person_members = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): name + for name, rows in ( + ("psam_pusa.csv", BASE_PERSONS[:4]), + ("psam_pusb.csv", BASE_PERSONS[4:]), + ) + for row in rows + } + assert_lineage(result.persons, list(person_map(result.persons)), person_members) + + +@pytest.mark.parametrize("archive", [HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE]) +def test_missing_role_archive_refuses(acs, archive): + (acs.source_dir / archive).unlink() + refuses(acs, acs.produce) + + +def test_corrupted_member_payload_fails_the_recorded_crc(tmp_path, monkeypatch): + """A stored member whose bytes were swapped keeps its declared size and CRC.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member( + "psam_husa.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_STORED + ), + ), + ) + payload = fixture.household_zip.read_bytes() + assert b"82000" in payload, "the stored member is not plain in the archive" + fixture.household_zip.write_bytes(payload.replace(b"82000", b"82001", 1)) + fixture.pin() + refuses(fixture, fixture.produce) + + +# -------------------------------------------------------------------------- +# Private capture: paths, pins, originals and byte bounds. +# -------------------------------------------------------------------------- + + +@contextmanager +def bounded_time(seconds=30): + """Fail rather than hang if a non-regular original is opened blocking.""" + + def expire(signum, frame): + raise TimeoutError("the call did not return within the bounded test window") + + previous = signal.signal(signal.SIGALRM, expire) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def test_missing_snapshot_root_refuses(acs): + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=acs.tmp_path / "absent-root", + output_dir=acs.output_dir(), + ) + + +def test_snapshot_root_that_is_a_file_refuses(acs): + root = acs.tmp_path / "root-file" + root.write_bytes(b"invented\n") + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=root, + output_dir=acs.output_dir(), + ) + + +def test_symlinked_snapshot_root_refuses(acs): + link = acs.tmp_path / "root-link" + link.symlink_to(acs.snapshot_root, target_is_directory=True) + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=link, + output_dir=acs.output_dir(), + ) + + +def test_existing_output_directory_refuses(acs): + output_dir = acs.output_dir() + output_dir.mkdir() + refuses(acs, acs.produce, output_dir=output_dir) + + +def test_existing_output_file_refuses(acs): + output_dir = acs.output_dir() + output_dir.write_bytes(b"invented\n") + refuses(acs, acs.produce, output_dir=output_dir) + + +def test_output_directory_without_a_parent_refuses(acs): + refuses(acs, acs.produce, output_dir=acs.tmp_path / "absent" / "artifact") + + +def test_missing_source_directory_refuses(acs): + refuses(acs, acs.produce, source_dir=acs.tmp_path / "absent-source") + + +def test_source_directory_that_is_a_file_refuses(acs): + path = acs.tmp_path / "source-file" + path.write_bytes(b"invented\n") + refuses(acs, acs.produce, source_dir=path) + + +def test_symlinked_source_directory_refuses(acs): + link = acs.tmp_path / "source-link" + link.symlink_to(acs.source_dir, target_is_directory=True) + refuses(acs, acs.produce, source_dir=link) + + +def test_symlinked_original_archive_refuses(acs): + real = acs.tmp_path / "real-csv_hus.zip" + real.write_bytes(acs.household_zip.read_bytes()) + acs.household_zip.unlink() + acs.household_zip.symlink_to(real) + refuses(acs, acs.produce) + + +def test_fifo_original_archive_refuses(acs): + acs.household_zip.unlink() + os.mkfifo(acs.household_zip) + with bounded_time(): + refuses(acs, acs.produce) + + +def test_directory_in_place_of_an_original_archive_refuses(acs): + acs.household_zip.unlink() + acs.household_zip.mkdir() + refuses(acs, acs.produce) + + +def test_pinned_digest_mismatch_at_the_same_size_refuses(acs): + """Authentication is over the whole archive bytes, not over the projection.""" + pins = acs.pin() + flip_archive_byte(acs.household_zip) + acs.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + refuses(acs, acs.produce) + + +def test_pinned_size_mismatch_refuses(acs): + pins = acs.pin() + with acs.household_zip.open("ab") as handle: + handle.write(b"\n") + acs.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + refuses(acs, acs.produce) + + +def test_original_archives_are_never_modified(acs): + before = { + path: (path.read_bytes(), path.stat().st_mtime_ns) + for path in (acs.household_zip, acs.person_zip) + } + acs.produce() + for path, (payload, mtime) in before.items(): + assert path.read_bytes() == payload + assert path.stat().st_mtime_ns == mtime + + +def test_capture_writes_only_under_the_supplied_snapshot_root(acs): + assert not any(acs.snapshot_root.iterdir()) + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + children = list(acs.snapshot_root.iterdir()) + assert children, "the private capture did not land under the snapshot root" + for child in children: + assert child.is_dir() and not child.is_symlink() + assert stat.S_IMODE(child.stat().st_mode) == 0o700 + assert sorted(path.name for path in output_dir.iterdir()) == sorted( + (NAMES["projection_filename"], NAMES["receipt_filename"]) + ) + + +def test_capture_never_falls_back_to_system_temporary_storage(acs, monkeypatch): + absent = acs.tmp_path / "no-such-tmpdir" + monkeypatch.setenv("TMPDIR", str(absent)) + monkeypatch.setattr(tempfile, "tempdir", str(absent)) + result = acs.produce() + assert len(result.households) == len(BASE_HOUSEHOLDS) + assert not absent.exists() + + +def test_two_produces_share_one_snapshot_root_without_collision(acs): + first = acs.produce(output_dir=acs.output_dir("first")) + second = acs.produce(output_dir=acs.output_dir("second")) + assert first.projection_json == second.projection_json + assert len(list(acs.snapshot_root.iterdir())) >= 2 + + +def test_a_refused_produce_leaves_no_receipt_behind(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, monkeypatch, households=with_household(0, TEN="5") + ) + output_dir = fixture.output_dir() + refuses(fixture, fixture.produce, output_dir=output_dir) + assert not (output_dir / NAMES["receipt_filename"]).exists() + + +def byte_cap_names(): + return tuple(sorted(name for name in dir(source) if name.endswith("_MAX_BYTES"))) + + +def test_module_declares_its_own_byte_caps(): + assert byte_cap_names(), "the module declares no explicit byte caps" + + +@pytest.mark.parametrize("cap", [0, 1, 64]) +def test_tiny_byte_caps_refuse_before_any_payload_is_returned(acs, cap): + """Exercise the streaming bounds with mocked caps, never with real GiBs.""" + for name in byte_cap_names(): + acs.monkeypatch.setattr(source, name, cap) + refuses(acs, acs.produce) + + +def test_declared_cap_values_match_the_approved_plan(): + """Reconciliation: the plan fixes both ACS source caps by name and value.""" + assert getattr(source, NAMES["source_max_bytes"]) == 8 * 1024**3 + assert getattr(source, NAMES["receipt_max_bytes"]) == 1024**2 + + +# -------------------------------------------------------------------------- +# Artifact custody: readback reconstructs, it never trusts what it decoded. +# -------------------------------------------------------------------------- + + +def make_writable(path): + """The producer publishes read-only files; tampering needs write access.""" + path = Path(path) + path.chmod(stat.S_IMODE(path.stat().st_mode) | 0o600) + return path + + +def forge(output_dir, old, new): + """Rewrite the artifact so its own internal digests stay self-consistent.""" + projection = (output_dir / NAMES["projection_filename"]).read_bytes() + receipt = (output_dir / NAMES["receipt_filename"]).read_bytes() + assert old in projection, "the invented forgery target is not in the projection" + forged = projection.replace(old, new) + assert len(forged) == len(projection), "a forgery must not change the length" + old_digest = hashlib.sha256(projection).hexdigest().encode() + new_digest = hashlib.sha256(forged).hexdigest().encode() + make_writable(output_dir / NAMES["projection_filename"]).write_bytes(forged) + make_writable(output_dir / NAMES["receipt_filename"]).write_bytes( + receipt.replace(old_digest, new_digest) + ) + return old_digest in receipt + + +def test_rehashed_projection_forgery_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + forge(output_dir, b"2024HU0000002", b"2024HU0000012") + refuses(acs, acs.load, output_dir) + + +def test_receipt_binds_the_projection_digest(acs): + """Otherwise a forged projection would be self-consistent on its face.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + assert forge(output_dir, b"2024HU0000002", b"2024HU0000012") + + +def test_forging_a_weight_to_another_valid_value_refuses(acs): + """Custody is byte reconstruction, so an in-domain forgery refuses too.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + forge(output_dir, b"118", b"119") + refuses(acs, acs.load, output_dir) + + +def test_rewritten_receipt_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + receipt = json.loads((output_dir / NAMES["receipt_filename"]).read_bytes()) + forged = json.dumps(receipt, sort_keys=True).encode() + make_writable(output_dir / NAMES["receipt_filename"]).write_bytes(forged) + refuses(acs, acs.load, output_dir) + + +@pytest.mark.parametrize( + "filename", [NAMES["projection_filename"], NAMES["receipt_filename"]] +) +def test_truncated_or_missing_artifact_files_refuse(acs, filename): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + make_writable(output_dir / filename).write_bytes( + (output_dir / filename).read_bytes()[:20] + ) + refuses(acs, acs.load, output_dir) + + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + (output_dir / filename).unlink() + refuses(acs, acs.load, output_dir) + + +def test_missing_artifact_directory_refuses(acs): + refuses(acs, acs.load, acs.tmp_path / "absent-artifact") + + +def test_extra_file_in_the_artifact_directory_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + (output_dir / "notes.json").write_bytes(b"{}\n") + refuses(acs, acs.load, output_dir) + + +def test_parent_change_between_produce_and_readback_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + flip_archive_byte(acs.household_zip) + refuses(acs, acs.load, output_dir) + + +def test_reparented_readback_refuses_even_after_repinning(acs): + """Fresh authentic parents that no longer reconstruct the artifact refuse.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + rows = with_household(1, TEN="4") + write_archive( + acs.household_zip, + ( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, rows[:5])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, rows[5:])), + ), + ) + acs.pin() + refuses(acs, acs.load, output_dir) + + +def test_readback_against_swapped_role_archives_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + household = acs.household_zip.read_bytes() + acs.household_zip.write_bytes(acs.person_zip.read_bytes()) + acs.person_zip.write_bytes(household) + acs.pin() + refuses(acs, acs.load, output_dir) + + +def test_readback_from_an_unrelated_artifact_refuses(tmp_path, monkeypatch): + other = build_fixture( + tmp_path / "other", + monkeypatch, + households=(("psam_husa.csv", BASE_HOUSEHOLDS[:4]),), + persons=(("psam_pusa.csv", BASE_PERSONS[:6]),), + ) + output_dir = other.output_dir() + other.produce(output_dir=output_dir) + + mine = build_fixture(tmp_path / "mine", monkeypatch) + refuses(mine, mine.load, output_dir) + + +@pytest.mark.parametrize("attribute", ["households", "persons", "codes"]) +def test_readback_frames_are_owned_copies(acs, attribute): + _produced, loaded, _output_dir = acs.produce_and_load() + first = getattr(loaded, attribute) + assert getattr(loaded, attribute) is not first + before = first.copy(deep=True) + first.iloc[0, 0] = first.iloc[-1, 0] + pd.testing.assert_frame_equal(getattr(loaded, attribute), before) + + +# -------------------------------------------------------------------------- +# Refusal semantics: sanitised, and one code per cause. +# -------------------------------------------------------------------------- + + +def collect_family_reasons(tmp_path, monkeypatch): + """One refusal per coarse family, each from its own invented source.""" + reasons = {} + + def record(family, build, call): + fixture = build(tmp_path / family, monkeypatch) + fixture.pin() + reasons[family] = refuses(fixture, call, fixture) + + def digest_change(fixture): + pins = fixture.pin() + flip_archive_byte(fixture.household_zip) + fixture.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + fixture.produce() + + def forged_artifact(fixture): + output_dir = fixture.output_dir() + fixture.produce(output_dir=output_dir) + forge(output_dir, b"2024HU0000002", b"2024HU0000012") + fixture.load(output_dir) + + def existing_output(fixture): + output_dir = fixture.output_dir() + output_dir.mkdir() + fixture.produce(output_dir=output_dir) + + plain = build_fixture + + record("capture_digest", plain, digest_change) + record("artifact_forgery", plain, forged_artifact) + record("output_precondition", plain, existing_output) + record("selection", plain, lambda f: f.produce(serialnos=("2024HU0000009",))) + record( + "zip_preflight", + lambda base, patch: household_archive_fixture( + base, + patch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + ), + ), + lambda f: f.produce(), + ) + record( + "csv_record", + lambda base, patch: household_bytes_fixture( + base, + patch, + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0][:-1]).encode() + + b"\n", + ), + lambda f: f.produce(), + ) + record( + "header_contract", + lambda base, patch: household_bytes_fixture( + base, + patch, + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "ST")), + ), + lambda f: f.produce(), + ) + record( + "value_domain", + lambda base, patch: single_member( + base, patch, households=with_household(0, TEN="5") + ), + lambda f: f.produce(), + ) + record( + "join_integrity", + lambda base, patch: single_member( + base, + patch, + persons=BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + ), + lambda f: f.produce(), + ) + return reasons + + +@pytest.fixture +def family_reasons(tmp_path, monkeypatch): + return collect_family_reasons(tmp_path, monkeypatch) + + +def test_refusal_type_is_a_sanitised_value_error(): + assert issubclass(source.ACSHousingSourceError, ValueError) + + +def test_every_refusal_family_is_exercised(family_reasons): + assert set(family_reasons) == set(DISTINCT_FAMILIES) + + +def test_distinct_causes_carry_distinct_reason_codes(family_reasons): + """A single catch-all code would make every refusal above unfalsifiable.""" + collisions = {} + for family, reason in family_reasons.items(): + collisions.setdefault(reason, []).append(family) + shared = {reason: names for reason, names in collisions.items() if len(names) > 1} + assert not shared, f"distinct causes share a reason code: {shared}" + + +def test_no_refusal_reason_echoes_invented_source_content(family_reasons): + """refuses() checks each call; this states the whole-sweep contract once.""" + forbidden = set(BASE_SERIALNOS) | {"psam_husa.csv", "82000", "52000"} + for family, reason in family_reasons.items(): + for secret in forbidden: + assert secret not in reason, f"{family} leaked {secret!r}" + + +def test_reason_codes_match_the_agreed_table(family_reasons): + """Shape always; exact spellings once EXPECTED_REASONS is filled in. + + The house idiom in the committed sibling source modules is a bare + UPPER_SNAKE token, so that much is asserted unconditionally and nothing is + parked behind a skip. Only the agreement on which token names which cause + waits on the production author. + """ + for family, reason in family_reasons.items(): + assert re.fullmatch(r"[A-Z][A-Z0-9_]*", reason), ( + f"{family} reason {reason!r} is not a bare reason code" + ) + if REASON_CODES_AGREED: + assert family_reasons == EXPECTED_REASONS + else: + assert not EXPECTED_REASONS, ( + "EXPECTED_REASONS is populated; set REASON_CODES_AGREED = True" + ) + + +# -------------------------------------------------------------------------- +# Naming reconciliation: everything this file had to guess, in one place. +# -------------------------------------------------------------------------- + + +def test_module_exposes_the_agreed_public_surface(): + for name in ( + "ACSHousingSourceError", + "produce_acs_housing_source", + "load_acs_housing_source", + "prepare_acs_housing_population", + ): + assert hasattr(source, name), f"missing agreed API name {name}" + + +def test_archive_pins_are_consulted_at_call_time(acs): + """The whole suite rests on this seam, so assert it directly. + + Every fixture authenticates its invented archives by pointing the private + pin tuple at them. That only works if the module reads the attribute when + it captures, rather than copying it at import. + """ + real = acs.pin() + assert acs.produce().households is not None + + wrong = tuple((role, name, "f" * 64, size) for role, name, _sha, size in real) + acs.monkeypatch.setattr(source, NAMES["archive_pins"], wrong) + refuses(acs, acs.produce) + + +def test_private_archive_pins_are_a_sentinel_or_the_two_fixed_archives(): + """Naming seam, reconciled with the production author, not a defect. + + The brief described `_ARCHIVE_PINS` as a populated tuple of + (role, filename, sha256, size). The module instead defaults it to None and + loads the packaged manifest lazily, while tests still patch the same + private name - which is what + ``test_archive_pins_are_consulted_at_call_time`` proves still works. Both + readings are admitted here; the shape is checked whenever it is populated, + so a wrong shape cannot hide behind the sentinel. + """ + pins = getattr(source, NAMES["archive_pins"]) + if pins is None: + return + assert type(pins) is tuple and len(pins) == 2 + assert [entry[0] for entry in pins] == [ + NAMES["household_role"], + NAMES["person_role"], + ] + assert [entry[1] for entry in pins] == [HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE] + for _role, _name, sha, size in pins: + assert type(sha) is str and len(sha) == 64 and sha == sha.lower() + assert int(sha, 16) >= 0 + assert type(size) is int and size > 0 + + +def test_lineage_and_state_columns_use_the_agreed_names(acs): + households = acs.produce().households + assert NAMES["member_column"] in households.columns + assert NAMES["ordinal_column"] in households.columns + assert NAMES["state_column"] in households.columns + + +# -------------------------------------------------------------------------- +# Tail coverage: cases an obvious pass over the five dimensions omits. +# -------------------------------------------------------------------------- + + +def set_encrypted_flag(path): + """Mark every entry encrypted in place; zipfile itself will not write this.""" + raw = bytearray(Path(path).read_bytes()) + marked = 0 + for signature, offset in ((b"PK\x03\x04", 6), (b"PK\x01\x02", 8)): + index = 0 + while True: + index = raw.find(signature, index) + if index < 0: + break + flag = struct.unpack_from("= 2 + Path(path).write_bytes(bytes(raw)) + + +def raw_household_record(**overrides): + """A physical household line written by hand, bypassing the csv writer.""" + values = dict(zip(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[0], strict=True)) + values.update(overrides) + return ",".join(values[name] for name in HOUSEHOLD_HEADER).encode() + + +def appended_record(record): + return csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + record + b"\n" + + +def test_encrypted_member_refuses(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, monkeypatch, (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + ) + set_encrypted_flag(fixture.household_zip) + fixture.pin() + refuses(fixture, fixture.produce) + + +def test_lzma_member_refuses(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_LZMA), + ), + ) + refuses(fixture, fixture.produce) + + +def test_bytes_that_are_not_an_archive_refuse(acs): + acs.household_zip.write_bytes(b"this is invented, and it is not a zip archive\n") + acs.pin() + refuses(acs, acs.produce) + + +def test_person_archive_holding_household_members_refuses(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + person_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + ) + refuses(fixture, fixture.produce) + + +UTF16_MEMBER = BASE_HOUSEHOLD_CSV.decode().encode("utf-16") + +TAIL_PARSE_VARIANTS = [ + pytest.param(UTF16_MEMBER, id="utf16_encoded_member"), + pytest.param( + appended_record(raw_household_record(ST='"06"x')), + id="character_after_a_closing_quote", + ), + pytest.param( + appended_record(raw_household_record(HINCP='"82000')), + id="unterminated_quote", + ), + pytest.param( + splice_line(BASE_HOUSEHOLD_CSV, 3, b" "), id="whitespace_only_record" + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, TEN="NA")), + id="na_token_is_not_a_missing_tenure", + ), + pytest.param( + csv_bytes( + rename_column(HOUSEHOLD_HEADER, "SERIALNO", "serialno"), BASE_HOUSEHOLDS + ), + id="lowercase_roster_column_name", + ), +] + + +@pytest.mark.parametrize("payload", TAIL_PARSE_VARIANTS) +def test_tail_parser_refusals(tmp_path, monkeypatch, payload): + fixture = household_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +def test_household_rows_absent_while_person_rows_remain_refuses(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, ())),), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_a_final_record_without_a_trailing_newline_is_accepted(tmp_path, monkeypatch): + fixture = household_bytes_fixture( + tmp_path, + monkeypatch, + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, trailing_newline=False), + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_many_unprojected_columns_are_accepted(tmp_path, monkeypatch): + """Genuine PUMS members carry hundreds of columns this projection ignores.""" + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(WIDE_HEADER, WIDE_ROWS) + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_roster_columns_in_any_header_position_are_accepted(tmp_path, monkeypatch): + """The plan pins header order across members of a role, not within one.""" + reversed_header = tuple(reversed(HOUSEHOLD_HEADER)) + rows = tuple(tuple(reversed(row)) for row in BASE_HOUSEHOLDS) + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(reversed_header, rows) + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_maximum_serialno_suffix_is_accepted(tmp_path, monkeypatch): + households, persons = rename_serialno("2024HU0000001", "2024HU9999999") + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + keys = household_keys(fixture.produce().households) + assert "2024HU9999999" in keys and "2024HU0000001" not in keys + + +def test_np_at_the_published_upper_bound_is_accepted(tmp_path, monkeypatch): + households = (household_row("2024HU0000020", "1", "20", "1", "300"),) + persons = tuple( + person_row("2024HU0000020", str(order), str(order + 10)) + for order in range(1, 21) + ) + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + result = fixture.produce() + assert lexical(result.households["NP"]) == ["20"] + assert len(result.persons) == 20 + assert result.codes["occupied_hu"].tolist() == [1] + + +def test_an_all_group_quarters_source_is_retained(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, + monkeypatch, + households=BASE_HOUSEHOLDS[4:6], + persons=BASE_PERSONS[6:8], + ) + result = fixture.produce() + assert codes_map(result) == { + key: EXPECTED_CODES[key] for key in ("2024GQ0000005", "2024GQ0000006") + } + + +def test_a_vacancy_only_source_is_retained(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:4])), + ), + person_members=(Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())),), + ) + result = fixture.produce() + assert household_keys(result.households) == ["2024HU0000004"] + assert len(result.persons) == 0 + assert codes_map(result) == {"2024HU0000004": EXPECTED_CODES["2024HU0000004"]} + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param((1,), id="integer_element"), + pytest.param((None,), id="none_element"), + pytest.param((b"2024HU0000001",), id="bytes_element"), + ], +) +def test_non_string_selection_elements_refuse(acs, serialnos): + refuses(acs, acs.produce, serialnos=serialnos) + + +def test_an_explicit_full_selection_matches_the_none_selection(acs): + whole = acs.produce(output_dir=acs.output_dir("whole")) + explicit = acs.produce( + output_dir=acs.output_dir("explicit"), serialnos=BASE_SERIALNOS + ) + assert explicit.projection_json == whole.projection_json + + +def test_a_published_artifact_survives_a_later_refused_produce(acs): + output_dir = acs.output_dir("kept") + produced = acs.produce(output_dir=output_dir) + before = {path.name: path.read_bytes() for path in sorted(output_dir.iterdir())} + acs.household_zip.write_bytes(b"not an archive\n") + acs.pin() + refuses(acs, acs.produce, output_dir=acs.output_dir("doomed")) + assert {path.name: path.read_bytes() for path in sorted(output_dir.iterdir())} == ( + before + ) + assert before[NAMES["projection_filename"]].rstrip(b"\n") == ( + produced.projection_json.rstrip(b"\n") + ) + + +def test_readback_with_a_missing_snapshot_root_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + refuses( + acs, + source.load_acs_housing_source, + acs.source_dir, + output_dir, + snapshot_root=acs.tmp_path / "absent-root", + ) + + +def test_wrong_size_candidate_refuses_as_a_candidate_not_as_a_parent(acs): + """Readback size failures must name the candidate, not the pinned archives.""" + for filename, mutate in ( + (NAMES["projection_filename"], lambda raw: raw + b" "), + (NAMES["projection_filename"], lambda raw: raw[:20]), + (NAMES["receipt_filename"], lambda raw: raw + b" "), + (NAMES["receipt_filename"], lambda raw: raw[:20]), + ): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + path = output_dir / filename + make_writable(path).write_bytes(mutate(path.read_bytes())) + assert refuses(acs, acs.load, output_dir) == "RECONSTRUCTION_SIZE" + + +def test_parent_archive_size_failures_keep_their_own_reason_codes(acs): + """The candidate code must not be borrowed by, or borrow from, the parents.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + intact = acs.household_zip.read_bytes() + acs.household_zip.write_bytes(intact[:-1]) + assert refuses(acs, acs.load, output_dir) == "SOURCE_SIZE" + acs.household_zip.write_bytes(intact + b"\x00") + assert refuses(acs, acs.load, output_dir) == "FILE_TOO_LARGE" + acs.household_zip.write_bytes(intact) + assert acs.load(output_dir) is not None + + +def test_failure_record_write_failure_does_not_mask_the_original_refusal( + acs, monkeypatch +): + """A failed failure record must never replace the refusal it was recording.""" + actual_write = source._write + attempted = [] + + def refuse_the_failure_record(path, data, cap): + if Path(path).name == "failure.json": + attempted.append(Path(path).name) + raise OSError(28, "No space left on device") + return actual_write(path, data, cap) + + monkeypatch.setattr(source, "_write", refuse_the_failure_record) + corrupted = bytearray(acs.household_zip.read_bytes()) + corrupted[-1] ^= 0xFF + acs.household_zip.write_bytes(bytes(corrupted)) + assert refuses(acs, acs.produce) == "SOURCE_SHA256" + assert attempted == ["failure.json"] diff --git a/packages/microcosm-build/tests/test_us_acs_multispine.py b/packages/microcosm-build/tests/test_us_acs_multispine.py index afc38b72d..fbaa393b7 100644 --- a/packages/microcosm-build/tests/test_us_acs_multispine.py +++ b/packages/microcosm-build/tests/test_us_acs_multispine.py @@ -356,15 +356,25 @@ def _test_puma_ladder() -> UsPumaLadder: tract_overlap_puma=puma.copy(), tract_overlap_tract=np.asarray([6_001_000_100, 36_001_000_100], dtype=np.int64), tract_overlap_population=population.copy(), + joint_overlap_puma=puma.copy(), + joint_overlap_tract=np.asarray([6_001_000_100, 36_001_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([601, 3_601], dtype=np.int64), + joint_overlap_population=np.asarray([100, 200], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py b/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py index 31742ad61..47e423dcb 100644 --- a/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py +++ b/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py @@ -193,15 +193,25 @@ def test_main_wires_verified_sources_transfer_audit_export_and_summary( tract_overlap_puma=np.asarray([100_100], dtype=np.int64), tract_overlap_tract=np.asarray([1_001_000_100], dtype=np.int64), tract_overlap_population=np.asarray([100.0]), + joint_overlap_puma=np.asarray([100_100], dtype=np.int64), + joint_overlap_tract=np.asarray([1_001_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([101], dtype=np.int64), + joint_overlap_population=np.asarray([100], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py b/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py new file mode 100644 index 000000000..b9d3c2ee4 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py @@ -0,0 +1,803 @@ +"""Invented-only source authority and explicitly ungranted native consistency.""" + +import csv +import hashlib +import importlib.util +import io +import json +import os +import sys +from dataclasses import FrozenInstanceError, replace +from pathlib import Path +from types import SimpleNamespace + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import acs_housing_universe_source as custody +from microcosm.build.us_runtime import acs_native_coverage_binding as binding +from microcosm.build.us_runtime import acs_person_coverage_authentication as owner +from microcosm.build.us_runtime import acs_person_coverage_columns as literal +from microcosm.build.us_runtime.acs_inputs import map_acs_native_inputs +from microcosm.build.us_runtime.acs_pums import AcsPumsSource, build_acs_pums_unit_frame + + +def _helper(filename): + spec = importlib.util.spec_from_file_location( + "coverage_helpers_" + filename, Path(__file__).with_name(filename + ".py") + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +_authority = _helper("test_us_acs_housing_source") +Member = _authority.Member +build_fixture = _authority.build_fixture +flip_archive_byte = _authority.flip_archive_byte +write_archive = _authority.write_archive +_native = _helper("test_us_acs_pums") +_household, _person = _native._household, _native._person + + +def _csv(rows, *, bom=False, ending="\n"): + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=list(rows[0]), quoting=csv.QUOTE_ALL, lineterminator=ending + ) + writer.writeheader() + writer.writerows(rows) + return (("\ufeff" if bom else "") + stream.getvalue()).encode() + + +@pytest.fixture +def invented(tmp_path, monkeypatch): + monkeypatch.setattr( + custody.shutil, "disk_usage", lambda _p: SimpleNamespace(free=64 * 1024**3) + ) + h = [_household("2024HU0000001", NP=2), _household("2024HU0000002", NP=1)] + p = [ + _person("2024HU0000001", 1, 20, AGEP=30, MIL="1", ESR="4"), + _person("2024HU0000001", 2, 25, AGEP=15, MIL="", ESR=""), + _person("2024HU0000002", 1, 20, AGEP=80, MIL="4", ESR="6"), + ] + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", _csv(h)),), + person_members=( + Member("psam_pusa.csv", _csv(p[:1])), + Member("psam_pusb.csv", _csv(p[1:])), + ), + ) + frame = map_acs_native_inputs( + build_acs_pums_unit_frame( + AcsPumsSource(fixture.household_zip, fixture.person_zip) + )[0] + ).frame + return fixture, frame, h, p + + +def _load(invented, **kwargs): + fixture, frame, _h, _p = invented + return owner.load_authenticated_acs_person_coverage( + fixture.source_dir, snapshot_root=fixture.snapshot_root, frame=frame, **kwargs + ) + + +def _rewrite( + invented, *, persons=None, households=None, members=None, bom=False, ending="\n" +): + fixture, _frame, h, p = invented + if households is not None: + write_archive( + fixture.household_zip, (Member("psam_husa.csv", _csv(households)),) + ) + if persons is not None or members is not None: + p = persons if persons is not None else p + write_archive( + fixture.person_zip, + members + if members is not None + else ( + Member("psam_pusa.csv", _csv(p[:1], bom=bom, ending=ending)), + Member("psam_pusb.csv", _csv(p[1:], bom=bom, ending=ending)), + ), + ) + fixture.pin() + + +def _refuses(code): + return pytest.raises(owner.ACSCoverageAuthenticationError, match="^" + code + "$") + + +def test_source_authentication_preserves_low_receipt_and_parent_without_population_grant( + invented, +): + fixture, frame, _h, _p = invented + before = custody.frame_content_sha256(frame) + original_tables = {e: frame.table(e).copy(deep=True) for e in frame.entities} + result = _load(invented) + table, receipt = literal.read_acs_person_coverage_columns( + AcsPumsSource(fixture.household_zip, fixture.person_zip), + person_keys=owner._native_roster(frame)[0], + chunksize=3, + ) + pd.testing.assert_frame_equal(result.table, table) + assert result.receipt["original_literal_receipt"] == receipt + assert receipt["source_authenticated"] is False + assert result.receipt["source_authenticated"] is True + assert result.native_binding.receipt["population_binding_authenticated"] is False + assert ( + result.native_binding.receipt["missing_authority"] + == "closed_prepared_acs_native_population" + ) + assert ( + result.native_binding.receipt["original_age"]["relation"] == "numeric_identity" + ) + for flag in ( + "domain_assignment_authenticated", + "period_harmonized", + "release_eligible", + "cross_survey_coverage_equivalence_established", + "population_binding_authenticated", + ): + assert result.receipt[flag] is False + assert custody.frame_content_sha256(frame) == before + for entity, original in original_tables.items(): + pd.testing.assert_frame_equal(frame.table(entity), original) + assert ( + owner.verify_acs_coverage_native_consistency(result, frame) + == result.native_binding + ) + assert [m["rows"] for m in result.receipt["members"]["person"]] == [1, 2] + assert all(len(m["sha256"]) == 64 for m in result.receipt["members"]["person"]) + + +def test_first_frame_hash_precedes_the_roster_snapshot(invented, monkeypatch): + """The returned binding must describe one state, including at hash entry.""" + _fixture, frame, _h, _p = invented + original_hash = custody.frame_content_sha256 + first = True + + def at_hash_entry(value): + nonlocal first + if first: + first = False + value.person.loc[:, "source_row_id"] += 100 + return original_hash(value) + + monkeypatch.setattr(custody, "frame_content_sha256", at_hash_entry) + result = _load(invented) + assert ( + owner.verify_acs_coverage_native_consistency(result, frame) + == result.native_binding + ) + + +def test_mutation_during_roster_snapshot_refuses_issuance(invented, monkeypatch): + original_roster = owner._native_roster + first = True + + def after_roster_read(value): + nonlocal first + result = original_roster(value) + if first: + first = False + value.person.loc[:, "source_row_id"] += 100 + return result + + monkeypatch.setattr(owner, "_native_roster", after_roster_read) + with _refuses("NATIVE_FRAME_CHANGED"): + _load(invented) + + +def test_closed_default_manifest_has_no_caller_pin_or_prepared_authority( + invented, monkeypatch +): + fixture, frame, _h, _p = invented + pins = custody._ARCHIVE_PINS + monkeypatch.setattr(custody, "_ARCHIVE_PINS", None) + calls = [] + + def default_manifest(*args, **kwargs): + assert not args and not kwargs + calls.append(True) + return SimpleNamespace( + artifacts=tuple( + SimpleNamespace(role=r, filename=n, sha256=d, size_bytes=s) + for r, n, d, s in pins + ) + ) + + monkeypatch.setattr(custody, "load_acs_source_manifest", default_manifest) + result = _load(invented) + assert len(calls) == 2 + with pytest.raises(TypeError): + _load(invented, manifest={"sha256": "0" * 64}) + for constructor, payload in ( + (owner.AuthenticatedACSPersonCoverage, result.payload), + (owner.UngrantedACSNativeBinding, result.native_binding.receipt_json), + ): + with _refuses("SOURCE_CONSTRUCTOR|BINDING_CONSTRUCTOR"): + constructor(payload) + for forged in ( + frame.person, + {"frame": frame, "frame_sha256": custody.frame_content_sha256(frame)}, + "0" * 64, + ): + with _refuses("NATIVE_FRAME_REQUIRED"): + owner.load_authenticated_acs_person_coverage( + fixture.source_dir, snapshot_root=fixture.snapshot_root, frame=forged + ) + + +@pytest.mark.parametrize( + "field,value,code", + [ + ("SPORDER", 3, "NATIVE_ALIASES"), + ("A_LINENO", 3, "NATIVE_ALIASES"), + ("source_person_id", "2", "NATIVE_ALIASES"), + ("source_household_id", 2, "NATIVE_ALIASES"), + ("source_year", 2023, "NATIVE_VINTAGE"), + ("person_household_id", 2, "NATIVE_ALIASES"), + ("source_row_id", 1, "NATIVE_IDS"), + ], +) +def test_native_alias_and_membership_changes_with_same_person_ids_refuse( + invented, field, value, code +): + invented[1].person.loc[0, field] = value + with _refuses(code): + _load(invented) + + +@pytest.mark.parametrize( + "change", ["serial", "age", "membership", "weight", "source_row"] +) +def test_live_binding_refuses_same_ids_changed_native_parent(invented, change): + _fixture, frame, _h, _p = invented + result = _load(invented) + if change == "serial": + h = frame.table("household") + h["SERIALNO"] = h.SERIALNO.iloc[::-1].to_numpy() + elif change == "age": + frame.person.loc[2, "age"] = 82 + elif change == "membership": + for c in ("person_household_id", "source_household_id"): + frame.person.loc[[1, 2], c] = frame.person.loc[[2, 1], c].to_numpy() + elif change == "weight": + frame.table("household")["extra_cell"] = ( + 123 # all parent cells bind, including additions + ) + else: + frame.person.loc[0, "source_row_id"] = 100 + with _refuses( + "NATIVE_FRAME_CHANGED|NATIVE_HOUSEHOLD_COUNT|NATIVE_CONSISTENCY_REFUSED" + ): + owner.verify_acs_coverage_native_consistency(result, frame) + + +@pytest.mark.parametrize( + "change", + [ + "missing", + "extra", + "duplicate", + "household_np", + "household_missing", + "wrong_key", + "wrong_vintage", + ], +) +def test_source_complete_household_mismatch_refuses(invented, change): + _fixture, _frame, h, p = invented + if change == "missing": + p = p[:2] + elif change == "extra": + p += [{**p[1], "SPORDER": 3}] + elif change == "duplicate": + p[1] = p[0] + elif change == "household_np": + h[0]["NP"] = 1 + elif change == "household_missing": + h = h[:1] + elif change == "wrong_vintage": + p[0]["SERIALNO"] = "2023HU0000001" + else: + p[0]["SPORDER"] = 3 + _rewrite(invented, persons=p, households=h) + with _refuses("SOURCE_PERSON_ROSTER|SOURCE_HOUSEHOLD_ROSTER|SOURCE_DUPLICATE_KEY"): + _load(invented) + + +@pytest.mark.parametrize( + "age,mil,esr", + [ + ("", "", ""), + ("NA", "NA", "1.0"), + ("30 ", " 1", "4 "), + ("30", '1,"quoted"\r\n\t', "4\r\n\t"), + ("30\t", "1", "4"), + ("100", "0", "7"), + ], +) +@pytest.mark.parametrize("ending", ["\n", "\r", "\r\n"]) +def test_bom_multiline_tabs_and_malformed_literals_survive_envelope( + invented, age, mil, esr, ending +): + _fixture, frame, _h, p = invented + p[0].update(AGEP=age, MIL=mil, ESR=esr) + _rewrite(invented, persons=p, bom=True, ending=ending) + result = _load(invented) + assert result.table.loc[0, ["AGEP", "MIL", "ESR"]].tolist() == [age, mil, esr] + assert not result.table.isna().any().any() + assert frame.person.loc[0, "age"] == 30 + if age != "30": + assert result.table.loc[0, "MIL_state"] == "age_unresolved" + assert result.native_binding.receipt["original_age"]["relation"] == "unproven" + + +@pytest.mark.parametrize( + "column,value", [("AGEP", 82), ("A_AGE", 82), ("age", 82), ("age", float("nan"))] +) +def test_model_age_never_fills_or_authenticates_original_relation( + invented, column, value +): + frame = invented[1] + if isinstance(value, float): + frame.person[column] = frame.person[column].astype("float64") + frame.person.loc[2, column] = value + result = _load(invented) + assert result.table.loc[2, "AGEP"] == "80" + assert result.native_binding.receipt["original_age"]["mismatching_rows"] == 1 + assert result.native_binding.receipt["population_binding_authenticated"] is False + + +def test_candidate_reconstructed_before_comparison_and_self_rehash_cannot_authorize( + invented, monkeypatch +): + result = _load(invented) + candidate = invented[0].tmp_path / "candidate.coverage" + candidate.write_bytes(result.payload) + assert _load(invented, candidate_path=candidate).payload == result.payload + header_raw, body = result._parts() + header = json.loads(header_raw) + tampered = body.replace(b'"30","1","4"', b'"30","2","4"', 1) + assert tampered != body + header["body_sha256"] = hashlib.sha256(tampered).hexdigest() + raw = owner._json(header) + candidate.write_bytes(owner._MAGIC + len(raw).to_bytes(4, "big") + raw + tampered) + with _refuses("CANDIDATE_MISMATCH"): + _load(invented, candidate_path=candidate) + flip_archive_byte(invented[0].person_zip) + original_copy = custody._copy + + def guarded_copy(source, destination, *args, **kwargs): + assert source != candidate, ( + "candidate was accessed before source reconstruction" + ) + return original_copy(source, destination, *args, **kwargs) + + monkeypatch.setattr(custody, "_copy", guarded_copy) + with _refuses("SOURCE_RECONSTRUCTION_REFUSED"): + _load(invented, candidate_path=candidate) + + +@pytest.mark.parametrize("replacement", [None, "", "different"]) +def test_candidate_null_empty_and_field_contract_tampering_refuse( + invented, replacement +): + result = _load(invented) + header, body = result._parts() + rows = [json.loads(row) for row in body.splitlines()] + rows[1][3] = replacement + if replacement == "": + metadata = json.loads(header) + metadata["field_contract"]["fields"]["MIL"]["minimum_age"] = 16 + else: + metadata = json.loads(header) + changed_body = b"".join(owner._json(row) + b"\n" for row in rows) + metadata.update( + body_sha256=hashlib.sha256(changed_body).hexdigest(), + body_bytes=len(changed_body), + ) + changed_header = owner._json(metadata) + candidate = invented[0].tmp_path / "candidate.coverage" + candidate.write_bytes( + owner._MAGIC + + len(changed_header).to_bytes(4, "big") + + changed_header + + changed_body + ) + with _refuses("CANDIDATE_MISMATCH|SOURCE_RECONSTRUCTION_REFUSED"): + _load(invented, candidate_path=candidate) + + +def test_views_and_closed_objects_cannot_mutate_owned_evidence(invented): + result = _load(invented) + before = result.payload + view = result.table + view.loc[0, "MIL"] = "2" + result.receipt["field_contract"].clear() + result.native_binding.receipt["population_binding_authenticated"] = True + assert result.payload == before + assert result.table.loc[0, "MIL"] == "1" + assert result.native_binding.receipt["population_binding_authenticated"] is False + with pytest.raises(FrozenInstanceError): + result.payload = b"forged" + with _refuses("SOURCE_CONSTRUCTOR"): + replace(result, payload=b"forged") + + +@pytest.mark.parametrize( + "kind", ["duplicate", "case_duplicate", "unsafe", "symlink", "prefix_non_csv"] +) +def test_entire_zip_directory_is_checked_before_literal_reader( + invented, monkeypatch, kind +): + p = invented[3] + extra = { + "duplicate": Member("psam_pusa.csv", _csv(p)), + "case_duplicate": Member("PSAM_PUSA.CSV", _csv(p)), + "unsafe": Member("../unrelated.txt", b"ignored"), + "symlink": Member("unrelated.txt", b"target", external_attr=0o120777 << 16), + "prefix_non_csv": Member("psam_pus.txt", b"ignored"), + }[kind] + if kind == "duplicate": + with pytest.warns(UserWarning, match="Duplicate name"): + _rewrite(invented, members=(Member("psam_pusa.csv", _csv(p)), extra)) + else: + _rewrite(invented, members=(Member("psam_pusa.csv", _csv(p)), extra)) + monkeypatch.setattr( + literal, + "read_acs_person_coverage_columns", + lambda *a, **k: pytest.fail("reader entered"), + ) + with _refuses( + "ZIP_DUPLICATE_MEMBER|ZIP_MEMBER_PATH|ZIP_MEMBER_TYPE|ZIP_PREFIX_NONCSV" + ): + _load(invented) + + +@pytest.mark.parametrize( + "target,constant,value,code", + [ + (custody, "_MEMBER_COUNT_MAX", 1, "ZIP_MEMBER_COUNT"), + (custody, "_MEMBER_MAX", 100, "ZIP_MEMBER_SIZE"), + (custody, "_EXPANDED_MAX", 100, "ZIP_EXPANDED_SIZE"), + (owner, "MAX_CSV_HEADER_BYTES", 30, "CSV_RECORD_BYTES"), + (owner, "MAX_RECORD_BYTES", 64, "CSV_RECORD_BYTES"), + (owner, "MAX_TOKEN_BYTES", 10, "CSV_TOKEN_BYTES"), + (owner, "MAX_ROSTER_BYTES", 100, "SELECTED_BODY_BUDGET"), + (owner, "MAX_BODY_BYTES", 40, "BODY_SIZE"), + (owner, "MAX_HEADER_BYTES", 100, "CANONICAL_SIZE"), + (literal, "MAX_ROWS", 1, "SOURCE_ROWS"), + (literal, "MAX_SELECTED_ROWS", 2, "NATIVE_ROWS"), + ], +) +def test_small_limits_refuse_before_relevant_allocation( + invented, monkeypatch, target, constant, value, code +): + monkeypatch.setattr(target, constant, value) + with _refuses(code): + _load(invented) + + +@pytest.mark.parametrize("kind", ["corrupt", "growth", "symlink", "fifo"]) +def test_exact_capture_refusals_are_static_and_carry_their_cause(invented, kind): + """The code is the whole message; the exception it caught is chained. + + A refused run names what refused. The retention lane recovered a 1/15 + refusal's cause only with a sys.monitoring observer because three nested + catch-alls each discarded it (docs/us-native-byte-transports.md). + """ + path = invented[0].person_zip + if kind == "corrupt": + flip_archive_byte(path) + elif kind == "growth": + with path.open("ab") as stream: + stream.write(b"extra") + else: + saved = path.with_suffix(".saved") + path.rename(saved) + if kind == "symlink": + path.symlink_to(saved) + else: + os.mkfifo(path) + # Keep directory roster exact; saved bytes live outside the source dir. + saved.rename(invented[0].tmp_path / "saved.zip") + with _refuses("SOURCE_RECONSTRUCTION_REFUSED") as exc: + _load(invented) + assert str(exc.value) == "SOURCE_RECONSTRUCTION_REFUSED" + assert isinstance(exc.value.__cause__, Exception) + assert not isinstance(exc.value.__cause__, owner.ACSCoverageAuthenticationError) + + +@pytest.mark.parametrize( + "race", ["private_bytes", "original_replace", "pins", "producer", "frame"] +) +def test_races_refuse_before_issuance(invented, monkeypatch, race): + original = literal.read_acs_person_coverage_columns + + def raced(source, **kwargs): + result = original(source, **kwargs) + if race == "private_bytes": + source.person_zip.chmod(0o600) + flip_archive_byte(source.person_zip) + elif race == "original_replace": + # Replace during _copy instead, below. + pass + elif race == "pins": + monkeypatch.setattr(custody, "_ARCHIVE_PINS", ()) + elif race == "producer": + monkeypatch.setattr(owner, "MAX_TOKEN_BYTES", owner.MAX_TOKEN_BYTES - 1) + else: + invented[1].person.loc[0, "age"] = 31 + return result + + if race == "original_replace": + old_identity = custody._identity + changed = False + + def replaced_identity(value): + nonlocal changed + if not changed: + changed = True + path = invented[0].household_zip + raw = path.read_bytes() + path.unlink() + path.write_bytes(raw) + return old_identity(value) + + monkeypatch.setattr(custody, "_identity", replaced_identity) + monkeypatch.setattr(literal, "read_acs_person_coverage_columns", raced) + with _refuses( + "SNAPSHOT_CHANGED|SOURCE_RECONSTRUCTION_REFUSED|PRODUCER_CHANGED|NATIVE_FRAME_CHANGED" + ): + _load(invented) + + +@pytest.mark.parametrize("ending", [b"\n", b"\r", b"\r\n", b""]) +def test_raw_record_byte_boundary_precedes_decoding_and_preserves_endings( + monkeypatch, ending +): + header = b"A,B\n" + body = b'"x\r\n\ty",z' + ending + monkeypatch.setattr(owner, "MAX_RECORD_BYTES", len(body)) + assert list(owner._records(io.BytesIO(header + body))) == [header, body] + monkeypatch.setattr(owner, "MAX_RECORD_BYTES", len(body) - 1) + with _refuses("CSV_RECORD_BYTES"): + list(owner._records(io.BytesIO(header + body))) + + +@pytest.mark.parametrize( + "value", + [ + "", + None, + "\r\n\t", + '😀"\\\b\f', + ["", None, "é"], + {"b": True, "a": [1, -2, False]}, + ], +) +def test_canonical_size_is_checked_before_string_encoding(monkeypatch, value): + expected = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + assert owner._json(value, len(expected)) == expected + + def forbidden_encoder(*a, **k): + pytest.fail("JSON encoder allocated before preflight refusal") + + monkeypatch.setattr(owner.json, "JSONEncoder", forbidden_encoder) + with _refuses("CANONICAL_SIZE"): + owner._json(value, len(expected) - 1) + + +# --------------------------------------------------------------------------- +# The whole-roster ceiling: one canonical stream, materialised three ways. +# --------------------------------------------------------------------------- + +# Measured over the whole captured 2024 ACS person file (3,422,888 records); +# experiments/native-byte-transports/byte-census-measurements.json. +ACS_PERSONS = 3_422_888 +ACS_HOUSEHOLDS = 1_531_614 +MEASURED_BODY_BYTES = 308_941_698 +MEASURED_LONGEST_BODY_LINE = 102 +MEASURED_KEY_LIST_BYTES = 68_461_900 +MEASURED_SERIALNO_LIST_BYTES = 24_505_825 + + +def _canonical_documents(): + return [ + {"z": '\u00e9\t"\\', "a": [0, 1, None, True, False], "n": -12}, + ["\U0001f600", {"k": "\u2028"}, 12345678901234567890, ""], + {"requested_serialnos": [f"2024HU{i:07d}" for i in range(1, 200)]}, + ] + + +@pytest.mark.parametrize("value", _canonical_documents()) +def test_canonical_size_digest_and_segments_are_one_stream(monkeypatch, value): + """_json, _json_size, _json_sha256 and _json_roster agree byte for byte.""" + whole = owner._json(value, owner.MAX_ROSTER_BYTES) + assert owner._json_size(value, owner.MAX_ROSTER_BYTES) == len(whole) + assert ( + owner._json_sha256(value, owner.MAX_ROSTER_BYTES) + == hashlib.sha256(whole).hexdigest() + ) + tokens = list(owner._json_chunks(value, owner.MAX_ROSTER_BYTES)) + longest = max(map(len, tokens)) + for segment in sorted({longest, longest + 1, 17, 4096, len(whole)}): + if segment >= longest: + assert owner._json_roster(value, segment=segment) == whole + # The refusal keeps its code on the condition a segmented stream can still + # reach: one token larger than one segment, or a stream above the total. + with _refuses("CANONICAL_SIZE"): + owner._json_roster(value, segment=longest - 1) + with _refuses("CANONICAL_SIZE"): + owner._json_roster(value, maximum=len(whole) - 1) + with _refuses("CANONICAL_SIZE"): + owner._json_sha256(value, len(whole) - 1) + + # The size pass refuses before the encoder allocates a token. + def forbidden_encoder(*_args, **_kwargs): + raise AssertionError("encoder reached before the size was charged") + + monkeypatch.setattr(owner.json, "JSONEncoder", forbidden_encoder) + with _refuses("CANONICAL_SIZE"): + owner._json_size(value, len(whole) - 1) + + +def _field_states(): + contract = literal.coverage_field_contract() + states = set() + for definition in contract["fields"].values(): + for age in ("", "x", "5", "16", "17", "80", "100"): + for raw in ("", "1", "9", "zz"): + states.add( + literal._field_state( + age, + raw, + minimum_age=definition["minimum_age"], + codes=definition["codes"], + ) + ) + return sorted(states) + + +def test_body_row_bound_covers_every_body_line(): + """The pre-allocation charge is an upper bound on the line the body holds.""" + states = _field_states() + longest = max(states, key=len) + assert longest == "value_below_age_universe" + assert owner._STATE_BYTES == 2 * (len(owner._json(longest)) + 1) + rows = [ + ["2024HU0000001", "1", "30", "1", "4"], + ["2024GQ0000002", "12", "80", "", ""], + ['\u00fc"\t', "9", "", "x", "\\"], + ] + for cells in rows: + for first in states: + for second in states: + line = ( + owner._json( + [ + cells[0], + int(cells[1]), + *cells[2:], + first, + second, + "psam_pusb.csv", + 1_679_137, + ], + owner.MAX_RECORD_BYTES - 1, + ) + + b"\n" + ) + assert owner._body_row_bound(cells, "psam_pusb.csv", 1_679_137) >= len( + line + ) + + +def test_body_charge_bounds_the_issued_body(invented): + """Over the invented archive, the summed charge covers the body actually issued.""" + _fixture, _frame, _h, people = invented + members = {"psam_pusa.csv": people[:1], "psam_pusb.csv": people[1:]} + charged = sum( + owner._body_row_bound( + [str(row[column]) for column in literal.READ_COLUMNS], member, ordinal + ) + for member, rows in members.items() + for ordinal, row in enumerate(rows, 1) + ) + receipt = _load(invented).receipt + assert receipt["rows"] == len(people) + assert receipt["body_bytes"] <= charged + + +def test_body_segments_reassemble_identically_at_every_segment_size( + invented, monkeypatch +): + """MAX_BODY_BYTES is now the size of one accumulation; the body is unchanged.""" + baseline = _load(invented) + body = baseline._parts()[1] + lines = body.splitlines(keepends=True) + longest = max(map(len, lines)) + assert len(lines) == 3 and longest < len(body) + for segment in sorted( + {longest, longest + 1, 2 * longest, len(body) - 1, len(body)} + ): + monkeypatch.setattr(owner, "MAX_BODY_BYTES", segment) + issued = _load(invented) + assert issued._parts()[1] == body + assert issued.receipt["body_sha256"] == baseline.receipt["body_sha256"] + monkeypatch.setattr(owner, "MAX_BODY_BYTES", longest - 1) + with _refuses("BODY_SIZE"): + _load(invented) + + +def test_body_total_refuses_closed_when_the_charge_admits(invented, monkeypatch): + """BODY_LIMIT is the body's own ceiling, behind the charge that normally fires first.""" + body = _load(invented)._parts()[1] + monkeypatch.setattr(owner, "_body_row_bound", lambda *_args: 0) + monkeypatch.setattr(owner, "MAX_ROSTER_BYTES", len(body) - 1) + with _refuses("BODY_LIMIT"): + _load(invented) + monkeypatch.setattr(owner, "MAX_ROSTER_BYTES", len(body)) + assert _load(invented)._parts()[1] == body + + +def test_roster_ceiling_admits_a_full_source_body_without_allocating(): + """The measured full-source body, charge and key list all fit the ceiling.""" + assert owner.MAX_ROSTER_BYTES == 64 * owner.MAX_BODY_BYTES + assert MEASURED_BODY_BYTES > owner.MAX_BODY_BYTES # why the single cap had to go + assert MEASURED_BODY_BYTES <= owner.MAX_ROSTER_BYTES + assert MEASURED_LONGEST_BODY_LINE * ACS_PERSONS <= owner.MAX_ROSTER_BYTES + widest = owner._body_row_bound( + ["2024HU0009999999", "99", "99", "9", "9"], "psam_pusb.csv", ACS_PERSONS + ) + assert widest * ACS_PERSONS <= owner.MAX_ROSTER_BYTES + assert MEASURED_KEY_LIST_BYTES > owner.MAX_BODY_BYTES # the digest's old cap bound + assert MEASURED_KEY_LIST_BYTES <= owner.MAX_ROSTER_BYTES + assert MEASURED_SERIALNO_LIST_BYTES <= owner.MAX_ROSTER_BYTES + assert 16 * ACS_HOUSEHOLDS + 1 == MEASURED_SERIALNO_LIST_BYTES + + +def _serialnos(invented): + return tuple(sorted({row["SERIALNO"] for row in invented[2]})) + + +def test_native_issuance_refusal_names_the_ceiling_that_refused(invented, monkeypatch): + """The 1/15 shape: NATIVE_ISSUANCE_REFUSED now carries CANONICAL_SIZE as its cause.""" + fixture = invented[0] + monkeypatch.setattr(owner, "MAX_ROSTER_BYTES", 8) + with pytest.raises( + binding.ACSNativeCoverageBindingError, match="^NATIVE_ISSUANCE_REFUSED$" + ) as exc: + binding.issue_acs_native_coverage( + fixture.source_dir, + snapshot_root=fixture.snapshot_root, + serialnos=_serialnos(invented), + ) + cause = exc.value.__cause__ + assert isinstance(cause, owner.ACSCoverageAuthenticationError) + assert str(cause) == "CANONICAL_SIZE" + + +def test_native_issuance_receipt_is_the_whole_canonical_stream(invented): + """The segmented receipt is byte-identical to the single encode, at any segment.""" + fixture = invented[0] + serialnos = _serialnos(invented) + issued = binding.issue_acs_native_coverage( + fixture.source_dir, snapshot_root=fixture.snapshot_root, serialnos=serialnos + ) + receipt = json.loads(issued.payload) + assert receipt["selection"]["requested_serialnos"] == list(serialnos) + assert owner._json(receipt, owner.MAX_ROSTER_BYTES) == issued.payload + longest = max(map(len, owner._json_chunks(receipt, owner.MAX_ROSTER_BYTES))) + for segment in (longest, longest + 1, 4096): + assert owner._json_roster(receipt, segment=segment) == issued.payload + assert owner._json_sha256(receipt) == owner._sha(issued.payload) diff --git a/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py b/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py new file mode 100644 index 000000000..72403c3c3 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py @@ -0,0 +1,516 @@ +"""Literal coverage projections from invented, multi-member ACS archives.""" + +import csv +import hashlib +import io +import json +from zipfile import ZipFile + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import acs_person_coverage_columns as module +from microcosm.build.us_runtime import acs_pums + + +def _row(serial="2024HU0000001", order="1", age="30", mil="4", esr="6"): + return dict(zip(module.READ_COLUMNS, (serial, order, age, mil, esr), strict=True)) + + +def _archive(tmp_path, members): + path = tmp_path / "persons.zip" + with ZipFile(path, "w") as archive: + for name, rows in members.items(): + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=module.READ_COLUMNS, lineterminator="\n" + ) + writer.writeheader() + writer.writerows(rows) + archive.writestr(name, stream.getvalue()) + return acs_pums.AcsPumsSource( + tmp_path / "unused-households.zip", path, max_households=1 + ) + + +def _keys(rows): + return pd.DataFrame(rows).loc[:, list(module.KEYS)] + + +def test_private_scanner_exhausts_members_despite_consumer_return_value(tmp_path): + first = _row(order="02", age="16", mil="", esr="1") + second = _row(order="01", age="17", mil="4", esr=" 6") + third = _row("2024GQ0000001", age="", mil='x,"\t', esr="") + source = _archive( + tmp_path, + {"psam_pusb.csv": [second, third], "psam_pusa.csv": [first]}, + ) + received = [] + + def consume(cells): + received.append(cells) + return True # No return value can turn a prefix into a successful scan. + + members, rows = module._scan_acs_person_coverage(source, consume) + assert members == ["psam_pusa.csv", "psam_pusb.csv"] + assert rows == 3 + assert received == [ + ("2024HU0000001", "02", "16", "", "1"), + ("2024HU0000001", "01", "17", "4", " 6"), + ("2024GQ0000001", "1", "", 'x,"\t', ""), + ] + + +@pytest.mark.parametrize( + "tail,match", + [ + ("2024HU0000009,1,30,4,6,EXTRA\n", "record width"), + ("2024HU0000009,1,30,4,6\x00\n", "forbidden control"), + ('2024HU0000009,1,30,"unfinished,6\n', "invalid literal CSV"), + ], +) +def test_private_scanner_refuses_invalid_tail_after_delivering_expected_rows( + tmp_path, tail, match +): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR\n2024HU0000001,1,30,4,6\n" + tail, + ) + received = [] + with pytest.raises(ValueError, match=match): + module._scan_acs_person_coverage(source, received.append) + assert received == [("2024HU0000001", "1", "30", "4", "6")] + + +def test_exact_household_roster_across_members_preserves_blanks_and_request_order( + tmp_path, +): + rows = [ + _row(mil="1", esr="4"), + _row(order="2", age="10", mil="", esr=""), + _row("2024GQ0000001", age="16", mil="", esr="1"), + ] + unrelated = _row("2024HU0000009") + source = _archive( + tmp_path, + { + "nested/psam_pusb.csv": [rows[0], unrelated], + "psam_pusa.csv": [rows[2], rows[1]], + }, + ) + requested = _keys([rows[1], rows[2], rows[0]]) + requested["SPORDER"] = requested.SPORDER.astype("int64") + result, receipt = module.read_acs_person_coverage_columns( + source, person_keys=requested, chunksize=1 + ) + assert result.MIL.tolist() == ["", "", "1"] + assert result.ESR.tolist() == ["", "1", "4"] + assert result.MIL_state.tolist() == [ + "outside_age_universe", + "outside_age_universe", + "observed_code", + ] + assert result.ESR_state.tolist() == [ + "outside_age_universe", + "observed_code", + "observed_code", + ] + assert result.SERIALNO.tolist() == requested.SERIALNO.tolist() + assert result.SPORDER.tolist() == requested.SPORDER.tolist() + assert not result.isna().any().any() + assert receipt["source_rows_streamed"] == 4 + assert receipt["selected_person_rows"] == 3 + assert receipt["source_authenticated"] is False + assert receipt["release_eligible"] is False + assert receipt["legacy_max_households_applied"] is False + assert receipt["coverage_status"] == "literal_source_fields_only" + second, other = module.read_acs_person_coverage_columns( + source, person_keys=requested, chunksize=3 + ) + pd.testing.assert_frame_equal(result, second) + assert other == receipt + + +@pytest.mark.parametrize( + "age,mil,esr,mil_state,esr_state", + [ + ("17", "2", "2", "observed_code", "observed_code"), + ("45", "3", "5", "observed_code", "observed_code"), + ("30", "", "", "missing_in_universe", "missing_in_universe"), + ("30", "0", "7", "unlabelled_code", "unlabelled_code"), + ("30", "NA", "1.0", "unlabelled_code", "unlabelled_code"), + ("30", " 1", "4 ", "unlabelled_code", "unlabelled_code"), + ("15", "1", "4", "value_below_age_universe", "value_below_age_universe"), + ("", "1", "4", "age_unresolved", "age_unresolved"), + ("100", "4", "6", "age_unresolved", "age_unresolved"), + ], +) +def test_unresolved_states_preserve_literal_source_tokens( + tmp_path, age, mil, esr, mil_state, esr_state +): + row = _row(age=age, mil=mil, esr=esr) + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + table, _ = module.read_acs_person_coverage_columns(source, person_keys=_keys([row])) + assert table.loc[0, ["AGEP", "MIL", "ESR"]].tolist() == [age, mil, esr] + assert table.MIL_state.tolist() == [mil_state] + assert table.ESR_state.tolist() == [esr_state] + + +@pytest.mark.parametrize( + "mutation,match", + [ + ("extra", "extra selected-household"), + ("missing", "native roster differ"), + ("duplicate", "keys repeat"), + ("wrong_order", "native roster differ"), + ], +) +def test_partial_household_or_conflicting_native_roster_refuses( + tmp_path, mutation, match +): + rows = [_row(), _row(order="2")] + observed = rows.copy() + if mutation == "extra": + observed += [_row(order="3")] + elif mutation == "missing": + observed.pop() + elif mutation == "duplicate": + observed[1] = rows[0] + else: + observed[1] = _row(order="3") + source = _archive(tmp_path, {"psam_pusa.csv": observed}) + with pytest.raises(ValueError, match=match): + module.read_acs_person_coverage_columns( + source, person_keys=_keys(rows), chunksize=1 + ) + + +@pytest.mark.parametrize( + "serial,order", + [ + ("2023HU0000001", "1"), + ("2024HU0000001", "0"), + ("2024HU0000001", "21"), + ("2024HU0000001", "1.0"), + ("2024HU0000001\x00other", "1"), + ("2024HU0000001", "1\x00other"), + (None, "1"), + ("2024HU0000001", True), + ], +) +def test_invalid_native_keys_refuse_before_file_read(tmp_path, serial, order): + source = acs_pums.AcsPumsSource( + tmp_path / "absent-h.zip", tmp_path / "absent-p.zip" + ) + with pytest.raises(ValueError, match="native"): + module.read_acs_person_coverage_columns( + source, person_keys=pd.DataFrame({"SERIALNO": [serial], "SPORDER": [order]}) + ) + + +@pytest.mark.parametrize( + "mutation", ["missing_column", "duplicate_column", "duplicate_member"] +) +def test_ambiguous_source_schema_refuses(tmp_path, mutation): + source = _archive(tmp_path, {"psam_pusa.csv": [_row()]}) + with ZipFile(source.person_zip, "r") as archive: + raw = archive.read("psam_pusa.csv") + if mutation == "missing_column": + raw = raw.replace(b"MIL", b"OTHER") + elif mutation == "duplicate_column": + raw = raw.replace(b"ESR", b"MIL") + with ZipFile(source.person_zip, "w") as archive: + archive.writestr("psam_pusa.csv", raw) + if mutation == "duplicate_member": + with pytest.warns(UserWarning, match="Duplicate name"): + archive.writestr("psam_pusa.csv", raw) + with pytest.raises(ValueError, match="missing or duplicated"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +def test_explicit_row_bound_refuses_without_truncation(tmp_path, monkeypatch): + row = _row() + source = _archive(tmp_path, {"psam_pusa.csv": [row, _row("2024HU0000002")]}) + monkeypatch.setattr(module, "MAX_ROWS", 1) + with pytest.raises(ValueError, match="row bound"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([row]), chunksize=1 + ) + + +def test_selected_row_ceiling_refuses_before_opening_the_source(tmp_path, monkeypatch): + source = acs_pums.AcsPumsSource( + tmp_path / "absent-h.zip", tmp_path / "absent-p.zip" + ) + monkeypatch.setattr(module, "MAX_SELECTED_ROWS", 1) + with pytest.raises(ValueError, match="selected person count is outside the bound"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row(), _row(order="2")]) + ) + + +def test_contract_is_owned_and_separate_from_legacy_native_roster(): + contract = module.coverage_field_contract() + assert contract["fields"]["MIL"]["minimum_age"] == 17 + assert contract["fields"]["ESR"]["minimum_age"] == 16 + contract["fields"]["MIL"]["codes"].clear() + assert len(module.coverage_field_contract()["fields"]["MIL"]["codes"]) == 4 + assert not {"MIL", "ESR"}.intersection(acs_pums._PERSON_REQUIRED) + + +def _raw_archive(tmp_path, raw): + source = acs_pums.AcsPumsSource( + tmp_path / "unused-households.zip", tmp_path / "literal-persons.zip" + ) + with ZipFile(source.person_zip, "w") as archive: + archive.writestr("psam_pusa.csv", raw) + return source + + +@pytest.mark.parametrize("field", module.READ_COLUMNS) +def test_nul_suffix_refuses_before_truncating_value_or_identity(tmp_path, field): + row = _row(mil="1") + row[field] += "\x00garbage" + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize( + "control", + [chr(code) for code in range(32) if code not in (9, 10, 13)] + + [chr(code) for code in range(127, 160)], +) +def test_non_csv_control_characters_refuse(tmp_path, control): + source = _archive(tmp_path, {"psam_pusa.csv": [_row(mil="1" + control)]}) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("location", ["header", "ignored_column", "unselected_row"]) +def test_control_validation_precedes_column_and_household_selection(tmp_path, location): + header = "SERIALNO,SPORDER,AGEP,MIL,ESR,OTHER\n" + body = "2024HU0000001,1,30,1,6,ok\n" + if location == "header": + header = header.replace("OTHER", "OTHER\x00suffix") + elif location == "ignored_column": + body = body.replace("ok", "ok\x00suffix") + else: + body += "2024HU0000009,1,30,1\x00suffix,6,ok\n" + source = _raw_archive(tmp_path, header + body) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("serial", ["2024HU0000001", "2024HU0000009"]) +@pytest.mark.parametrize( + "tail", + ["1,30,4", "1,30,4,6", "1,30,4,6,ok,extra", "1,30,4,6,ok,"], + ids=["missing_required", "missing_unused", "too_many", "trailing_delimiter"], +) +def test_full_record_width_refuses_even_outside_projection(tmp_path, serial, tail): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR,OTHER\n" + serial + "," + tail + "\n", + ) + with pytest.raises(ValueError, match="record width"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize( + "body", + ["\n", '2024HU0000001,1,30,4,"unterminated\n', '2024HU0000001,1,30,"1"garbage,6\n'], +) +def test_blank_records_and_malformed_quoting_refuse(tmp_path, body): + source = _raw_archive(tmp_path, "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + body) + with pytest.raises(ValueError, match="CSV|record width"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("line_end", ["\n", "\r", "\r\n"]) +def test_quoted_delimiters_newlines_tabs_and_digest_preserve_tokens(tmp_path, line_end): + stream = io.StringIO(newline="") + writer = csv.writer(stream, quoting=csv.QUOTE_ALL, lineterminator=line_end) + writer.writerow(["OTHER", "ESR", "MIL", "AGEP", "SPORDER", "SERIALNO"]) + writer.writerow( + [ + 'ignored, "quoted"\r\nfield', + "4\r\n\t", + '1,"quoted"\n\r', + "30", + "01", + "2024HU0000001", + ] + ) + source = _raw_archive(tmp_path, "\ufeff" + stream.getvalue()) + table, receipt = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]), chunksize=1 + ) + expected = [ + { + "SERIALNO": "2024HU0000001", + "SPORDER": 1, + "AGEP": "30", + "MIL": '1,"quoted"\n\r', + "ESR": "4\r\n\t", + "MIL_state": "unlabelled_code", + "ESR_state": "unlabelled_code", + } + ] + assert table.to_dict("records") == expected + assert receipt["source_rows_streamed"] == 1 + + def digest(records): + return hashlib.sha256( + json.dumps( + records, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + + assert receipt["projection_sha256"] == digest(expected) + for token in ("4\n\t", "4\r\t", r"4\r\n\t", "4"): + assert receipt["projection_sha256"] != digest([{**expected[0], "ESR": token}]) + + +@pytest.mark.parametrize("field", ["AGEP", "MIL", "ESR"]) +@pytest.mark.parametrize("control", ["\t", "\r", "\n", "\r\n"]) +def test_accepted_csv_whitespace_is_literal_and_never_an_observed_code( + tmp_path, field, control +): + row = _row(mil="1", esr="4") + row[field] += control + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=module.READ_COLUMNS, quoting=csv.QUOTE_ALL + ) + writer.writeheader() + writer.writerow(row) + source = _raw_archive(tmp_path, stream.getvalue()) + table, _ = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]) + ) + assert table.loc[0, field] == row[field] + if field == "AGEP": + assert ( + table.MIL_state.tolist() == table.ESR_state.tolist() == ["age_unresolved"] + ) + else: + assert table[field + "_state"].tolist() == ["unlabelled_code"] + + +@pytest.mark.parametrize("field", module.KEYS) +@pytest.mark.parametrize("suffix", ["\t", "\n", "\r\n", " other", ",other"]) +def test_literal_identity_suffix_cannot_join_canonical_requested_key( + tmp_path, field, suffix +): + row = _row() + row[field] += suffix + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + with pytest.raises(ValueError, match="missing|native"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +def test_duplicate_canonical_person_order_in_later_member_refuses(tmp_path): + source = _archive( + tmp_path, + {"psam_pusa.csv": [_row(order="01")], "psam_pusb.csv": [_row(order="1")]}, + ) + with pytest.raises(ValueError, match="keys repeat"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row(), _row(order="2")]), chunksize=1 + ) + + +def test_invalid_utf8_refuses(tmp_path): + source = _raw_archive( + tmp_path, b"SERIALNO,SPORDER,AGEP,MIL,ESR\n2024HU0000001,1,30,1\xff,6\n" + ) + with pytest.raises(ValueError): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("multiline", [False, True]) +def test_record_character_bound_includes_all_quoted_physical_lines( + tmp_path, monkeypatch, multiline +): + monkeypatch.setattr(module, "MAX_CSV_RECORD_CHARS", 100, raising=False) + suffix = ("x\n" if multiline else "xx") * 60 + source = _archive(tmp_path, {"psam_pusa.csv": [_row(mil=suffix)]}) + with pytest.raises(ValueError, match="record character bound"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("line_end", ["", "\n", "\r\n"]) +def test_record_character_bound_accepts_exact_limit_and_resets_per_record( + tmp_path, monkeypatch, line_end +): + monkeypatch.setattr(module, "MAX_CSV_RECORD_CHARS", 100) + prefix, suffix = '2024HU0000001,1,30,"', '",6' + line_end + token = "x\n" + "x" * (100 - len(prefix) - len(suffix) - 2) + body = prefix + token + suffix + assert len(body) == 100 + source = _raw_archive(tmp_path, "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + body) + table, receipt = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]) + ) + assert table.MIL.tolist() == [token] + assert receipt["source_rows_streamed"] == 1 + + +def test_stream_opens_member_once_and_stops_at_refusal_without_full_validation( + tmp_path, monkeypatch +): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + + "2024HU0000001,1,30,4,6\n" * 2 + + "2024HU0000009,1,30,4,6\n" * 20_000, + ) + opened, consumed = [], [] + + class GuardedZipFile(ZipFile): + def open(self, name, *args, **kwargs): + opened.append(name) + member = super().open(name, *args, **kwargs) + original_read, original_read1 = member.read, member.read1 + + def bounded(method, size=-1): + assert size >= 0, "unbounded archive read" + result = method(size) + consumed.append(len(result)) + return result + + member.read = lambda size=-1: bounded(original_read, size) + member.read1 = lambda size=-1: bounded(original_read1, size) + return member + + def read(self, *args, **kwargs): + pytest.fail("whole-member validation read") + + monkeypatch.setattr(module, "ZipFile", GuardedZipFile) + with pytest.raises(ValueError, match="extra selected-household"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]), chunksize=1 + ) + assert opened == ["psam_pusa.csv"] + assert sum(consumed) < 32_768 + + +def test_selected_row_ceiling_refuses_at_its_own_number(tmp_path, monkeypatch): + """The refusal is `<= MAX_SELECTED_ROWS`, whatever that number is. + + At exactly the ceiling the read proceeds and fails later, on the absent + archive it then opens. `test_us_native_row_ceilings.py` carries the separate + assertion that the shipped number admits the 3,422,888 persons a full-source + ACS selection requests. + """ + source = acs_pums.AcsPumsSource( + tmp_path / "absent-h.zip", tmp_path / "absent-p.zip" + ) + monkeypatch.setattr(module, "MAX_SELECTED_ROWS", 2) + at_ceiling = _keys([_row(), _row(order="2")]) + with pytest.raises(FileNotFoundError): + module.read_acs_person_coverage_columns(source, person_keys=at_ceiling) + over = _keys([_row(), _row(order="2"), _row(order="3")]) + with pytest.raises(ValueError, match="selected person count is outside the bound"): + module.read_acs_person_coverage_columns(source, person_keys=over) diff --git a/packages/microcosm-build/tests/test_us_acs_population_catalogue.py b/packages/microcosm-build/tests/test_us_acs_population_catalogue.py new file mode 100644 index 000000000..ab98b77dc --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_population_catalogue.py @@ -0,0 +1,966 @@ +"""Actual pinned invented archives; never a native Frame or fake source issuer.""" + +import _csv +import ast +import copy +import csv +import gc +import hashlib +import inspect +import io +import json +import pickle +import sys +import weakref +import zipfile +from pathlib import Path +from types import CodeType, FunctionType, SimpleNamespace + +import pytest + +from microcosm.build.us_runtime import acs_population_catalogue as owner +from microcosm.frame import Frame + + +def csv_bytes(rows): + stream = io.StringIO(newline="") + writer = csv.DictWriter(stream, list(rows[0]), lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + return stream.getvalue().encode() + + +@pytest.fixture +def invented(tmp_path, monkeypatch): + source, snapshots = tmp_path / "source", tmp_path / "snapshots" + source.mkdir() + snapshots.mkdir() + households = [ + dict( + SERIALNO="2024HU0000001", + TYPEHUGQ="1", + NP="2", + WGTP="0010", + TEN="1", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024GQ0000001", + TYPEHUGQ="2", + NP="1", + WGTP="0", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024HU0000002", + TYPEHUGQ="1", + NP="0", + WGTP="20", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024GQ0000002", + TYPEHUGQ="3", + NP="1", + WGTP="0", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024HU0000003", + TYPEHUGQ="1", + NP="1", + WGTP="30", + TEN="3", + ST="01", + PUMA="00100", + ), + ] + people = [ + dict( + SERIALNO="2024HU0000001", + SPORDER="01", + PWGTP="0011", + AGEP="30", + MIL="1", + ESR="4", + ), + dict( + SERIALNO="2024GQ0000001", + SPORDER="01", + PWGTP="0077", + AGEP="40", + MIL="4", + ESR="6", + ), + dict( + SERIALNO="2024HU0000001", + SPORDER="02", + PWGTP="12", + AGEP="15", + MIL="", + ESR="", + ), + dict( + SERIALNO="2024GQ0000002", + SPORDER="1", + PWGTP="78", + AGEP="50", + MIL="4", + ESR="6", + ), + dict( + SERIALNO="2024HU0000003", SPORDER="1", PWGTP="31", AGEP="99", MIL="", ESR="" + ), + ] + + def write(): + for _role, filename, prefix, rows in ( + ("household", "csv_hus.zip", "psam_hus", households), + ("person", "csv_pus.zip", "psam_pus", people), + ): + with zipfile.ZipFile(source / filename, "w") as archive: + empty = ( + b"SERIALNO,SPORDER,PWGTP,AGEP,MIL,ESR\n" + if _role == "person" + else b"SERIALNO,TYPEHUGQ,NP,WGTP,TEN,ST,PUMA\n" + ) + archive.writestr( + prefix + "a.csv", csv_bytes(rows[:2]) if rows[:2] else empty + ) + archive.writestr( + prefix + "b.csv", csv_bytes(rows[2:]) if rows[2:] else empty + ) + pins = tuple( + ( + role, + name, + hashlib.sha256((source / name).read_bytes()).hexdigest(), + (source / name).stat().st_size, + ) + for role, name in (("household", "csv_hus.zip"), ("person", "csv_pus.zip")) + ) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", pins) + + monkeypatch.setattr( + owner.housing.shutil, + "disk_usage", + lambda _path: SimpleNamespace(free=64 * 1024**3), + ) + monkeypatch.setattr(owner, "_BATCH_PEOPLE", 2) + write() + return SimpleNamespace( + source=source, + snapshots=snapshots, + households=households, + people=people, + write=write, + root=tmp_path, + ) + + +def issue(fixture, **kwargs): + return owner.issue_acs_source_catalogue( + fixture.source, snapshot_root=fixture.snapshots, **kwargs + ) + + +def test_real_complete_catalogue_without_population_or_classification(invented): + forbidden = [] + + def trace(frame, event, arg): + module = frame.f_globals.get("__name__", "") + if event == "call" and ( + frame.f_code is Frame.__init__.__code__ + or module.startswith("microunit") + or frame.f_code.co_name + in { + "build_acs_pums_unit_frame", + "prepare_acs_housing_population", + "classify_household", + "classify_households", + } + ): + forbidden.append(frame.f_code.co_name) + raise AssertionError("Population construction/classification occurred") + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = issue(invented) + rows = result.households + receipt = result.receipt + finally: + sys.setprofile(previous) + assert not forbidden + assert result.validate() is result + assert owner.verify_acs_source_catalogue(result) is result + assert len(rows) == 4 and len(result.exclusion_ledger) == 1 + assert {r.key.native_id for r in rows} == { + r["SERIALNO"] for r in invented.households if r["NP"] != "0" + } + row = rows[0] + assert row.wgtp == "0010" + assert [(p.sporder, p.age, p.esr, p.mil, p.pwgtp) for p in row.persons] == [ + ("01", "30", "4", "1", "0011"), + ("02", "15", "", "", "12"), + ] + assert row.persons[1].esr_state == "outside_age_universe" + assert rows[-1].persons[0].esr_state == "missing_in_universe" + assert rows[1].persons[0].pwgtp == "0077" + assert result.lineage[row.key.native_id]["persons"] == ( + ("01", "psam_pusa.csv", 1), + ("02", "psam_pusb.csv", 1), + ) + assert receipt["counts"]["people"] == 5 + assert receipt["counts"]["literal_batches"] == 3 + assert ( + receipt["source_authenticated"] + and not receipt["population_binding_authenticated"] + ) + assert ( + not receipt["domain_assignment_authenticated"] + and not receipt["selection_performed"] + ) + assert not receipt["execution"]["full_person_scan_per_literal_batch"] + assert receipt["execution"]["literal_full_scans"] == 1 + assert issue(invented, candidate=result.to_bytes()).to_bytes() == result.to_bytes() + (invented.root / "catalogue.receipt.json").write_bytes(result.to_bytes()) + held = owner._lookup(result) + (invented.root / "invented-records.json").write_bytes( + owner.native.coverage._json( + {"households": held.records, "vacancy_ledger": held.vacancies} + ) + ) + + +def test_independent_views_cannot_rewrite_issued_rows(invented): + result = issue(invented) + original = result.to_bytes() + row = result.households[0] + object.__setattr__(row, "wgtp", "9999") + object.__setattr__(row.persons[0], "esr", "6") + result.receipt["source_authenticated"] = False + result.lineage.clear() + assert result.households[0].wgtp == "0010" + assert result.households[0].persons[0].esr == "4" + assert result.to_bytes() == original + + +def expected_records(): + """Explicit invented source-order records, independent of collector logic.""" + return ( + ( + "2024HU0000001", + "1", + "2", + "0010", + "psam_husa.csv", + 1, + ( + ( + "01", + "30", + "4", + "observed_code", + "1", + "observed_code", + "0011", + "psam_pusa.csv", + 1, + ), + ( + "02", + "15", + "", + "outside_age_universe", + "", + "outside_age_universe", + "12", + "psam_pusb.csv", + 1, + ), + ), + ), + ( + "2024GQ0000001", + "2", + "1", + "0", + "psam_husa.csv", + 2, + ( + ( + "01", + "40", + "6", + "observed_code", + "4", + "observed_code", + "0077", + "psam_pusa.csv", + 2, + ), + ), + ), + ("2024HU0000002", "1", "0", "20", "psam_husb.csv", 1, ()), + ( + "2024GQ0000002", + "3", + "1", + "0", + "psam_husb.csv", + 2, + ( + ( + "1", + "50", + "6", + "observed_code", + "4", + "observed_code", + "78", + "psam_pusb.csv", + 2, + ), + ), + ), + ( + "2024HU0000003", + "1", + "1", + "30", + "psam_husb.csv", + 3, + ( + ( + "1", + "99", + "", + "missing_in_universe", + "", + "missing_in_universe", + "31", + "psam_pusb.csv", + 3, + ), + ), + ), + ) + + +@pytest.mark.parametrize("batch_limit", [2, 3, 5]) +def test_explicit_canonical_records_and_bytes_ignore_batch_boundaries( + invented, monkeypatch, batch_limit +): + monkeypatch.setattr(owner, "_BATCH_PEOPLE", batch_limit) + result = issue(invented) + held, expected = owner._lookup(result), expected_records() + assert held.records == (expected[0], expected[1], expected[3], expected[4]) + assert held.vacancies == (expected[2],) + raw = b"".join( + json.dumps(row, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + + b"\n" + for row in expected + ) + assert result.receipt["counts"]["canonical_record_bytes"] == len(raw) + assert ( + result.receipt["counts"]["canonical_record_sha256"] + == hashlib.sha256(raw).hexdigest() + ) + + +@pytest.mark.parametrize("batch_limit", [2, 5]) +def test_catalogue_literal_member_scan_count_is_independent_of_batches( + invented, monkeypatch, batch_limit +): + monkeypatch.setattr(owner, "_BATCH_PEOPLE", batch_limit) + scanner = owner.literal._scan_acs_person_coverage.__code__ + active, scans, literal_opens, all_person_opens = set(), [], [], [] + + def trace(frame, event, arg): + if frame.f_code is scanner: + if event == "call": + scans.append(True) + active.add(id(frame)) + elif event == "return": + active.remove(id(frame)) + if event == "call" and frame.f_code is zipfile.ZipFile.open.__code__: + name = frame.f_locals["name"] + name = name.filename if isinstance(name, zipfile.ZipInfo) else name + if name.startswith("psam_pus"): + all_person_opens.append(name) + if active: + literal_opens.append(name) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = issue(invented) + finally: + sys.setprofile(previous) + assert scans == [True] and not active + assert literal_opens == ["psam_pusa.csv", "psam_pusb.csv"] + assert all_person_opens == literal_opens * 3 + assert result.receipt["counts"]["literal_batches"] == (3 if batch_limit == 2 else 1) + + +@pytest.mark.parametrize( + "mutation,code", + [ + ("duplicate", "DUPLICATE_LITERAL_PERSON"), + ("extra", "UNEXPECTED_LITERAL_PERSON"), + ("orphan", "UNEXPECTED_LITERAL_PERSON"), + ("missing", "GLOBAL_PERSON_KEYS"), + ("key_suffix", "LITERAL_PERSON_KEY"), + ], +) +def test_collector_reconciles_every_literal_slot_after_housing_validation( + invented, mutation, code +): + # Exercise this private reconciliation separately from the unchanged + # housing/preflight checks. A mismatched helper input issues no authority. + original = issue(invented) + projection = json.loads(owner._lookup(original).projection_path.read_bytes()) + if mutation == "duplicate": + invented.people.append(dict(invented.people[0], SPORDER="1")) + elif mutation == "extra": + invented.people.append(dict(invented.people[0], SPORDER="3")) + elif mutation == "orphan": + invented.people[-1]["SERIALNO"] = "2024HU0000099" + elif mutation == "missing": + invented.people.pop() + else: + invented.people[-1]["SPORDER"] = "1 " + invented.write() + paths = { + "household": invented.source / "csv_hus.zip", + "person": invented.source / "csv_pus.zip", + } + with pytest.raises(owner.ACSSourceCatalogueError, match=f"^{code}$"): + owner._collect(projection, paths) + + +def test_prospective_byte_charge_refuses_before_retaining_the_next_person( + invented, monkeypatch +): + original = issue(invented) + held = owner._lookup(original) + projection = json.loads(held.projection_path.read_bytes()) + expected = expected_records() + + def encoded(value): + return json.dumps(value, separators=(",", ":"), allow_nan=False).encode() + + overhead = sum(len(encoded((*row[:6], ()))) + 1 for row in expected) + monkeypatch.setattr( + owner, "_CATALOGUE_BYTES", overhead + len(encoded(expected[0][6][0])) + ) + retained = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_globals.get("__name__") == owner.__name__ + and frame.f_code.co_qualname == "_collect..consume_row" + ): + retained.append( + sum(person is not None for person in frame.f_locals["people"]) + ) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises( + owner.native.coverage.ACSCoverageAuthenticationError, + match="^CANONICAL_SIZE$", + ): + owner._collect(projection, dict(held.paths)) + finally: + sys.setprofile(previous) + assert retained == [1, 1] + + +def test_source_order_and_original_person_order_survive_reverse_zip_insertion(invented): + # Keep serials unsorted and make the split household's raw person order + # descending; canonical order remains source-member then source-row order. + invented.people[0]["SPORDER"], invented.people[2]["SPORDER"] = "02", "01" + invented.write() + first = issue(invented) + before = owner._lookup(first) + for name in ("csv_hus.zip", "csv_pus.zip"): + path = invented.source / name + with zipfile.ZipFile(path) as archive: + members = [ + (member.filename, archive.read(member)) for member in archive.infolist() + ] + with zipfile.ZipFile(path, "w") as archive: + for member, raw in reversed(members): + archive.writestr(member, raw) + pins = tuple( + ( + role, + name, + hashlib.sha256((invented.source / name).read_bytes()).hexdigest(), + (invented.source / name).stat().st_size, + ) + for role, name in (("household", "csv_hus.zip"), ("person", "csv_pus.zip")) + ) + # New physical ZIPs have their own invented pins; compare source records, + # not archive/producer receipts from different physical artifacts. + owner.housing._ARCHIVE_PINS = pins + second = issue(invented) + after = owner._lookup(second) + assert after.records == before.records and after.vacancies == before.vacancies + assert [p[0] for p in after.records[0][6]] == ["02", "01"] + + +@pytest.mark.parametrize("kind", ["new", "copy", "deepcopy", "pickle", "payload"]) +def test_foreign_changed_or_copied_authority_refuses(invented, kind): + result = issue(invented) + if kind == "new": + other = object.__new__(owner.AuthenticatedACSSourceCatalogue) + object.__setattr__(other, "payload", result.payload) + elif kind == "payload": + other = result + raw = json.loads(result.payload) + raw["counts"]["people"] = 16 + object.__setattr__(other, "payload", owner.native.coverage._json(raw)) + else: + try: + other = { + "copy": copy.copy, + "deepcopy": copy.deepcopy, + "pickle": lambda x: pickle.loads(pickle.dumps(x)), + }[kind](result) + except (TypeError, owner.ACSSourceCatalogueError): + return + with pytest.raises( + owner.ACSSourceCatalogueError, match="^ISSUANCE_(NOT_OWNED|CHANGED)$" + ): + other.validate() + + +def test_independent_reissue_and_weak_cleanup(invented): + one, two = issue(invented), issue(invented) + assert one.to_bytes() == two.to_bytes() and one is not two + identity, reference = id(one), weakref.ref(one) + del one + gc.collect() + assert reference() is None and identity not in owner._ISSUED + assert two.validate() is two + + +@pytest.mark.parametrize("kind", ["duplicate", "orphan", "missing", "late_extra", "np"]) +def test_global_complete_roster_refuses_across_members_and_batches(invented, kind): + if kind == "duplicate": + invented.people[-1] = dict(invented.people[0]) + elif kind == "orphan": + invented.people[-1]["SERIALNO"] = "2024HU0000099" + elif kind == "missing": + invented.people.pop() + elif kind == "late_extra": + extra = dict(invented.people[0], SPORDER="3") + invented.people.append(extra) + else: + invented.households[0]["NP"] = "3" + invented.write() + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +@pytest.mark.parametrize( + "cap,value", + [ + ("_ARCHIVE_BYTES", 1), + ("_EXPANDED_BYTES", 1), + ("_MEMBERS", 1), + ("_SOURCE_ROWS", 1), + ("_BATCH_PEOPLE", 1), + ("_CATALOGUE_BYTES", 1), + ("_RECEIPT_BYTES", 1), + ], +) +def test_prospective_small_caps_refuse(invented, monkeypatch, cap, value): + monkeypatch.setattr(owner, cap, value) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +@pytest.mark.parametrize("escaped_literals", [False, True]) +def test_byte_ceiling_exact_boundary_is_record_based( + invented, monkeypatch, escaped_literals +): + if escaped_literals: + invented.people[0]["MIL"] = 'x,"\\\t' + invented.people[0]["ESR"] = "é" + invented.write() + size = issue(invented).receipt["counts"]["canonical_record_bytes"] + monkeypatch.setattr(owner, "_CATALOGUE_BYTES", size) + assert issue(invented).receipt["counts"]["canonical_record_bytes"] == size + monkeypatch.setattr(owner, "_CATALOGUE_BYTES", size - 1) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +def test_candidate_is_reconstructed_and_cannot_authorize_changed_values(invented): + result = issue(invented) + candidate = result.to_bytes() + invented.people[0]["ESR"] = "6" + invented.write() + with pytest.raises(owner.ACSSourceCatalogueError, match="^CANDIDATE_MISMATCH$"): + issue(invented, candidate=candidate) + assert issue(invented).households[0].persons[0].esr == "6" + with pytest.raises( + owner.ACSSourceCatalogueError, match="^SOURCE_AUTHORITY_CHANGED$" + ): + result.validate() + + +@pytest.mark.parametrize("target", ["original", "projection", "producer"]) +def test_live_source_projection_and_producer_changes_refuse( + invented, monkeypatch, target +): + result = issue(invented) + if target == "original": + path = invented.source / "csv_pus.zip" + raw = path.read_bytes() + path.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + elif target == "projection": + owned = owner._lookup(result) + raw = owned.projection_path.read_bytes() + owned.projection_path.chmod(0o600) + owned.projection_path.write_bytes(b" " + raw[1:]) + else: + monkeypatch.setattr(owner, "_BATCH_PEOPLE", 3) + with pytest.raises(owner.ACSSourceCatalogueError): + result.validate() + + +def test_new_owner_static_surface_excludes_population_selection_and_classification(): + tree = ast.parse(Path(owner.__file__).read_text()) + calls = { + node.func.id if isinstance(node.func, ast.Name) else node.func.attr + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, (ast.Name, ast.Attribute)) + } + assert not calls & { + "Frame", + "build_acs_pums_unit_frame", + "prepare_acs_housing_population", + "assign_us_unit_structure", + "classify_household", + "classify_households", + "issue_acs_native_coverage", + "sample", + "choice", + } + assert {"_reconstruct", "_scan_acs_person_coverage", "_live_code"} <= calls + assert "read_acs_person_coverage_columns" not in calls + + +@pytest.mark.parametrize("mutation", ["source", "limit"]) +def test_mutations_at_literal_return_refuse(invented, mutation): + target = owner.literal._scan_acs_person_coverage + fired = [] + + def timing(frame, event, arg): + if frame.f_code is target.__code__ and event == "return" and not fired: + fired.append(True) + if mutation == "source": + path = invented.source / "csv_pus.zip" + raw = path.read_bytes() + path.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + else: + owner._BATCH_PEOPLE = 3 + + before, previous = owner._BATCH_PEOPLE, sys.getprofile() + sys.setprofile(timing) + try: + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + finally: + sys.setprofile(previous) + owner._BATCH_PEOPLE = before + assert fired + + +def test_returned_housing_capsule_cannot_replace_actual_projected_anchors(invented): + target, fired = owner.housing._reconstruct, [] + + def timing(frame, event, arg): + if frame.f_code is target.__code__ and event == "return" and not fired: + fired.append(True) + changed = json.loads(arg.projection_json) + changed["households"][0][changed["household_columns"].index("WGTP")] = ( + "9999" + ) + object.__setattr__(arg, "projection_json", owner.housing._json(changed)) + + previous = sys.getprofile() + sys.setprofile(timing) + try: + result = issue(invented) + finally: + sys.setprofile(previous) + assert fired and result.households[0].wgtp == "0010" + + +@pytest.mark.parametrize("aliases", ["csv", "native", "both"]) +@pytest.mark.parametrize("moment", ["entry", "before_call"]) +def test_actual_csv_alias_timing_cannot_supply_catalogue_values( + invented, aliases, moment +): + target = owner.literal._literal_csv_records + source, first = inspect.getsourcelines(target) + call_line = next( + first + i for i, line in enumerate(source) if "reader = csv_reader(" in line + ) + original, native, fired, calls = csv.reader, _csv.reader, [], [] + + def restore(): + csv.reader, _csv.reader = original, native + + def replacement(*args, **kwargs): + calls.append(True) + restore() + return original(*args, **kwargs) + + def timing(frame, event, arg): + if frame.f_code is target.__code__: + trigger = (moment == "entry" and event == "call") or ( + moment == "before_call" + and event == "line" + and frame.f_lineno == call_line + ) + if trigger and not fired: + fired.append(True) + if aliases in ("csv", "both"): + csv.reader = replacement + if aliases in ("native", "both"): + _csv.reader = replacement + if event == "return": + restore() + return timing + + previous = sys.gettrace() + sys.settrace(timing) + try: + if moment == "entry": + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + else: + result = issue(invented) + finally: + sys.settrace(previous) + restore() + assert fired and not calls + if moment == "before_call": + assert result.households[0].persons[0].esr == "4" + + +def test_public_constructor_has_no_authority(invented): + raw = issue(invented).to_bytes() + with pytest.raises(owner.ACSSourceCatalogueError, match="^ISSUANCE_CONSTRUCTOR$"): + owner.AuthenticatedACSSourceCatalogue(raw) + + +def test_full_source_can_exceed_unchanged_selected_reader_cap(invented, monkeypatch): + monkeypatch.setattr(owner.literal, "MAX_SELECTED_ROWS", 2) + result = issue(invented) + assert result.receipt["counts"]["people"] == 5 + assert result.receipt["counts"]["literal_batches"] == 3 + + +def test_vacancy_only_catalogue_has_no_literal_batch(invented): + invented.households[:] = [dict(invented.households[2])] + invented.people.clear() + invented.write() + result = issue(invented) + assert result.households == () + assert len(result.exclusion_ledger) == 1 + assert result.exclusion_ledger[0].persons == () + assert result.receipt["counts"]["literal_batches"] == 0 + assert result.receipt["execution"]["literal_full_scans"] == 0 + + +def test_reordered_literal_column_declaration_refuses_before_capture( + invented, monkeypatch +): + monkeypatch.setattr( + owner.literal, "READ_COLUMNS", ("SERIALNO", "SPORDER", "ESR", "MIL", "AGEP") + ) + with pytest.raises( + owner.ACSSourceCatalogueError, match="^LITERAL_CONTRACT_CHANGED$" + ): + issue(invented) + assert list(invented.snapshots.iterdir()) == [] + + +@pytest.mark.parametrize( + "interface", + ["validate", "to_bytes", "receipt", "households", "exclusion_ledger", "lineage"], +) +@pytest.mark.parametrize("moment", ["source_checks", "final_producer"]) +def test_final_source_registry_seal_all_borrows( + invented, monkeypatch, interface, moment +): + result = issue(invented) + pins = owner.housing._ARCHIVE_PINS + changed = tuple((r, n, "0" * 64, s) for r, n, d, s in pins) + fired, producers = [], [] + + def timing(frame, event, arg): + if event != "return": + return + if frame.f_code is owner._producer.__code__: + producers.append(True) + if not fired and ( + moment == "source_checks" + and frame.f_code is owner._source_checks.__code__ + or moment == "final_producer" + and frame.f_code is owner._producer.__code__ + and len(producers) == 2 + ): + fired.append(True) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", changed) + + previous = sys.getprofile() + try: + sys.setprofile(timing) + with pytest.raises( + owner.ACSSourceCatalogueError, match="SOURCE_AUTHORITY_CHANGED" + ): + value = getattr(result, interface) + if callable(value): + value() + finally: + sys.setprofile(previous) + assert fired + + +@pytest.mark.parametrize("semantic", [False, True]) +def test_import_time_bytes_and_matching_live_code_remain_authority( + invented, monkeypatch, semantic +): + source = Path(owner.__file__).read_text() + if semantic: + before = " weight,\n tuple(" + assert source.count(before) == 1 + source = source.replace(before, ' "9999",\n tuple(') + else: + source += "\n# invented drift after the real module import\n" + replacement = invented.root / "changed-catalogue.py" + replacement.write_text(source) + monkeypatch.setattr(owner, "__file__", str(replacement)) + if semantic: + code = next( + c + for c in compile( + source, str(replacement), "exec", dont_inherit=True + ).co_consts + if isinstance(c, CodeType) and c.co_name == "_household" + ) + monkeypatch.setattr(owner._household, "__code__", code) + with pytest.raises(owner.ACSSourceCatalogueError, match="PRODUCER_CODE_CHANGED"): + issue(invented) + + +@pytest.mark.parametrize("change", ["bytes", "generated_constructor"]) +def test_raw_domain_import_authority_precedes_first_issuance( + invented, monkeypatch, change +): + if change == "bytes": + replacement = invented.root / "changed-raw-types.py" + replacement.write_bytes( + Path(owner.domains.__file__).read_bytes() + b"\n# drift\n" + ) + monkeypatch.setattr(owner.domains, "__file__", str(replacement)) + else: + # Native live-code traversal explicitly skips generated code; + # the local import-time seal must still bind those actual constructors. + code = next( + c + for c in compile( + "def __init__(self, *args, **kwargs): pass", "", "exec" + ).co_consts + if isinstance(c, CodeType) + ) + monkeypatch.setattr( + owner.domains.AcsHousehold, + "__init__", + FunctionType(code, vars(owner.domains)), + ) + with pytest.raises(owner.ACSSourceCatalogueError, match="PRODUCER_CODE_CHANGED"): + issue(invented) + + +@pytest.mark.parametrize("issuance", [False, True]) +@pytest.mark.parametrize("change", ["pins", "live_code"]) +def test_final_producer_return_seal_issuance_and_borrow( + invented, monkeypatch, issuance, change +): + result = None if issuance else issue(invented) + fired, producers = [], [] + original = owner._household + + def changed(record): + return original(record) + + def timing(frame, event, arg): + if event == "return" and frame.f_code is owner._producer.__code__: + producers.append(True) + if len(producers) == 2: + fired.append(True) + if change == "pins": + monkeypatch.setattr( + owner.housing, + "_ARCHIVE_PINS", + tuple( + (r, n, "0" * 64, s) + for r, n, d, s in owner.housing._ARCHIVE_PINS + ), + ) + else: + monkeypatch.setattr(owner, "_household", changed) + + previous = sys.getprofile() + try: + sys.setprofile(timing) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) if issuance else result.households + finally: + sys.setprofile(previous) + assert fired + + +@pytest.mark.parametrize( + "shape", ["outer_list", "inner_list", "string_subclass", "bool_size"] +) +def test_source_authority_requires_deeply_immutable_primitive_pins( + invented, monkeypatch, shape +): + original = owner.housing._ARCHIVE_PINS + if shape == "outer_list": + changed = list(original) + elif shape == "inner_list": + changed = tuple(list(pin) for pin in original) + elif shape == "string_subclass": + + class MutableString(str): + pass + + changed = ((MutableString(original[0][0]), *original[0][1:]), original[1]) + else: + changed = ((*original[0][:3], True), original[1]) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", changed) + with pytest.raises(owner.ACSSourceCatalogueError, match="SOURCE_AUTHORITY_TYPE"): + issue(invented) + assert list(invented.snapshots.iterdir()) == [] diff --git a/packages/microcosm-build/tests/test_us_acs_pums.py b/packages/microcosm-build/tests/test_us_acs_pums.py index 48b656d96..1dc4edb28 100644 --- a/packages/microcosm-build/tests/test_us_acs_pums.py +++ b/packages/microcosm-build/tests/test_us_acs_pums.py @@ -8,13 +8,13 @@ import pytest from microcosm.build.serialization_dtypes import CANONICAL_STRING_DTYPE +from microcosm.build.us_runtime import acs_pums from microcosm.build.us_runtime.acs_pums import ( ACS_2024_1YR_SPINE, AcsPumsSource, build_acs_pums_unit_frame, load_acs_pums_tables, ) -from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights @@ -126,6 +126,15 @@ def _source(tmp_path: Path) -> AcsPumsSource: return AcsPumsSource(household_zip=household_zip, person_zip=person_zip) +def _axis_household(kinds: list[int]) -> pd.DataFrame: + return pd.DataFrame( + { + "SERIALNO": [f"2024HU000000{index}" for index in range(len(kinds))], + "TYPEHUGQ": kinds, + } + ) + + def _asec_shaped_frame() -> Frame: """Return one ASEC-like row sharing only structural and lineage fields.""" @@ -264,6 +273,8 @@ def test_built_acs_lineage_assembles_with_asec_without_measured_coercion( measured_wages = acs.table("person")["WAGP"].copy() raw_serials = acs.table("household")["SERIALNO"].copy() + from microcosm.build.us_runtime.spine_assembly import assemble_spines + assembled = assemble_spines( {"asec": _asec_shaped_frame(), "acs": acs}, household_mass_shares={"asec": 0.5, "acs": 0.5}, @@ -595,3 +606,227 @@ def test_load_acs_pums_tables_requires_native_mapping_columns( load_acs_pums_tables( AcsPumsSource(household_zip=household_zip, person_zip=person_zip) ) + + +def test_load_acs_pums_tables_rejects_group_quarters_with_positive_weight( + tmp_path: Path, +) -> None: + source = _source(tmp_path) + _write_csv_zip( + source.household_zip, + { + "psam_husa.csv": [ + _household( + "2024GQ0000001", + WGTP=7, + TYPEHUGQ=2, + TEN=None, + TAXAMT=None, + ) + ] + }, + ) + + with pytest.raises(ValueError, match="TYPEHUGQ/WGTP disagree"): + load_acs_pums_tables(source, chunksize=1) + + +def test_load_acs_pums_tables_rejects_housing_unit_with_zero_weight( + tmp_path: Path, +) -> None: + source = _source(tmp_path) + _write_csv_zip( + source.household_zip, + {"psam_husa.csv": [_household("2024HU0000001", WGTP=0)]}, + ) + + with pytest.raises(ValueError, match="TYPEHUGQ/WGTP disagree"): + load_acs_pums_tables(source, chunksize=1) + + +def test_build_acs_pums_unit_frame_reports_household_axis_composition( + tmp_path: Path, +) -> None: + pytest.importorskip("microunit") # sanctioned tax-unit constructor (us extra) + household_zip = tmp_path / "csv_hus.zip" + person_zip = tmp_path / "csv_pus.zip" + _write_csv_zip( + household_zip, + { + "psam_husa.csv": [ + _household("2024HU0000001", WGTP=10, NP=2), + _household("2024HU0000002", WGTP=20), + _household( + "2024GQ0000001", + WGTP=0, + TYPEHUGQ=2, + TEN=None, + TAXAMT=None, + ), + _household( + "2024GQ0000002", + WGTP=0, + TYPEHUGQ=3, + TEN=None, + TAXAMT=None, + ), + _household("2024HU0000003", WGTP=40, NP=0), + ] + }, + ) + _write_csv_zip( + person_zip, + { + "psam_pusa.csv": [ + _person("2024HU0000001", 1, 20, PWGTP=11), + _person("2024HU0000001", 2, 25, MAR=5, PWGTP=12), + _person("2024HU0000002", 1, 20, PWGTP=13), + _person("2024GQ0000001", 1, 37, MAR=5, PWGTP=99), + _person("2024GQ0000002", 1, 38, MAR=5, PWGTP=55), + ] + }, + ) + + frame, metadata = build_acs_pums_unit_frame( + AcsPumsSource(household_zip=household_zip, person_zip=person_zip) + ) + + composition = metadata["household_axis_composition"] + assert composition == { + "occupied_housing_unit_rows": 2, + "occupied_housing_unit_design_weight_total": 30.0, + "institutional_gq_person_rows": 1, + "institutional_gq_person_design_weight_total": 99.0, + "noninstitutional_gq_person_rows": 1, + "noninstitutional_gq_person_design_weight_total": 55.0, + } + # The dropped vacant housing unit is outside the described axis entirely. + assert metadata["vacant_household_rows_dropped"] == 1 + # Units conserve, and design mass conserves separately from units. + assert ( + composition["occupied_housing_unit_rows"] + + composition["institutional_gq_person_rows"] + + composition["noninstitutional_gq_person_rows"] + == metadata["household_rows"] + == frame.n("household") + == 4 + ) + assert composition["occupied_housing_unit_design_weight_total"] + composition[ + "institutional_gq_person_design_weight_total" + ] + composition["noninstitutional_gq_person_design_weight_total"] == pytest.approx( + metadata["weighted_household_population"] + ) + + +# Reached through the module, not imported by name, so this file still imports +# against a build without the helper: the four checks below then fail on their +# own instead of erroring the whole module at collection. +def test_household_axis_composition_separates_units_and_design_mass() -> None: + household = _axis_household([1, 1, 2, 3]) + weights = Weights( + np.asarray([10.0, 20.0, 99.0, 55.0]), + WeightKind.DESIGN, + ) + + composition = acs_pums._household_axis_composition(household, weights) + + assert composition == { + "occupied_housing_unit_rows": 2, + "occupied_housing_unit_design_weight_total": 30.0, + "institutional_gq_person_rows": 1, + "institutional_gq_person_design_weight_total": 99.0, + "noninstitutional_gq_person_rows": 1, + "noninstitutional_gq_person_design_weight_total": 55.0, + } + assert sum( + value for key, value in composition.items() if key.endswith("_rows") + ) == len(household) + assert sum( + value + for key, value in composition.items() + if key.endswith("_design_weight_total") + ) == pytest.approx(weights.total) + + +def test_household_axis_composition_requires_typehugq() -> None: + household = _axis_household([1]).drop(columns=["TYPEHUGQ"]) + weights = Weights(np.asarray([10.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="requires the TYPEHUGQ column"): + acs_pums._household_axis_composition(household, weights) + + +def test_household_axis_composition_refuses_unpartitioned_rows() -> None: + household = _axis_household([1, 4]) + weights = Weights(np.asarray([10.0, 20.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="does not partition the loaded rows"): + acs_pums._household_axis_composition(household, weights) + + +def test_household_axis_composition_refuses_misaligned_weights() -> None: + household = _axis_household([1, 2]) + weights = Weights(np.asarray([10.0, 99.0, 20.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="one DESIGN weight per loaded"): + acs_pums._household_axis_composition(household, weights) + + +def _serialnos(count: int) -> tuple[str, ...]: + return tuple(f"2024HU{index:07d}" for index in range(1, count + 1)) + + +def test_exact_household_ceiling_refuses_at_its_own_number(monkeypatch) -> None: + """The refusal is `<= MAX_EXACT_HOUSEHOLDS`, whatever that number is. + + Driven at a patched-down ceiling so the boundary is exercised without + building a full-source key tuple; `test_us_native_row_ceilings.py` carries + the separate assertion that the shipped number admits the 1,531,614 + households a full-source ACS selection supplies. + """ + monkeypatch.setattr(acs_pums, "MAX_EXACT_HOUSEHOLDS", 3) + accepted = _serialnos(3) + assert AcsPumsSource.snapshot_serialnos(accepted) == accepted + with pytest.raises(ValueError, match="bounded unique raw native keys"): + AcsPumsSource.snapshot_serialnos(_serialnos(4)) + with pytest.raises(ValueError, match="bounded unique raw native keys"): + AcsPumsSource.snapshot_serialnos(()) + + +def test_exact_person_row_ceiling_counts_np_not_person_records( + tmp_path: Path, monkeypatch +) -> None: + """NP is a declared count, so the ceiling is driven at full-source scale free. + + The check sums NP over the selected households before the person archive is + opened, so a two-row household table declaring 3,422,888 people -- the whole + 2024 ACS person file -- exercises the shipped ceiling without materialising + a single person row. + """ + household_zip = tmp_path / "h.zip" + person_zip = tmp_path / "p.zip" + serialnos = ("2024HU0000001", "2024HU0000002") + _write_csv_zip( + household_zip, + { + "psam_hus.csv": [ + _household(serialnos[0], NP=3_422_887), + _household(serialnos[1], NP=1), + ] + }, + ) + _write_csv_zip(person_zip, {"psam_pus.csv": [_person(serialnos[0], 1, 20)]}) + source = AcsPumsSource(household_zip, person_zip) + + # The shipped ceiling admits the whole 2024 ACS person file: the load runs + # past this check and refuses later, on the person archive it then opens. + assert acs_pums.MAX_EXACT_PERSON_ROWS > 3_422_888 + with pytest.raises(ValueError, match="NP/person row-count mismatch"): + load_acs_pums_tables(source, serialnos=serialnos) + + # Patched below that roster, the same expression refuses it. + monkeypatch.setattr(acs_pums, "MAX_EXACT_PERSON_ROWS", 3_422_887) + with pytest.raises( + ValueError, match="selected complete roster exceeds native person budget" + ): + load_acs_pums_tables(source, serialnos=serialnos) diff --git a/packages/microcosm-build/tests/test_us_acs_record_fence_scan.py b/packages/microcosm-build/tests/test_us_acs_record_fence_scan.py new file mode 100644 index 000000000..ee77c8d07 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_record_fence_scan.py @@ -0,0 +1,327 @@ +"""Byte-exact parity for the ACS record fence's chunked scan. + +Invented byte strings only: no archive, no source tree, no engine. ``_records`` +now locates record boundaries with ``bytes.find``/``bytes.count`` over 1 MiB +blocks instead of a Python loop over individual bytes. The sequence of yielded +records and the refusal it raises must be identical to the byte loop's for every +input, so every test here runs both implementations over the same bytes and +compares the whole outcome -- records and refusal together. + +``_reference_records`` below is a verbatim copy of the pre-change body. +""" + +from __future__ import annotations + +import io +import random + +import pytest + +from microcosm.build.us_runtime import acs_person_coverage_authentication as owner + +MAX_RECORD_BYTES = owner.MAX_RECORD_BYTES +MAX_CSV_HEADER_BYTES = owner.MAX_CSV_HEADER_BYTES +MAX_TOKEN_BYTES = owner.MAX_TOKEN_BYTES + + +def _reference_records(stream): + """The exact pre-change body, kept verbatim as the parity reference.""" + + record = bytearray() + quoted, pending_cr, first, token_bytes = False, False, True, 0 + cap = MAX_CSV_HEADER_BYTES + while block := stream.read(4096): + for byte in block: + if pending_cr: + if byte == 10: + owner._require(len(record) < cap, "CSV_RECORD_BYTES") + record.append(byte) + yield bytes(record) + record.clear() + first, pending_cr, token_bytes = False, False, 0 + if byte == 10: + continue + cap = MAX_CSV_HEADER_BYTES if first else MAX_RECORD_BYTES + owner._require(len(record) < cap, "CSV_RECORD_BYTES") + if byte == 44 and not quoted: + token_bytes = 0 + else: + token_bytes += 1 + owner._require(token_bytes <= MAX_TOKEN_BYTES, "CSV_TOKEN_BYTES") + record.append(byte) + if byte == 34: + quoted = not quoted + if not quoted and byte in (10, 13): + if byte == 13: + pending_cr = True + else: + yield bytes(record) + record.clear() + first, token_bytes = False, 0 + if record: + yield bytes(record) + + +def _outcome(fence, data): + """Every record the fence yielded, and the refusal code that stopped it.""" + + records, refusal = [], None + try: + for record in fence(io.BytesIO(data)): + records.append(record) + except owner.ACSCoverageAuthenticationError as error: + refusal = str(error) + return tuple(records), refusal + + +def _assert_identical(data): + reference = _outcome(_reference_records, data) + assert _outcome(owner._records, data) == reference + # The records must also reassemble the input exactly up to a refusal. + if reference[1] is None: + assert b"".join(reference[0]) == data + return reference + + +BOUNDARIES = { + "empty": b"", + "no-terminator": b"a,b,c", + "line-feed": b"a,b\nc,d\n", + "carriage-line-feed": b"a,b\r\nc,d\r\n", + "carriage-only": b"a,b\rc,d\r", + "carriage-then-byte": b"a\rb\n", + "mixed-endings": b"a\r\nb\rc\nd", + "quoted-line-feed": b'"a\nb",c\nd,e\n', + "quoted-carriage": b'"a\rb",c\nd,e\n', + "quoted-carriage-line-feed": b'"a\r\nb",c\nd,e\n', + "odd-quote-runs-on": b'a,"b\nc,d\n', + "parity-carried-across-records": b'a"b\nc"d\ne\n', + "byte-order-mark": b"\xef\xbb\xbfa,b\nc,d\n", + "trailing-carriage-at-eof": b"a,b\r", + "empty-lines": b"\n\n\n", + "carriage-carriage": b"a\r\rb\n", + "quote-inside-unquoted-field": b'a"b",c\n', + "commas-only": b",,,,\n", + "tab-and-space": b"a\t ,b\n", + "nul-byte": b"a\x00b,c\n", +} + + +@pytest.mark.parametrize("name", sorted(BOUNDARIES)) +def test_boundary_cases_fence_identically(name): + _assert_identical(BOUNDARIES[name]) + + +BLOCK = owner._SCAN_BLOCK +SPANNING = { + "record-spans-two-blocks": b"x" * (BLOCK + 500) + b"\n", + "carriage-line-feed-split-across-blocks": b"x" * (BLOCK - 1) + b"\r\ny\n", + "carriage-at-eof-on-a-block-edge": b"x" * (BLOCK - 1) + b"\r", + "carriage-then-byte-across-blocks": b"x" * (BLOCK - 1) + b"\rz\n", + "quoted-region-spans-two-blocks": b'"' + b"x" * BLOCK + b'"\ny\n', + "terminator-exactly-at-block-edge": b"x" * (BLOCK - 1) + b"\ny\n", +} + + +@pytest.mark.parametrize("name", sorted(SPANNING)) +def test_block_boundary_cases_fence_identically(name): + _assert_identical(SPANNING[name]) + + +CEILINGS = { + "first-record-one-over-its-cap": ( + b"x" * MAX_CSV_HEADER_BYTES + b"\n", + "CSV_RECORD_BYTES", + ), + "first-record-exactly-at-its-cap": ( + b"x" * (MAX_CSV_HEADER_BYTES - 1) + b"\n", + None, + ), + "later-record-token-one-over": ( + b"h\n" + b"x" * MAX_TOKEN_BYTES + b"\n", + "CSV_TOKEN_BYTES", + ), + "later-record-token-exactly-at-cap": ( + b"h\n" + b"x" * (MAX_TOKEN_BYTES - 1) + b"\n", + None, + ), + "later-record-many-short-tokens-over-cap": ( + b"h\n" + b",".join([b"y" * 1000] * 500) + b"\n", + "CSV_RECORD_BYTES", + ), + "token-ceiling-fires-before-record-ceiling": ( + b"h\n" + b"a," * 100 + b"w" * (MAX_TOKEN_BYTES + 1) + b"\n", + "CSV_TOKEN_BYTES", + ), + "unterminated-record-past-its-cap": ( + b"h\n" + b"q" * (MAX_RECORD_BYTES + 100), + "CSV_TOKEN_BYTES", + ), + "quoted-terminators-do-not-reset-the-ceiling": ( + b"h\n" + b'"' + b"z" * MAX_TOKEN_BYTES + b"\n", + "CSV_TOKEN_BYTES", + ), +} + + +@pytest.mark.parametrize("name", sorted(CEILINGS)) +def test_ceiling_cases_refuse_identically(name): + data, expected = CEILINGS[name] + assert _assert_identical(data)[1] == expected + + +def test_records_yielded_before_a_refusal_are_the_same_records(): + """A refusal stops the fence; everything it already yielded still matches.""" + + data = b"h\n" + b"good,row\n" + b"x" * (MAX_TOKEN_BYTES + 1) + b"\n" + records, refusal = _assert_identical(data) + assert refusal == "CSV_TOKEN_BYTES" + assert records == (b"h\n", b"good,row\n") + + +def test_randomised_byte_strings_fence_identically(): + """4,200 random strings over the alphabet the fence actually branches on.""" + + alphabet = b'ab,"\r\n\t 0' + generator = random.Random(20260915) + for _ in range(4200): + _assert_identical( + bytes(generator.choice(alphabet) for _ in range(generator.randrange(0, 60))) + ) + + +def test_randomised_record_shaped_inputs_fence_identically(): + """Row-shaped inputs, each ending with one of the three line endings.""" + + generator = random.Random(20260916) + for _ in range(300): + rows = [] + for _ in range(generator.randrange(0, 40)): + fields = [ + bytes( + generator.choice(b"abc0123") + for _ in range(generator.randrange(0, 6)) + ) + for _ in range(generator.randrange(1, 6)) + ] + if generator.random() < 0.2: + fields[0] = b'"' + fields[0] + b'\n"' + rows.append(b",".join(fields)) + ending = generator.choice([b"\n", b"\r\n", b"\r"]) + data = ending.join(rows) + if generator.random() < 0.7: + data += ending + _assert_identical(data) + + +def test_quote_parity_still_persists_across_records(): + """The fence never resets parity at a boundary; both implementations agree.""" + + records, refusal = _assert_identical(b'a"b\nc,d\ne"f\ng,h\n') + assert refusal is None + # The unbalanced quote swallows the next terminator, exactly as before. + assert records == (b'a"b\nc,d\ne"f\n', b"g,h\n") + + +# The suite's other fence tests drive the ceilings by monkeypatching them to +# small values, which is the only practical way to exercise the bounded paths. +# The scan must agree with the byte loop under those ceilings too -- the +# short-circuit that skips the replay is bounded by the smaller of the two live +# ceilings, not by a compile-time relationship between them. +CEILING_SETTINGS = ( + {"MAX_RECORD_BYTES": 8}, + {"MAX_RECORD_BYTES": 12}, + {"MAX_CSV_HEADER_BYTES": 4}, + {"MAX_TOKEN_BYTES": 3}, + {"MAX_TOKEN_BYTES": 2, "MAX_RECORD_BYTES": 9}, + {"MAX_RECORD_BYTES": 5, "MAX_CSV_HEADER_BYTES": 9}, + {"MAX_TOKEN_BYTES": 100_000, "MAX_RECORD_BYTES": 6}, + {"MAX_CSV_HEADER_BYTES": 100_000, "MAX_TOKEN_BYTES": 4}, +) + + +@pytest.mark.parametrize("settings", CEILING_SETTINGS, ids=repr) +def test_small_ceilings_refuse_identically(monkeypatch, settings): + for name, value in settings.items(): + monkeypatch.setattr(owner, name, value) + global MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES + previous = (MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES) + MAX_RECORD_BYTES = owner.MAX_RECORD_BYTES + MAX_CSV_HEADER_BYTES = owner.MAX_CSV_HEADER_BYTES + MAX_TOKEN_BYTES = owner.MAX_TOKEN_BYTES + try: + for data in BOUNDARIES.values(): + _assert_identical(data) + generator = random.Random(20260917) + alphabet = b'ab,"\r\n\t 0' + for _ in range(600): + _assert_identical( + bytes( + generator.choice(alphabet) + for _ in range(generator.randrange(0, 40)) + ) + ) + finally: + MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES = previous + + +def test_the_small_ceiling_case_the_first_scan_got_wrong(monkeypatch): + """A record cap below the token cap must still refuse at the record cap. + + The first version of the scan short-circuited on MAX_TOKEN_BYTES alone, + which silently skipped the record ceiling whenever the record ceiling was + the smaller of the two. This is that case, pinned. + """ + + body = b'"x\r\n\ty",z\n' + monkeypatch.setattr(owner, "MAX_RECORD_BYTES", len(body) - 1) + assert owner.MAX_RECORD_BYTES < owner.MAX_TOKEN_BYTES + with pytest.raises(owner.ACSCoverageAuthenticationError, match="CSV_RECORD_BYTES"): + list(owner._records(io.BytesIO(b"A,B\n" + body))) + + +def _exhaustive(alphabet, length): + yield b"" + current = [b""] + for _ in range(length): + current = [ + prefix + bytes((symbol,)) for prefix in current for symbol in alphabet + ] + yield from current + + +EXHAUSTIVE_SETTINGS = ( + {}, + {"MAX_TOKEN_BYTES": 2}, + {"MAX_RECORD_BYTES": 3, "MAX_CSV_HEADER_BYTES": 3}, + {"MAX_RECORD_BYTES": 4, "MAX_TOKEN_BYTES": 2}, + {"MAX_CSV_HEADER_BYTES": 2, "MAX_RECORD_BYTES": 5, "MAX_TOKEN_BYTES": 4}, +) + + +@pytest.mark.parametrize("settings", EXHAUSTIVE_SETTINGS, ids=repr) +def test_every_short_string_over_the_branching_alphabet(monkeypatch, settings): + """Exhaustive: all 9,331 strings up to five bytes over the fence's alphabet. + + Every byte the fence branches on is in the alphabet -- comma, quote, CR, LF + -- plus one ordinary byte and one whitespace byte. Under each ceiling + setting the scan and the byte loop must agree on every one of them. + """ + + for name, value in settings.items(): + monkeypatch.setattr(owner, name, value) + global MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES + previous = (MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES) + MAX_RECORD_BYTES = owner.MAX_RECORD_BYTES + MAX_CSV_HEADER_BYTES = owner.MAX_CSV_HEADER_BYTES + MAX_TOKEN_BYTES = owner.MAX_TOKEN_BYTES + checked = 0 + try: + for data in _exhaustive(b',"\r\n a', 5): + assert _outcome(owner._records, data) == _outcome( + _reference_records, data + ), data + checked += 1 + finally: + MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES = previous + assert checked == 9331 diff --git a/packages/microcosm-build/tests/test_us_acs_serial_membership.py b/packages/microcosm-build/tests/test_us_acs_serial_membership.py new file mode 100644 index 000000000..ac926e1ee --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_serial_membership.py @@ -0,0 +1,156 @@ +"""Prepared serial lookup agrees with pandas; every archive row still validates.""" + +from pathlib import Path +from zipfile import ZipFile + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import acs_pums + + +@pytest.mark.parametrize("storage", ["python", "pyarrow"]) +@pytest.mark.parametrize("na_value", [pd.NA, np.nan]) +@pytest.mark.parametrize( + "keys", + [ + frozenset(), + frozenset({"001", "nan", "é", "", "東京"}), + frozenset({"001", None, pd.NA, float("nan")}), + frozenset({"001", 1, b"nan"}), + frozenset({"001", True}), + frozenset({np.str_("001")}), + ], +) +def test_lookup_matches_pandas_for_string_storage_and_missing_policies( + storage, na_value, keys +): + values = pd.Series( + ["001", "1", "nan", None, "", "é", "東京", "missing", "001"], + dtype=pd.StringDtype(storage=storage, na_value=na_value), + index=pd.Index([9, 7, 7, 2, 1, 3, 4, 6, 0], name="source_row"), + name="SERIALNO", + ) + lookup = acs_pums._serial_lookup(keys) + pd.testing.assert_series_equal( + acs_pums._serial_isin(values, keys, lookup), values.isin(keys) + ) + + +@pytest.mark.parametrize("dtype", [object, "category"]) +def test_unreviewed_values_keep_original_pandas_membership(dtype): + values = pd.Series(["001", None, 1, True, "1"], dtype=dtype, name="SERIALNO") + keys = frozenset({"001", "1"}) + pd.testing.assert_series_equal( + acs_pums._serial_isin(values, keys, acs_pums._serial_lookup(keys)), + values.isin(keys), + ) + + +def test_prepared_lookup_is_reused_without_caching_chunk_verdicts(monkeypatch): + keys = frozenset({"0001", "0002"}) + lookup = acs_pums._serial_lookup(keys) + values = pd.Series(["0001", "absent", None], dtype="string") + expected = values.isin(keys) + + def forbid_original(*args, **kwargs): + pytest.fail("Rebuilding the all-string membership set for each chunk") + + monkeypatch.setattr(pd.Series, "isin", forbid_original) + pd.testing.assert_series_equal( + acs_pums._serial_isin(values, keys, lookup), expected + ) + values.iloc[0] = "changed" + assert acs_pums._serial_isin(values, keys, lookup).tolist() == [False] * 3 + later = pd.Series(["0002", "0001"], dtype="string") + assert acs_pums._serial_isin(later, keys, lookup).tolist() == [True, True] + + +def _archive(tmp_path, payload): + path = tmp_path / "invented.zip" + with ZipFile(path, "w") as archive: + archive.writestr("psam_pusa.csv", payload) + return path + + +def _read(path, *, chunksize=2): + return acs_pums._read_archive( + path, + member_prefix="psam_pus", + required=("SERIALNO", "AGEP"), + optional=(), + chunksize=chunksize, + valid_serials=frozenset({"001", "002", "003"}), + retained_serials=frozenset({"001", "003"}), + ) + + +@pytest.mark.parametrize("chunksize", [1, 2, 20]) +def test_archive_result_and_order_match_original_isin(tmp_path, monkeypatch, chunksize): + path = _archive(tmp_path, "SERIALNO,AGEP\n003,09\n002,42\n001,18\n003,10\n") + actual, members = _read(path, chunksize=chunksize) + monkeypatch.setattr( + acs_pums, "_serial_isin", lambda values, keys, lookup: values.isin(keys) + ) + expected, original_members = _read(path, chunksize=chunksize) + pd.testing.assert_frame_equal(actual, expected) + assert members == original_members + assert actual.SERIALNO.tolist() == ["003", "001", "003"] + assert actual.AGEP.tolist() == [9, 18, 10] + + +@pytest.mark.parametrize( + "payload", + [ + "SERIALNO,AGEP\n001,18\nBAD2,20\nBAD1,30\n", + "SERIALNO,AGEP\n001,18\n,20\n", + # Even an unselected household's age must be checked before selection. + "SERIALNO,AGEP\n001,18\n002,unknown\n", + ], +) +@pytest.mark.parametrize("chunksize", [1, 2, 20]) +def test_bad_archive_refusal_and_examples_match_original( + tmp_path, monkeypatch, payload, chunksize +): + path = _archive(tmp_path, payload) + with pytest.raises(ValueError) as actual: + _read(path, chunksize=chunksize) + monkeypatch.setattr( + acs_pums, "_serial_isin", lambda values, keys, lookup: values.isin(keys) + ) + with pytest.raises(ValueError) as original: + _read(path, chunksize=chunksize) + assert type(actual.value) is type(original.value) + assert str(actual.value) == str(original.value) + + +def test_native_owner_accepts_the_reviewed_lookup_implementation(): + from microcosm.build.us_runtime import acs_native_coverage_binding as native + + producer = native._producer() + assert ( + producer["preparation"]["modules"]["microcosm.build/us_runtime/acs_pums.py"] + == native._ACCEPTED["acs_pums.py"] + ) + + +def test_native_owner_still_refuses_changed_parser_bytes(monkeypatch): + from microcosm.build.us_runtime import acs_native_coverage_binding as native + + original = Path.read_bytes + parser_path = Path(acs_pums.__file__).resolve() + + def changed(path): + payload = original(path) + return ( + payload + b"\n# changed source\n" + if path.resolve() == parser_path + else payload + ) + + monkeypatch.setattr(Path, "read_bytes", changed) + with pytest.raises( + native.ACSNativeCoverageBindingError, match="UNREVIEWED_PREPARATION" + ): + native._producer() diff --git a/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py b/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py new file mode 100644 index 000000000..82f359abd --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py @@ -0,0 +1,611 @@ +"""Invented controls for the real ACS owner's source-only compilation cache.""" + +from __future__ import annotations +import __future__ + +import ast +import builtins +import sys +from contextlib import contextmanager +from pathlib import Path +from types import CodeType, FunctionType, ModuleType + +import pytest + +from microcosm.build.us_runtime import acs_native_coverage_binding as native + + +@pytest.fixture(autouse=True) +def empty_compile_cache(): + native._clear_compile_cache() + yield + native._clear_compile_cache() + + +@contextmanager +def _observed_calls(): + """Observe real compile/read calls without replacing attested functions.""" + assert sys.getprofile() is None + counts = {"compile": 0, "read": 0} + helper_code = native._compile_source.__code__ + read_code = Path.read_bytes.__code__ + + def observe(frame, event, arg): + if ( + event == "c_call" + and frame.f_code is helper_code + and arg is builtins.compile + ): + counts["compile"] += 1 + elif event == "call" and frame.f_code is read_code: + counts["read"] += 1 + + sys.setprofile(observe) + try: + yield counts + finally: + sys.setprofile(None) + + +def _module(tmp_path, monkeypatch, source, *, name="invented_acs_compile_cache"): + path = tmp_path / f"{name}.py" + path.write_bytes(source) + module = ModuleType(name) + module.__file__ = str(path) + module.__package__ = "" + monkeypatch.setitem(sys.modules, name, module) + exec(builtins.compile(source, str(path), "exec", dont_inherit=True), vars(module)) + return module, path + + +@contextmanager +def _declaration_parse_calls(): + assert sys.getprofile() is None + calls = [] + + def observe(frame, event, _arg): + if event == "call" and frame.f_code is native._DECLARATION_PARSER_CODE: + calls.append(None) + + sys.setprofile(observe) + try: + yield calls + finally: + sys.setprofile(None) + + +def test_declarations_cache_exact_bytes_and_keep_source_order(): + first_source = b"from math import pi as circle\ndef first(): pass\n" + second_source = b"def first(): pass\nfrom math import e as circle\n" + with _declaration_parse_calls() as calls: + first = native._source_declarations(first_source) + assert native._source_declarations(first_source) is first + second = native._source_declarations(second_source) + assert len(calls) == 2 + assert first == ( + ("import", 0, "math", (("pi", "circle"),)), + ("definition", "first"), + ) + assert second == ( + ("definition", "first"), + ("import", 0, "math", (("e", "circle"),)), + ) + with pytest.raises(TypeError): + first[0] = second[0] + + +def test_declarations_oversize_and_changed_parser_bypass(monkeypatch): + source = b"def first(): pass\n" + monkeypatch.setattr(native, "_DECLARATION_MAX_ENTRY_BYTES", 1) + with _declaration_parse_calls() as calls: + native._source_declarations(source) + native._source_declarations(source) + assert len(calls) == 2 + assert native._cached_declarations.cache_info().currsize == 0 + monkeypatch.setattr(native, "_DECLARATION_MAX_ENTRY_BYTES", 128 * 1024) + expected = native._source_declarations(source) + original = ast.parse + invoked = [] + + def changed(*args, **kwargs): + invoked.append(None) + return original(*args, **kwargs) + + monkeypatch.setattr(ast, "parse", changed) + assert native._source_declarations(source) == expected + assert native._source_declarations(source) == expected + assert len(invoked) == 2 + + +def test_declarations_changed_ast_compiler_bypasses_warm_cache(monkeypatch): + source = b"def first(): pass\n" + expected = native._source_declarations(source) + invoked = [] + + def changed(*args, **kwargs): + invoked.append(None) + return builtins.compile(*args, **kwargs) + + monkeypatch.setattr(ast, "compile", changed, raising=False) + assert native._source_declarations(source) == expected + assert native._source_declarations(source) == expected + assert len(invoked) == 2 + + +def test_declarations_changed_parser_defaults_bypass_warm_cache(monkeypatch): + source = b"def first(): pass\n" + expected = native._source_declarations(source) + with monkeypatch.context() as context: + context.setattr(ast.parse, "__defaults__", ("", "eval")) + with pytest.raises(SyntaxError): + native._source_declarations(source) + with pytest.raises(SyntaxError): + native._parse_declarations(source) + assert native._cached_declarations.cache_info().hits == 0 + assert native._source_declarations(source) is expected + + +def test_declarations_changed_keyword_type_bypasses_warm_cache(monkeypatch): + source = b"def first(): pass\n" + expected = native._source_declarations(source) + key = "optimize" if "optimize" in ast.parse.__kwdefaults__ else "feature_version" + with monkeypatch.context() as context: + context.setitem(ast.parse.__kwdefaults__, key, -1.0) + with pytest.raises(TypeError): + native._source_declarations(source) + with pytest.raises(TypeError): + native._parse_declarations(source) + assert native._cached_declarations.cache_info().hits == 0 + # False and 0 compare equal, but changing the exact default type still bypasses. + with monkeypatch.context() as context, _declaration_parse_calls() as calls: + context.setitem(ast.parse.__kwdefaults__, "type_comments", 0) + assert native._source_declarations(source) == expected + assert native._source_declarations(source) == expected + assert len(calls) == 2 + assert native._cached_declarations.cache_info().hits == 0 + + +@pytest.mark.parametrize("flag", ["PyCF_ONLY_AST", "PyCF_TYPE_COMMENTS"]) +def test_declarations_changed_parser_flags_bypass_warm_cache(monkeypatch, flag): + source = b"def first(): pass\n" + expected = native._source_declarations(source) + with monkeypatch.context() as context, _declaration_parse_calls() as calls: + context.setattr(ast, flag, 0) + if flag == "PyCF_ONLY_AST": + with pytest.raises(AttributeError): + native._source_declarations(source) + with pytest.raises(AttributeError): + native._parse_declarations(source) + else: + assert native._source_declarations(source) == expected + assert native._source_declarations(source) == expected + assert len(calls) == 2 + assert native._cached_declarations.cache_info().hits == 0 + + +def test_declaration_failures_do_not_stick_and_cache_is_bounded(): + with _declaration_parse_calls() as calls: + for _ in range(2): + with pytest.raises(SyntaxError): + native._source_declarations(b"def invalid(:") + assert len(calls) == 2 + assert native._cached_declarations.cache_info().currsize == 0 + for index in range(130): + native._source_declarations(f"def value_{index}(): pass\n".encode()) + assert native._cached_declarations.cache_info().currsize == 128 + native._clear_compile_cache() + assert native._cached_declarations.cache_info().currsize == 0 + + +def test_warm_declarations_still_refuse_changed_live_alias(tmp_path, monkeypatch): + module, path = _module(tmp_path, monkeypatch, b"from math import pi as circle\n") + with _declaration_parse_calls() as calls: + native._live_code(module, {}) + native._live_code(module, {}) + assert len(calls) == 1 + module.circle = 0 + with pytest.raises(native.ACSNativeCoverageBindingError, match="LOADED_PRODUCER"): + native._live_code(module, {}) + # A changed source declaration is read again even with the old entry warm. + path.write_bytes(b"def missing(): pass\n") + with pytest.raises(native.ACSNativeCoverageBindingError, match="LOADED_PRODUCER"): + native._live_code(module, {}) + + +def _index(code): + result = {} + + def visit(item): + result[item.co_qualname] = item + for value in item.co_consts: + if isinstance(value, CodeType): + visit(value) + + visit(code) + return result + + +@pytest.mark.parametrize("optimize", [0, 1, 2]) +def test_plain_helper_preserves_compiled_code_and_qualified_name_order(optimize): + source = """# coding: utf-8 +from __future__ import annotations +label = "café" +def outer(value: Missing): + "docstring" + assert value is not None + def nested(): + return [item for item in range(value)] + def nested(): + return lambda: {item: item for item in range(value)} + return nested +class Example: + @property + def item(self): + return label + @item.setter + def item(self, value): + self._item = value +""".encode() + filename = "invented/café/../module.py" + expected = builtins.compile( + source, filename, "exec", dont_inherit=True, optimize=optimize + ) + with _observed_calls() as calls: + first = native._compile_source(source, filename, optimize=optimize) + second = native._compile_source(source, filename, optimize=optimize) + assert type(native._compile_source) is FunctionType + assert first is second + assert first == expected + assert first.co_filename == filename + assert _index(first) == _index(expected) + assert calls["compile"] == 1 + + +def test_complete_inputs_separate_cache_entries(): + source = b"42" + options = [ + ("literal/../source.py", "exec", 0, 0), + ("source.py", "exec", 0, 0), + ("source.py", "eval", 0, 0), + ("source.py", "exec", __future__.annotations.compiler_flag, 0), + ("source.py", "exec", 0, 1), + ("source.py", "exec", 0, 2), + ] + with _observed_calls() as calls: + for filename, mode, flags, optimize in options: + expected = builtins.compile( + source, + filename, + mode, + flags=flags, + dont_inherit=True, + optimize=optimize, + ) + first = native._compile_source( + source, filename, mode, flags=flags, optimize=optimize + ) + assert first == expected + assert first.co_filename == filename + assert ( + native._compile_source( + source, filename, mode, flags=flags, optimize=optimize + ) + is first + ) + assert calls["compile"] == len(options) + assert len(native._COMPILE_CACHE) == len(options) + with _observed_calls() as calls: + effective = native._compile_source( + source, "source.py", optimize=sys.flags.optimize + ) + assert native._compile_source(source, "source.py", optimize=-1) is effective + assert calls["compile"] == 0 + + +def test_same_literal_path_tracks_a_b_a_source_bytes(): + first_source, second_source = b"value = 1", b"value = 2" + assert len(first_source) == len(second_source) + with _observed_calls() as calls: + first = native._compile_source(first_source, "unchanged.py") + second = native._compile_source(second_source, "unchanged.py") + again = native._compile_source(first_source, "unchanged.py") + assert first is again + assert second != first + assert calls["compile"] == 2 + + +def test_changed_compiler_is_called_each_time_and_original_entry_remains(monkeypatch): + source = b"value = 1" + original = native._compile_source(source, "compiler.py") + replacements = [] + + def replacement(raw, filename, mode, **options): + replacements.append(raw) + return builtins.compile( + f"value = {len(replacements) + 1}".encode(), filename, mode, **options + ) + + with monkeypatch.context() as context: + context.setattr(native, "compile", replacement, raising=False) + first = native._compile_source(source, "compiler.py") + second = native._compile_source(source, "compiler.py") + assert replacements == [source, source] + assert first != second + assert first != original + assert len(native._COMPILE_CACHE) == 1 + assert native._compile_source(source, "compiler.py") is original + + +@pytest.mark.parametrize( + "options", [{"dont_inherit": False}, {"flags": ast.PyCF_ONLY_AST}] +) +def test_context_dependent_or_mutable_compiler_outputs_are_not_retained(options): + with _observed_calls() as calls: + first = native._compile_source(b"value = 1", "bypass.py", **options) + second = native._compile_source(b"value = 1", "bypass.py", **options) + assert first is not second + assert calls["compile"] == 2 + assert not native._COMPILE_CACHE + + +def test_failures_are_not_memoized(): + with _observed_calls() as calls: + for _ in range(2): + with pytest.raises(SyntaxError): + native._compile_source(b"def invalid(:", "repair.py") + assert not native._COMPILE_CACHE + fixed = native._compile_source(b"value = 1", "repair.py") + assert calls["compile"] == 3 + assert type(fixed) is CodeType + assert len(native._COMPILE_CACHE) == 1 + + +def test_fifo_entry_limit_and_clear(monkeypatch): + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_ENTRIES", 2) + source = b"value = 1" + first = native._compile_source(source, "first.py") + second = native._compile_source(source, "second.py") + assert native._compile_source(source, "first.py") is first + native._compile_source(source, "third.py") + assert len(native._COMPILE_CACHE) == 2 + assert native._compile_source(source, "second.py") is second + with _observed_calls() as calls: + assert native._compile_source(source, "first.py") is not first + assert calls["compile"] == 1 + native._clear_compile_cache() + assert not native._COMPILE_CACHE + with _observed_calls() as calls: + native._compile_source(source, "first.py") + assert calls["compile"] == 1 + + +def test_source_byte_budget_counts_distinct_filename_keys(monkeypatch): + source = b"value = 1" + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_SOURCE_BYTES", len(source) * 2) + first = native._compile_source(source, "first.py") + native._compile_source(source, "second.py") + native._compile_source(source, "third.py") + assert sum(len(key[0]) for key in native._COMPILE_CACHE) == len(source) * 2 + with _observed_calls() as calls: + assert native._compile_source(source, "first.py") is not first + assert calls["compile"] == 1 + + +@pytest.mark.parametrize( + "limit_name", ["_COMPILE_CACHE_MAX_ENTRY_BYTES", "_COMPILE_CACHE_MAX_SOURCE_BYTES"] +) +def test_oversize_source_is_compiled_without_cache(limit_name, monkeypatch): + source = b"value = 1" + monkeypatch.setattr(native, limit_name, len(source) - 1) + with _observed_calls() as calls: + first = native._compile_source(source, "oversize.py") + second = native._compile_source(source, "oversize.py") + assert first == second + assert first is not second + assert calls["compile"] == 2 + assert not native._COMPILE_CACHE + + +def test_live_check_reads_source_twice_even_after_warming(tmp_path, monkeypatch): + module, _ = _module(tmp_path, monkeypatch, b"def value():\n return 1\n") + with _observed_calls() as cold: + first_index = {} + native._live_code(module, first_index) + with _observed_calls() as warm: + second_index = {} + native._live_code(module, second_index) + assert cold == {"compile": 1, "read": 2} + assert warm == {"compile": 0, "read": 2} + assert first_index == second_index + assert first_index is not second_index + first_index[module.__file__]["value"] = None + native._live_code(module, {}) + assert second_index[module.__file__]["value"] is not None + + +def test_real_producer_keeps_read_counts_and_full_evidence_on_warm_cache(): + # Settle unrelated import/manifest memoization, then compare only compile + # cache cold versus warm with the real producer and unchanged source files. + native._producer() + native._clear_compile_cache() + with _observed_calls() as cold: + first = native._producer() + with _observed_calls() as warm: + second = native._producer() + assert cold["compile"] > 0 + assert warm["compile"] == 0 + assert cold["read"] > 0 + assert warm["read"] == cold["read"] + assert first == second + assert len(native._COMPILE_CACHE) <= native._COMPILE_CACHE_MAX_ENTRIES + assert ( + sum(len(key[0]) for key in native._COMPILE_CACHE) + <= native._COMPILE_CACHE_MAX_SOURCE_BYTES + ) + + +@pytest.mark.parametrize("drift", ["code", "globals", "declaration"]) +def test_warm_cache_does_not_hide_loaded_function_drift(drift, tmp_path, monkeypatch): + module, _ = _module(tmp_path, monkeypatch, b"def value():\n return 1\n") + native._live_code(module, {}) + if drift == "code": + module.value.__code__ = module.value.__code__.replace(co_consts=(None, 2)) + elif drift == "globals": + module.value = FunctionType(module.value.__code__, dict(vars(module))) + else: + del module.value + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + + +def test_warm_cache_preserves_imported_alias_identity(tmp_path, monkeypatch): + origin, _ = _module( + tmp_path, + monkeypatch, + b"def value():\n return 1\n", + name="invented_acs_origin", + ) + consumer, _ = _module( + tmp_path, + monkeypatch, + b"from invented_acs_origin import value\n", + name="invented_acs_consumer", + ) + native._live_code(consumer, {}) + consumer.value = FunctionType(origin.value.__code__, vars(origin)) + assert consumer.value.__code__ is origin.value.__code__ + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(consumer, {}) + + +def test_warm_cache_checks_distinct_functions_with_shared_code_and_closures( + tmp_path, monkeypatch +): + source = b"""def make(): + def target(): + return 1 + def entry(): + return target() + return entry +first = make() +second = make() +""" + module, _ = _module(tmp_path, monkeypatch, source) + native._live_code(module, {}) + assert module.first.__code__ is module.second.__code__ + first_target = module.first.__closure__[0].cell_contents + second_target = module.second.__closure__[0].cell_contents + assert first_target is not second_target + second_target.__code__ = second_target.__code__.replace(co_consts=(None, 2)) + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + + +def test_second_source_read_can_invalidate_a_warm_entry(tmp_path, monkeypatch): + original = b"def value():\n return 1\n" + changed = b"def value():\n return 2\n" + module, path = _module(tmp_path, monkeypatch, original) + native._live_code(module, {}) + read_bytes = Path.read_bytes + calls = 0 + + def change_on_second_read(self): + nonlocal calls + if self == path: + calls += 1 + if calls == 2: + path.write_bytes(changed) + return read_bytes(self) + + monkeypatch.setattr(Path, "read_bytes", change_on_second_read) + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + assert calls == 2 + + +def test_malformed_cache_entry_is_recompiled_and_replaced(): + source = b"value = 1" + first = native._compile_source(source, "malformed.py") + key = next(iter(native._COMPILE_CACHE)) + native._COMPILE_CACHE[key] = None + with _observed_calls() as calls: + repaired = native._compile_source(source, "malformed.py") + again = native._compile_source(source, "malformed.py") + assert repaired == first + assert repaired is not first + assert again is repaired + assert calls["compile"] == 1 + + +def test_nested_compilation_keeps_current_fifo_bound(monkeypatch): + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_ENTRIES", 1) + assert sys.getprofile() is None + nested = [] + helper_code = native._compile_source.__code__ + + def after_compile(frame, event, arg): + if ( + not nested + and event == "c_return" + and frame.f_code is helper_code + and arg is builtins.compile + ): + nested.append(True) + native._compile_source(b"nested = 2", "nested.py") + + sys.setprofile(after_compile) + try: + outer = native._compile_source(b"outer = 1", "outer.py") + finally: + sys.setprofile(None) + assert nested == [True] + assert outer == builtins.compile( + b"outer = 1", "outer.py", "exec", dont_inherit=True + ) + assert len(native._COMPILE_CACHE) == 1 + assert next(iter(native._COMPILE_CACHE))[1] == "outer.py" + + +def test_swapped_valid_entries_are_recompiled_for_their_complete_inputs(): + first_source, second_source = b"value = 1", b"value = 2" + first = native._compile_source(first_source, "first.py") + second = native._compile_source(second_source, "second.py") + first_key, second_key = tuple(native._COMPILE_CACHE) + native._COMPILE_CACHE[first_key], native._COMPILE_CACHE[second_key] = ( + native._COMPILE_CACHE[second_key], + native._COMPILE_CACHE[first_key], + ) + with _observed_calls() as calls: + repaired_first = native._compile_source(first_source, "first.py") + repaired_second = native._compile_source(second_source, "second.py") + assert native._compile_source(first_source, "first.py") is repaired_first + assert native._compile_source(second_source, "second.py") is repaired_second + assert calls["compile"] == 2 + assert repaired_first == first and repaired_first is not first + assert repaired_second == second and repaired_second is not second + assert repaired_first.co_filename == "first.py" + assert repaired_second.co_filename == "second.py" + + +def test_python_compiler_captured_as_initial_compiler_still_bypasses(monkeypatch): + # This is the exact captured/current state after a pre-import replacement; + # leave all issued-owner modules loaded and exercise the real cache helper. + calls = [] + + def replacement(source, filename, mode, **options): + calls.append(source) + return builtins.compile( + f"value = {len(calls)}".encode(), filename, mode, **options + ) + + # Textual labels alone must not classify a Python replacement as builtin. + replacement.__module__ = "builtins" + replacement.__name__ = "compile" + replacement.__self__ = builtins + monkeypatch.setattr(native, "_COMPILE_CACHE_COMPILER", replacement) + monkeypatch.setattr(native, "compile", replacement, raising=False) + first = native._compile_source(b"value = 0", "captured.py") + second = native._compile_source(b"value = 0", "captured.py") + assert calls == [b"value = 0", b"value = 0"] + assert first != second + assert not native._COMPILE_CACHE diff --git a/packages/microcosm-build/tests/test_us_acs_spm_assembler.py b/packages/microcosm-build/tests/test_us_acs_spm_assembler.py new file mode 100644 index 000000000..2c59bf2bd --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_spm_assembler.py @@ -0,0 +1,442 @@ +"""Assembler capability probe, its refusal, and the observed role label. + +The capability branches no real runtime exhibits are driven by synthetic stub +assemblers installed in ``sys.modules``, so every branch is exercised wherever +this file runs. The tests that need the actual imported assembler carry the +same strict conditional marker as the partition suite. +""" + +import dataclasses +import hashlib +import json +import sys +import types +from pathlib import Path + +import pandas as pd +import pytest + +from microcosm.build.acs_spm_partition import ( + ACS_SPM_DEVELOPMENT_POLICY, + AcsSpmAssemblerProbe, + AcsSpmRoleDecision, + UnsupportedAssembler, + probe_acs_spm_assembler, + reconstruct_acs_spm_partition, +) + +REASONS = { + "supported", + "assembler_unavailable", + "incompatible_parent_link_order", + "incompatible_foster_boundary", + "incompatible_call_contract", +} +# The probe's own roster, pinned here so a change to it has to be reviewed. +PROBE_IDS = [1, 2, 3, 4, 5, 6] +CORRECT = {1: 0, 2: 1, 3: 1, 4: 2, 5: 2, 6: 3} +# The installed 1.0.0 ordering: the residual under-15 reaches the reference unit +# before the accepted parent link is applied, so household 1 collapses. +MERGED_PARENT = {1: 0, 2: 0, 3: 0, 4: 2, 5: 2, 6: 3} +# The foster boundary moved by one year: the 21-year-old leaves the reference +# unit and pools with the 22-year-old instead. +SPLIT_FOSTER = {1: 0, 2: 1, 3: 1, 4: 2, 5: 3, 6: 3} +BOTH_WRONG = {1: 0, 2: 0, 3: 0, 4: 2, 5: 3, 6: 3} +GOOD_DIAGNOSTICS = { + "method": "inferred", + "fallback_rules_used": ["link_parent_child"], +} + +requires_assembler = pytest.mark.xfail( + not probe_acs_spm_assembler().supported, + strict=True, + raises=UnsupportedAssembler, + reason="Assembled membership requires the reviewed canonical assembler", +) + + +def fake(mapping, *, info=GOOD_DIAGNOSTICS, calls=None): + """A synthetic assembler: fixed units for the probe roster, singletons else.""" + + def spm_unit_id(persons, *, diagnostics=False, **_): + if calls is not None: + calls.append(persons.copy(deep=True)) + ids = pd.Series( + [mapping[pid] for pid in persons.person_id] + if list(persons.person_id) == PROBE_IDS + else range(len(persons)), + dtype="int64", + ) + return (ids, info) if diagnostics else ids + + return spm_unit_id + + +def install(monkeypatch, spm_unit_id): + module = types.ModuleType("spm_calculator") + module.spm_unit_id = spm_unit_id + monkeypatch.setitem(sys.modules, "spm_calculator", module) + + +def roster(relationships, ages, *, marital=None, gq=False): + n = len(ages) + if marital is None: + marital = [1 if rel in {20, 21, 23} else 5 for rel in relationships] + return pd.DataFrame( + { + "person_id": range(101, 101 + n), + "person_household_id": [10] * n, + "person_spm_unit_id": [700] * n, + "SPORDER": range(1, n + 1), + "RELSHIPP": relationships, + "AGEP": ages, + "MAR": marital, + "TYPEHUGQ": [2 if gq else 1] * n, + } + ) + + +def assemble(persons, **kwargs): + old_units = pd.DataFrame( + {"spm_unit_id": persons.person_spm_unit_id.unique(), "snap": 123.0} + ) + return reconstruct_acs_spm_partition( + persons, + old_units, + household_person_counts=persons.groupby("person_household_id").size().to_dict(), + **kwargs, + ) + + +def test_probe_verdict_is_frozen_typed_and_json_safe(): + probe = probe_acs_spm_assembler() + assert isinstance(probe, AcsSpmAssemblerProbe) + assert probe.reason in REASONS + assert probe.supported is (probe.reason == "supported") + with pytest.raises(dataclasses.FrozenInstanceError): + probe.supported = True + provenance = probe.as_provenance() + assert set(provenance) == { + "supported", + "reason", + "module_file", + "module_sha256", + } + assert json.loads(json.dumps(provenance)) == provenance + + +def test_probe_binds_the_actual_assembler_file_hash(): + probe = probe_acs_spm_assembler() + if probe.reason == "assembler_unavailable": + assert probe.module_file is None and probe.module_sha256 is None + return + if probe.module_file is None: + assert probe.module_sha256 is None + return + digest = hashlib.sha256(Path(probe.module_file).read_bytes()).hexdigest() + assert probe.module_sha256 == digest + + +def test_probe_reports_an_absent_assembler_without_raising(monkeypatch): + monkeypatch.setitem(sys.modules, "spm_calculator", None) + probe = probe_acs_spm_assembler() + assert probe == AcsSpmAssemblerProbe(False, "assembler_unavailable", None, None) + + +def test_probe_is_re_evaluated_rather_than_cached(monkeypatch): + baseline = probe_acs_spm_assembler() + with monkeypatch.context() as patched: + patched.setitem(sys.modules, "spm_calculator", None) + assert probe_acs_spm_assembler().reason == "assembler_unavailable" + assert probe_acs_spm_assembler() == baseline + + +def test_probe_records_the_stub_it_actually_exercised(monkeypatch): + install(monkeypatch, fake(CORRECT)) + probe = probe_acs_spm_assembler() + assert probe.supported and probe.reason == "supported" + # ``fake`` is defined in this file, so the probe must name this file. + assert Path(probe.module_file) == Path(__file__) + assert ( + probe.module_sha256 == hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + ) + + +@pytest.mark.parametrize( + "mapping,reason", + [ + (CORRECT, "supported"), + (MERGED_PARENT, "incompatible_parent_link_order"), + (SPLIT_FOSTER, "incompatible_foster_boundary"), + # Both capabilities wrong: the parent-order verdict is reported first. + (BOTH_WRONG, "incompatible_parent_link_order"), + ], +) +def test_probe_names_the_capability_that_diverged(monkeypatch, mapping, reason): + install(monkeypatch, fake(mapping)) + probe = probe_acs_spm_assembler() + assert probe.reason == reason + assert probe.supported is (reason == "supported") + + +def raises_on_call(persons, **_): + raise RuntimeError("synthetic assembler failure") + + +@pytest.mark.parametrize( + "spm_unit_id", + [ + pytest.param(raises_on_call, id="raises"), + pytest.param(len, id="unreadable-builtin"), + pytest.param( + lambda persons, **_: pd.Series(range(len(persons))), id="not-a-pair" + ), + pytest.param( + lambda persons, **_: (pd.Series(range(len(persons))), None, None), + id="three-tuple", + ), + pytest.param( + lambda persons, **_: (pd.Series([0]), GOOD_DIAGNOSTICS), + id="short-ids", + ), + pytest.param( + lambda persons, **_: ( + pd.Series([pd.NA] * len(persons), dtype="Int64"), + GOOD_DIAGNOSTICS, + ), + id="missing-ids", + ), + pytest.param(fake(CORRECT, info=["not", "a", "mapping"]), id="no-mapping"), + pytest.param(fake(CORRECT, info={1: "not a string key"}), id="nonstring-key"), + pytest.param(fake(CORRECT, info={"fallback_rules_used": []}), id="no-method"), + pytest.param( + fake(CORRECT, info={"method": 7, "fallback_rules_used": []}), + id="nonstring-method", + ), + pytest.param( + fake( + CORRECT, + info={"method": "native_spm_id", "fallback_rules_used": []}, + ), + id="echoed-native-id", + ), + pytest.param(fake(CORRECT, info={"method": "inferred"}), id="no-rules"), + pytest.param( + fake(CORRECT, info={"method": "inferred", "fallback_rules_used": "link"}), + id="rules-are-a-string", + ), + pytest.param( + fake(CORRECT, info={"method": "inferred", "fallback_rules_used": [7]}), + id="nonstring-rule", + ), + ], +) +def test_probe_reports_call_contract_failures_without_raising(monkeypatch, spm_unit_id): + install(monkeypatch, spm_unit_id) + probe = probe_acs_spm_assembler() + assert probe == AcsSpmAssemblerProbe( + False, "incompatible_call_contract", probe.module_file, probe.module_sha256 + ) + + +def test_probe_identity_is_omitted_when_the_assembler_file_is_unreadable( + monkeypatch, +): + install(monkeypatch, len) + probe = probe_acs_spm_assembler() + assert probe.module_file is None and probe.module_sha256 is None + + +def test_probe_sends_the_same_view_shape_the_partition_sends(monkeypatch): + calls = [] + install(monkeypatch, fake(CORRECT, calls=calls)) + assemble(roster([20, 21, 25], [45, 44, 10])) + probed, actual = calls[0], calls[-1] + assert list(probed.person_id) == PROBE_IDS + assert list(probed.columns) == list(actual.columns) + assert probed.dtypes.astype(str).tolist() == actual.dtypes.astype(str).tolist() + # The probe must not hand the assembler evidence the adapter withholds. + assert not {"person_spm_unit_id", "spm_unit_id", "SPM_ID", "A_EXPRRP"} & set( + probed.columns + ) + + +@pytest.mark.parametrize( + "reason", + [ + "assembler_unavailable", + "incompatible_parent_link_order", + "incompatible_foster_boundary", + "incompatible_call_contract", + ], +) +def test_unsupported_probe_refuses_with_the_dedicated_exception(monkeypatch, reason): + probe = AcsSpmAssemblerProbe(False, reason, None, None) + monkeypatch.setattr( + "microcosm.build.acs_spm_partition.probe_acs_spm_assembler", lambda: probe + ) + with pytest.raises(UnsupportedAssembler) as excinfo: + assemble(roster([20, 21, 25], [45, 44, 10])) + assert excinfo.value.probe is probe + assert reason in str(excinfo.value) + + +def test_unsupported_assembler_is_not_a_value_error(): + assert issubclass(UnsupportedAssembler, Exception) + assert not issubclass(UnsupportedAssembler, ValueError) + + +def test_source_evidence_is_validated_before_the_assembler_refusal(monkeypatch): + monkeypatch.setattr( + "microcosm.build.acs_spm_partition.probe_acs_spm_assembler", + lambda: AcsSpmAssemblerProbe(False, "assembler_unavailable", None, None), + ) + persons = roster([20, 21], [45, 44]) + persons["RELSHIPP"] = [20, 20] + # A refusable roster must still refuse the same way on every runtime. + with pytest.raises(ValueError, match="reference person"): + assemble(persons) + + +@pytest.mark.parametrize( + "decisions", + [ + [AcsSpmRoleDecision(101, False, "head_override")], + [AcsSpmRoleDecision(999, True, "absent_person")], + [AcsSpmRoleDecision(102, 1, "not_bool")], + [AcsSpmRoleDecision(102, True, "")], + [AcsSpmRoleDecision(102, True, "one"), AcsSpmRoleDecision(102, False, "two")], + [object()], + ], +) +def test_invalid_role_decisions_refuse_before_any_assembler_probe( + monkeypatch, decisions +): + def must_not_probe(): + pytest.fail("Invalid source decisions reached the assembler probe") + + monkeypatch.setattr( + "microcosm.build.acs_spm_partition.probe_acs_spm_assembler", must_not_probe + ) + persons = roster([20, 25], [45, 16]) + with pytest.raises(ValueError): + assemble(persons, role_decisions=decisions) + + +@pytest.mark.parametrize("age,gq", [(14, False), (18, False), (17, True)]) +def test_inapplicable_role_decisions_refuse_without_assembler(monkeypatch, age, gq): + def must_not_probe(): + pytest.fail("Inapplicable source decision reached the assembler probe") + + monkeypatch.setattr( + "microcosm.build.acs_spm_partition.probe_acs_spm_assembler", must_not_probe + ) + persons = roster([37, 37] if gq else [20, 25], [45, age], gq=gq) + with pytest.raises(ValueError, match="15–17"): + assemble( + persons, role_decisions=[AcsSpmRoleDecision(102, True, "outside_scope")] + ) + + +@pytest.mark.parametrize("kind", ["group-quarters", "unresolved"]) +def test_no_refusal_when_no_household_reaches_the_assembler(monkeypatch, kind): + monkeypatch.setattr( + "microcosm.build.acs_spm_partition.probe_acs_spm_assembler", + lambda: AcsSpmAssemblerProbe(False, "assembler_unavailable", None, None), + ) + persons = ( + roster([37, 37], [17, 40], gq=True) + if kind == "group-quarters" + else roster([20, 34, 36], [45, 35, 8]) + ) + if kind == "group-quarters": + persons["person_spm_unit_id"] = [700, 701] + result = assemble(persons) + assert result.provenance["assembler"] is None + assert result.provenance["canonical_diagnostics"] is None + + +@requires_assembler +def test_partition_provenance_binds_the_actual_probe_identity(): + provenance = assemble(roster([20, 21, 25], [45, 44, 10])).provenance + assert provenance["assembler"] == probe_acs_spm_assembler().as_provenance() + assert provenance["assembler"]["supported"] is True + assert json.loads(json.dumps(provenance["assembler"])) == provenance["assembler"] + + +@requires_assembler +def test_observed_head_and_spouse_carry_the_relationship_rule_role_label(): + membership = assemble(roster([20, 21, 25, 36], [45, 44, 10, 16])).membership + assert membership.role_source.tolist() == [ + "observed_relationship_rule", + "observed_relationship_rule", + "age_not_role_sensitive", + "unresolved", + ] + assert "source_observed" not in set(membership.role_source) + assert membership.role_rule.iloc[0] == "acs_relshipp_reference_head_or_spouse" + + +@requires_assembler +def test_observed_link_source_label_survives_the_role_rename(): + result = assemble(roster([20, 21, 25], [45, 44, 10])) + assert set(result.links.source) == {"source_observed"} + assert result.membership.parent_link_status.iloc[2] == "observed_parent_link" + assert set(result.membership.role_source) <= { + "observed_relationship_rule", + "approved_inference", + "modeled_assumption", + "unresolved", + "age_not_role_sensitive", + "outside_acs_household_universe", + } + + +@requires_assembler +def test_development_modeled_roles_are_not_relabelled_as_observed(): + membership = assemble( + roster([20, 36, 34], [45, 16, 17]), policy=ACS_SPM_DEVELOPMENT_POLICY + ).membership + assert membership.role_source.tolist() == [ + "observed_relationship_rule", + "modeled_assumption", + "modeled_assumption", + ] + + +def test_group_quarters_roles_stay_null_and_unlabelled_on_any_runtime(): + persons = roster([37, 37], [17, 40], gq=True) + persons["person_spm_unit_id"] = [700, 701] + membership = assemble(persons).membership + assert membership.independent_minor_role.isna().all() + assert membership.role_source.eq("outside_acs_household_universe").all() + + +def test_probe_refuses_units_that_cross_households(monkeypatch): + # Correct components within each household do not permit shared IDs across them. + install(monkeypatch, fake({1: 0, 2: 1, 3: 1, 4: 0, 5: 0, 6: 1})) + assert probe_acs_spm_assembler().reason == "incompatible_call_contract" + + +def test_probe_classifies_a_broken_import_separately_from_absence(monkeypatch): + import builtins + + original = builtins.__import__ + + def broken_import(name, *args, **kwargs): + if name == "spm_calculator": + raise RuntimeError("synthetic broken import") + return original(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", broken_import) + assert probe_acs_spm_assembler().reason == "incompatible_call_contract" + + +def test_probe_returns_typed_refusal_if_roster_construction_fails(monkeypatch): + install(monkeypatch, fake(CORRECT)) + + def broken_frame(*args, **kwargs): + raise ValueError("synthetic DataFrame failure") + + monkeypatch.setattr(pd, "DataFrame", broken_frame) + assert probe_acs_spm_assembler().reason == "incompatible_call_contract" diff --git a/packages/microcosm-build/tests/test_us_acs_spm_native_construction.py b/packages/microcosm-build/tests/test_us_acs_spm_native_construction.py new file mode 100644 index 000000000..8f6832621 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_spm_native_construction.py @@ -0,0 +1,327 @@ +"""Invented archives and real issuers; no population files or country imports.""" + +import builtins +import copy +import json +from dataclasses import asdict +from fractions import Fraction +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_acs_person_coverage_authentication import ( + Member, + _csv, + _household, + _person, + build_fixture, +) + +from microcosm.build import acs_spm_partition as partition +from microcosm.build.acs_spm_source_assembly import ( + AcsSpmSourceAssemblyOptions, + require_acs_spm_source_capability, +) +from microcosm.build.us_runtime import acs_native_coverage_binding as native +from microcosm.build.us_runtime import survey_population_preparation as preparation + +OPTIONS = AcsSpmSourceAssemblyOptions(partition.ACS_SPM_DEVELOPMENT_POLICY, True) +requires_assembler = pytest.mark.xfail( + not partition.probe_acs_spm_assembler().supported, + raises=partition.UnsupportedAssembler, + strict=True, + reason="Explicit native construction requires the reviewed canonical assembler", +) + + +def fixture(tmp_path, monkeypatch): + monkeypatch.setattr( + native.housing.shutil, + "disk_usage", + lambda _: SimpleNamespace(free=64 * 1024**3), + ) + households = [ + _household("2024HU0000001", NP=3, TEN=3), + _household("2024GQ0000001", NP=1, TYPEHUGQ=2, WGTP=0, TEN=""), + ] + people = [ + _person("2024HU0000001", 1, 20, AGEP=45, MIL="4", ESR="6"), + _person("2024HU0000001", 2, 36, AGEP=16, MIL="", ESR=""), + _person("2024HU0000001", 3, 35, AGEP=14, MIL="", ESR=""), + _person("2024GQ0000001", 1, 37, AGEP=40, MIL="4", ESR="6", PWGTP=77), + ] + value = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", _csv(households)),), + person_members=(Member("psam_pusa.csv", _csv(people)),), + ) + return dict( + source_dir=value.source_dir, + snapshot_root=value.snapshot_root, + serialnos=tuple(sorted(row["SERIALNO"] for row in households)), + ) + + +def request(path, options=OPTIONS): + document = json.loads(path.read_bytes()) + document["protocol"] = preparation.SPM_REQUEST_PROTOCOL + document["acs_spm_construction"] = asdict(options) + path.write_bytes(preparation._encode(document, preparation.MAX_REQUEST_BYTES)) + return document + + +def rows(table): + return pd.DataFrame(table["rows"], columns=table["columns"]) + + +def test_default_native_route_never_imports_optional_assembler(tmp_path, monkeypatch): + arguments = fixture(tmp_path, monkeypatch) + original = builtins.__import__ + + def guarded(name, *args, **kwargs): + assert not name.startswith("spm_calculator") + return original(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded) + result = native.issue_acs_native_coverage(**arguments) + assert result.construction_evidence is None + assert "spm_construction" not in result.receipt + + +@requires_assembler +def test_real_native_owner_retains_actual_rows_and_conservation(tmp_path, monkeypatch): + require_acs_spm_source_capability(OPTIONS) + arguments = fixture(tmp_path, monkeypatch) + original = native.issue_acs_native_coverage(**arguments) + result = native.issue_acs_native_coverage(**arguments, spm_construction=OPTIONS) + evidence = result.construction_evidence + frame, before = result.frame, original.frame + assert "acs_spm_source_assembly" not in frame.metadata + assert frame.metadata == before.metadata + assert result.receipt["spm_construction"]["metadata_transport"] == ( + "aggregate_and_rowwise_evidence_retained_by_owner_not_normative_frame_metadata" + ) + assert evidence["implementation"]["options"] == asdict(OPTIONS) + assert evidence["archives"] == result.receipt["archives"] + assert ( + evidence["source_binding"] == "actual_construction_from_owner_captured_archives" + ) + assert not evidence["engine_role_delivered"] + assert not evidence["annual_universe_declared"] + assert not evidence["release_eligible"] + crosswalk = rows(evidence["native_crosswalk"]) + membership = rows(evidence["partition"]["membership"]) + assert len(crosswalk) == len(membership) == frame.n("person") == 4 + assert crosswalk.set_index("person_id").new_spm_unit_id.to_dict() == ( + frame.person.set_index("person_id").person_spm_unit_id.to_dict() + ) + assert frame.n("spm_unit") == 3 + assert set(membership.role_source) >= { + "observed_relationship_rule", + "outside_acs_household_universe", + } + assert evidence["registry"]["entries"] == crosswalk.to_dict("records") + for entity in before.entities: + if entity == "spm_unit": + continue + columns = [c for c in before.table(entity) if c != "person_spm_unit_id"] + pd.testing.assert_frame_equal( + before.table(entity)[columns], frame.table(entity)[columns] + ) + assert ( + before.weights_for("household").values.tobytes() + == frame.weights_for("household").values.tobytes() + ) + assert "is_spm_independent_minor_role" not in frame.person + assert "spm_unit_spm_universe_status" not in frame.table("spm_unit") + evidence["partition"]["membership"]["rows"].clear() + assert len(result.construction_evidence["partition"]["membership"]["rows"]) == 4 + with pytest.raises( + native.ACSNativeCoverageBindingError, match="ISSUANCE_NOT_OWNED" + ): + native.verify_acs_native_coverage(copy.copy(result)) + + +@requires_assembler +@pytest.mark.parametrize("change", ["bytes", "missing", "frame"]) +def test_native_mutated_evidence_or_frame_refuses(tmp_path, monkeypatch, change): + require_acs_spm_source_capability(OPTIONS) + result = native.issue_acs_native_coverage( + **fixture(tmp_path, monkeypatch), spm_construction=OPTIONS + ) + owned = native._owned(result) + if change == "frame": + owned.frame.person.loc[0, "person_spm_unit_id"] += 100 + else: + value = ( + None + if change == "missing" + else owned.prepared.construction_evidence_json + b" " + ) + object.__setattr__(owned.prepared, "construction_evidence_json", value) + with pytest.raises(native.ACSNativeCoverageBindingError): + native.verify_acs_native_coverage(result) + + +@pytest.mark.parametrize( + "options", [{}, True, {"policy": "invented", "minor_partner_role": True}] +) +def test_bad_native_options_refuse_before_any_source_path(tmp_path, options): + with pytest.raises(ValueError, match="OPTIONS_TYPE"): + native.issue_acs_native_coverage( + tmp_path / "absent", + snapshot_root=tmp_path / "capture", + spm_construction=options, + ) + + +def test_unsupported_native_capability_refuses_before_any_source_path( + tmp_path, monkeypatch +): + probe = partition.AcsSpmAssemblerProbe(False, "assembler_unavailable", None, None) + monkeypatch.setattr(partition, "probe_acs_spm_assembler", lambda: probe) + with pytest.raises(partition.UnsupportedAssembler): + native.issue_acs_native_coverage( + tmp_path / "absent", + snapshot_root=tmp_path / "capture", + spm_construction=OPTIONS, + ) + + +@pytest.mark.parametrize( + "change", ["missing", "extra", "integer_bool", "policy", "v1_extra", "null"] +) +def test_request_v2_is_closed_and_explicit(tmp_path, change): + path = tmp_path / "selection-request.json" + document = { + "protocol": preparation.SPM_REQUEST_PROTOCOL, + "declaration": preparation.domains.DECLARATION, + "fraction": [1, 1], + "seed": 41, + "acs_spm_construction": asdict(OPTIONS), + } + if change == "missing": + document.pop("acs_spm_construction") + elif change == "extra": + document["acs_spm_construction"]["links"] = [] + elif change == "integer_bool": + document["acs_spm_construction"]["minor_partner_role"] = 1 + elif change == "policy": + document["acs_spm_construction"]["policy"] = "guessed" + elif change == "v1_extra": + document["protocol"] = preparation.REQUEST_PROTOCOL + else: + document["acs_spm_construction"] = None + path.write_bytes(preparation._encode(document)) + with pytest.raises(preparation.SurveyPopulationPreparationError): + preparation.read_survey_population_request(tmp_path) + + +@requires_assembler +def test_v2_preparation_preserves_owned_native_evidence_and_replays( + tmp_path, monkeypatch +): + require_acs_spm_source_capability(OPTIONS) + from test_us_survey_population_preparation import fixture as population_fixture + + arguments = population_fixture(tmp_path, monkeypatch) + path = arguments["source_dir"] / "selection-request.json" + default = preparation.prepare_authenticated_survey_population(**arguments) + default_payload = default.to_bytes() + assert default.acs_spm_construction_evidence is None + document = request(path) + assert preparation.read_survey_population_request(arguments["source_dir"]) == ( + Fraction(1), + 41, + ) + result = preparation.prepare_authenticated_survey_population(**arguments) + payload = result.to_bytes() + assert payload != default_payload + assert result.receipt["request"] == document + evidence = result.acs_spm_construction_evidence + native_ids = {r["new_spm_unit_id"] for r in evidence["registry"]["entries"]} + origin = result.receipt["origins"]["entities"]["spm_unit"] + assert {row[2] for row in origin if row[1] == "acs"} == native_ids + replay = preparation.prepare_authenticated_survey_population( + **arguments, candidate=payload + ) + assert replay.to_bytes() == payload + assert replay.acs_spm_construction_evidence == evidence + with pytest.raises( + preparation.SurveyPopulationPreparationError, match="CANDIDATE_MISMATCH" + ): + preparation.prepare_authenticated_survey_population( + **arguments, candidate=default_payload + ) + owned = native._owned(preparation._ISSUED[id(result)][2].native[0]) + object.__setattr__(owned.prepared, "construction_evidence_json", b"{}") + with pytest.raises(preparation.SurveyPopulationPreparationError): + result.checked_view() + + +@requires_assembler +def test_optional_native_source_mutation_still_refuses(tmp_path, monkeypatch): + require_acs_spm_source_capability(OPTIONS) + arguments = fixture(tmp_path, monkeypatch) + result = native.issue_acs_native_coverage(**arguments, spm_construction=OPTIONS) + path = arguments["source_dir"] / "csv_pus.zip" + with path.open("ab") as stream: + stream.write(b"changed invented archive") + with pytest.raises(native.ACSNativeCoverageBindingError): + native.verify_acs_native_coverage(result) + + +@requires_assembler +def test_v2_graph_cold_required_replay_and_option_key_separation(tmp_path, monkeypatch): + require_acs_spm_source_capability(OPTIONS) + from test_us_graph_survey_population import authenticated_arguments + + from microcosm.build.us_runtime import graph_survey_population as graph + + arguments = authenticated_arguments(tmp_path, monkeypatch) + path = arguments["source_dir"] / "selection-request.json" + request(path) + cold_values = graph.run_authenticated_survey_population( + **arguments, clones=False, return_values=True + ) + cold = cold_values.manifest + assert cold_values.preparation.acs_spm_construction_evidence["implementation"][ + "options" + ] == asdict(OPTIONS) + warm = graph.run_authenticated_survey_population( + **arguments, clones=False, resume="require" + ) + assert cold.key == warm.key + assert all(node.store_hit for node in warm.nodes.values()) + graph._same_frame( + cold.population(graph.CREATE_NODE), warm.population(graph.CREATE_NODE) + ) + changed_options = AcsSpmSourceAssemblyOptions(OPTIONS.policy, False) + request(path, changed_options) + from microcosm.graph.errors import StoreMissError + + with pytest.raises(StoreMissError, match="resume='require' found cache misses"): + graph.run_authenticated_survey_population( + **arguments, clones=False, resume="require" + ) + # A changed request cannot silently load the old required graph result. + changed = graph.run_authenticated_survey_population(**arguments, clones=False) + assert changed.key != cold.key + + +def test_unreviewed_compatible_assembler_refuses_before_source_path( + tmp_path, monkeypatch +): + probe = partition.AcsSpmAssemblerProbe( + True, "supported", "/invented/units.py", "a" * 64 + ) + monkeypatch.setattr(partition, "probe_acs_spm_assembler", lambda: probe) + with pytest.raises( + native.housing.ACSHousingSourceError, match="SPM_ASSEMBLER_UNREVIEWED" + ): + native.issue_acs_native_coverage( + tmp_path / "absent", + snapshot_root=tmp_path / "capture", + spm_construction=OPTIONS, + ) diff --git a/packages/microcosm-build/tests/test_us_acs_spm_partition.py b/packages/microcosm-build/tests/test_us_acs_spm_partition.py new file mode 100644 index 000000000..b0a62b868 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_spm_partition.py @@ -0,0 +1,518 @@ +"""Invented ACS households; real canonical assembly, no country calculations.""" + +import pandas as pd +import pytest + +from microcosm.build.acs_spm_partition import ( + ACS_SPM_DEVELOPMENT_POLICY, + ACS_SPM_PARTITION_POLICY, + AcsSpmLink, + AcsSpmLinkAssessment, + AcsSpmRoleDecision, + UnsupportedAssembler, + probe_acs_spm_assembler, + reconstruct_acs_spm_partition, +) + +# Marks exactly the cases that reach the canonical assembler. Strict, so an +# unexpected pass stays a failure, and narrowed to the dedicated exception, so +# the ValueError refusals some of these reach afterwards are never masked. The +# source-validation, group-quarters and unresolved-household cases carry no +# marker: they must keep passing on every runtime. +requires_assembler = pytest.mark.xfail( + not probe_acs_spm_assembler().supported, + strict=True, + raises=UnsupportedAssembler, + reason="Assembled membership requires the reviewed canonical assembler", +) + + +def roster(relationships, ages, *, marital=None): + n = len(ages) + if marital is None: + marital = [5] * n + if any(rel in {21, 23} for rel in relationships): + marital = [1 if rel in {20, 21, 23} else 5 for rel in relationships] + return pd.DataFrame( + { + "person_id": range(101, 101 + n), + "person_household_id": [10] * n, + "person_spm_unit_id": [700] * n, + "person_tax_unit_id": range(900, 900 + n), + "SPORDER": range(1, n + 1), + "RELSHIPP": relationships, + "AGEP": ages, + "MAR": marital, + "TYPEHUGQ": [1] * n, + # Authentic raw evidence is not replaced by the old CPS recode. + "A_EXPRRP": [13] * n, + "PEPAR2": [1] * n, + "SPM_ID": [700] * n, + "family_id": [700] * n, + "source_person_id": [f"raw-{i}" for i in range(n)], + } + ) + + +def assemble(persons, **kwargs): + old_units = pd.DataFrame( + { + "spm_unit_id": persons.person_spm_unit_id.unique(), + "snap": 123.0, + "spm_unit_net_income": 456.0, + } + ) + counts = kwargs.pop( + "household_person_counts", + persons.groupby("person_household_id").size().to_dict(), + ) + return reconstruct_acs_spm_partition( + persons, old_units, household_person_counts=counts, **kwargs + ) + + +def groups(result): + return { + frozenset(group.person_id) + for _, group in result.membership.groupby("proposed_spm_unit_id") + } + + +def complete(pid): + return AcsSpmLinkAssessment(pid, "complete", "invented_reviewed_links_v1") + + +@requires_assembler +def test_reference_family_and_partner_keep_raw_evidence(): + persons = roster([20, 21, 25, 29, 22], [45, 44, 10, 70, 40]) + original = persons.copy(deep=True) + result = assemble(persons) + assert groups(result) == {frozenset(persons.person_id)} + assert result.membership.RELSHIPP.tolist() == persons.RELSHIPP.tolist() + assert result.provenance["canonical_diagnostics"]["method"] != "native_spm_id" + assert result.regrouping.action.eq("membership_unchanged").all() + pd.testing.assert_frame_equal(persons, original) + + +@requires_assembler +def test_adult_roommates_split_without_external_pointer_graph(): + persons = roster([20, 34, 36], [45, 35, 39]) + result = assemble(persons) + assert groups(result) == {frozenset([101]), frozenset([102]), frozenset([103])} + result.require_resolved() + assert result.crosswalk.person_count.tolist() == [1, 1, 1] + assert set(result.regrouping.field) == {"snap", "spm_unit_net_income"} + assert result.regrouping.action.eq("requires_regrouping").all() + assert "snap" not in result.membership + assert "person_tax_unit_id" not in result.membership + assert result.provenance["modeled_residual_separation_people"] == 2 + + +@requires_assembler +def test_parent_link_precedes_unrelated_child_fallback(): + # Census WP2011-22 printed p.7: under-15 fallback excludes another member's child. + persons = roster([20, 34, 36], [45, 35, 8]) + result = assemble( + persons, + links=[AcsSpmLink(103, 102, "parent", "invented_parent_rule_v1")], + assessments=[complete(102), complete(103)], + ) + assert groups(result) == {frozenset([101]), frozenset([102, 103])} + assert result.links.source.eq("approved_inference").all() + assert ( + "link_parent_child" + in result.provenance["canonical_diagnostics"]["fallback_rules_used"] + ) + + +@requires_assembler +def test_partner_child_links_join_primary_component(): + result = assemble( + roster([20, 22, 36], [45, 35, 8]), + links=[AcsSpmLink(103, 102, "parent", "invented_parent_rule_v1")], + assessments=[complete(103)], + ) + assert groups(result) == {frozenset([101, 102, 103])} + assert "source_observed" in set(result.links.source) + assert "approved_inference" in set(result.links.source) + + +@requires_assembler +@pytest.mark.parametrize("age", [14, 17, 21, 22, 25]) +def test_foster_age_boundary(age): + result = assemble(roster([20, 35], [45, age])) + assert len(groups(result)) == (1 if age < 22 else 2) + + +def test_unassessed_possible_parent_does_not_become_head_attachment(): + result = assemble(roster([20, 34, 36], [45, 35, 8])) + assert result.membership.proposed_spm_unit_id.isna().all() + assert result.provenance["unresolved_households"] == 1 + assert result.regrouping.action.eq("unresolved_partition").all() + with pytest.raises(ValueError, match="unresolved"): + result.require_resolved() + + +def test_explicit_ambiguous_assessment_is_not_overridden_by_a_link(): + result = assemble( + roster([20, 34, 36, 36], [45, 35, 36, 8]), + links=[AcsSpmLink(104, 102, "parent", "one_candidate")], + assessments=[ + complete(102), + complete(103), + AcsSpmLinkAssessment(104, "ambiguous", "two_possible_parents"), + ], + ) + assert result.membership.proposed_spm_unit_id.isna().all() + assert "ambiguous" in set(result.membership.secondary_link_status) + + +@requires_assembler +def test_married_roommates_split_by_assumption_without_an_accepted_spouse_link(): + persons = roster([20, 34, 34], [45, 35, 36], marital=[5, 1, 1]) + assumed = assemble(persons) + assert len(groups(assumed)) == 3 + assert assumed.provenance["modeled_residual_separation_people"] == 2 + assert assumed.membership.MAR.tolist() == [5, 1, 1] + assumed.require_resolved() + result = assemble( + persons, + links=[AcsSpmLink(102, 103, "spouse", "invented_spouse_rule_v1")], + assessments=[complete(102), complete(103)], + ) + assert groups(result) == {frozenset([101]), frozenset([102, 103])} + + +@requires_assembler +def test_residual_under15_vs_independent_minor_role_is_separate(): + child = assemble(roster([20, 36], [45, 14])) + assert len(groups(child)) == 1 + minor = assemble(roster([20, 36], [45, 16])) + assert len(groups(minor)) == 2 + assert pd.isna(minor.membership.independent_minor_role.iloc[1]) + with pytest.raises(ValueError, match="role"): + minor.require_resolved() + resolved = assemble( + roster([20, 36], [45, 16]), + role_decisions=[AcsSpmRoleDecision(102, True, "reviewed_singleton_role_v1")], + ) + resolved.require_resolved() + assert resolved.membership.role_source.iloc[1] == "approved_inference" + + +@requires_assembler +def test_minor_head_and_spouse_known_but_partner_role_unresolved(): + result = assemble(roster([20, 21, 22], [17, 16, 16])) + assert result.membership.independent_minor_role.iloc[:2].tolist() == [True, True] + assert pd.isna(result.membership.independent_minor_role.iloc[2]) + # The role primitive does not promote a 14-year-old to a measurement adult. + young = assemble(roster([20], [14])) + assert young.membership.independent_minor_role.iloc[0] + assert young.membership.AGEP.iloc[0] == 14 + + +@pytest.mark.parametrize( + "policy", [ACS_SPM_PARTITION_POLICY, ACS_SPM_DEVELOPMENT_POLICY] +) +def test_gq_membership_is_preserved_separately_from_exclusion(policy): + persons = roster([37, 37], [17, 40]) + persons["TYPEHUGQ"] = 2 + persons["person_spm_unit_id"] = [700, 701] + result = assemble(persons, policy=policy) + assert len(groups(result)) == 2 + assert result.membership.measurement_status.eq( + "outside_acs_household_universe" + ).all() + assert result.membership.independent_minor_role.isna().all() + assert result.regrouping.action.eq("membership_unchanged").all() + result.require_resolved() + + +@requires_assembler +def test_membership_stable_under_shuffle_and_household_complete_chunks(): + first = roster([20, 34], [45, 35]) + second = first.copy() + second["person_id"] += 100 + second["person_household_id"] += 10 + second["person_spm_unit_id"] += 100 + persons = pd.concat([first, second], ignore_index=True) + combined = assemble(persons).membership.set_index("person_id").proposed_spm_unit_id + shuffled = ( + assemble(persons.sample(frac=1, random_state=42)) + .membership.set_index("person_id") + .proposed_spm_unit_id + ) + chunked = ( + pd.concat([assemble(first).membership, assemble(second).membership]) + .set_index("person_id") + .proposed_spm_unit_id + ) + pd.testing.assert_series_equal(combined.sort_index(), shuffled.sort_index()) + pd.testing.assert_series_equal(combined.sort_index(), chunked.sort_index()) + + +@pytest.mark.parametrize("column", ["person_id", "SPORDER"]) +def test_duplicate_pointer_keys_refused(column): + persons = roster([20, 34], [45, 35]) + persons[column] = 1 + with pytest.raises(ValueError, match="unique"): + assemble(persons) + + +@pytest.mark.parametrize("relative", [103, 101]) +def test_missing_or_self_parent_refused(relative): + with pytest.raises(ValueError, match="[Ll]ink"): + assemble( + roster([20, 34], [45, 35]), + links=[AcsSpmLink(101, relative, "parent", "bad")], + ) + + +def test_cross_household_link_refused(): + persons = roster([20, 20], [45, 35]) + persons["person_household_id"] = [10, 20] + persons["person_spm_unit_id"] = [700, 701] + with pytest.raises(ValueError, match="household"): + assemble(persons, links=[AcsSpmLink(101, 102, "parent", "bad")]) + + +def test_direct_role_contradiction_refused(): + with pytest.raises(ValueError, match="observed"): + assemble( + roster([20], [16]), role_decisions=[AcsSpmRoleDecision(101, False, "bad")] + ) + + +def test_partial_household_is_not_accepted_as_complete(): + with pytest.raises(ValueError, match="complete households"): + assemble(roster([20, 34], [45, 35]), household_person_counts={10: 3}) + + +@pytest.mark.parametrize( + "column,value", + [("AGEP", None), ("AGEP", True), ("RELSHIPP", 99), ("person_id", 1.5)], +) +def test_invalid_or_missing_source_evidence_refused(column, value): + persons = roster([20], [45]) + persons[column] = value + with pytest.raises(ValueError): + assemble(persons) + + +@requires_assembler +def test_raw_second_parent_alias_is_not_treated_as_an_observation(): + result = assemble(roster([20, 25], [45, 10])) + parents = result.links.loc[result.links.kind.eq("parent")] + assert parents[["person_id", "relative_id"]].values.tolist() == [[102, 101]] + assert parents.source.tolist() == ["source_observed"] + + +@requires_assembler +def test_inventory_does_not_read_or_reallocate_old_unit_amounts(): + persons = roster([20, 34], [45, 35]) + amount = object() + old = pd.DataFrame({"spm_unit_id": [700], "opaque_old_amount": [amount]}) + result = reconstruct_acs_spm_partition( + persons, old, household_person_counts={10: 2} + ) + assert result.regrouping.field.tolist() == ["opaque_old_amount"] + assert old.opaque_old_amount.iloc[0] is amount + assert "opaque_old_amount" not in result.membership + + +def test_inferred_parent_cannot_create_cycle_with_observed_child(): + with pytest.raises(ValueError, match="cycle"): + assemble( + roster([20, 25], [45, 25]), + links=[AcsSpmLink(101, 102, "parent", "bad_cycle")], + ) + + +def test_boolean_link_id_cannot_alias_person_one(): + persons = roster([20, 34], [45, 35]) + persons["person_id"] = [1, 2] + with pytest.raises(ValueError, match="exact"): + assemble(persons, links=[AcsSpmLink(True, 2, "parent", "bad_id")]) + + +@requires_assembler +def test_unobserved_adult_child_partner_is_a_declared_separation_assumption(): + # RELSHIPP observes only the reference relation, not a child's partner. + # This proposal does not claim that a secondary partner has been ruled out. + result = assemble(roster([20, 25, 36], [65, 30, 30])) + assert groups(result) == {frozenset([101, 102]), frozenset([103])} + assert ( + result.membership.secondary_link_status.iloc[2] == "modeled_residual_separation" + ) + assert result.provenance["modeled_residual_separation_people"] == 1 + resolved = assemble( + roster([20, 25, 36], [65, 30, 30]), + links=[AcsSpmLink(103, 102, "partner", "reviewed_secondary_partner_v1")], + assessments=[complete(103)], + ) + assert groups(resolved) == {frozenset([101, 102, 103])} + + +@pytest.mark.parametrize("from_person,to_person", [(101, 103), (103, 101), (102, 103)]) +def test_inferred_spouse_cannot_contradict_either_observed_pair_endpoint( + from_person, to_person +): + with pytest.raises(ValueError, match="[Cc]onflicting|cardinality"): + assemble( + roster([20, 21, 36], [45, 44, 43], marital=[1, 1, 1]), + links=[ + AcsSpmLink(from_person, to_person, "spouse", "contradictory_spouse") + ], + ) + + +def test_inferred_spouse_cannot_contradict_raw_never_married(): + with pytest.raises(ValueError, match="MAR"): + assemble( + roster([20, 34, 36], [45, 35, 36]), + links=[AcsSpmLink(102, 103, "spouse", "bad_spouse")], + ) + + +def test_observed_spouse_requires_coherent_raw_marital_status(): + with pytest.raises(ValueError, match="MAR"): + assemble(roster([20, 21], [45, 44], marital=[5, 1])) + + +@requires_assembler +def test_development_preset_pools_unknown_parentage_without_fabricating_links(): + result = assemble( + roster([20, 34, 36], [45, 35, 8]), policy=ACS_SPM_DEVELOPMENT_POLICY + ) + assert groups(result) == {frozenset([101, 103]), frozenset([102])} + assert result.links.empty + assert ( + result.membership.partition_assumption.iloc[2] + == "parent_unknown_reference_pooling" + ) + assert result.membership.parent_link_status.iloc[2] == "unknown" + assert result.provenance["source_unresolved_relationship_households"] == 1 + assert result.provenance["policy"] == ACS_SPM_DEVELOPMENT_POLICY + result.require_resolved() + + +@requires_assembler +def test_development_preset_assigns_modeled_minor_singleton_reference_roles(): + result = assemble( + roster([20, 36, 34], [45, 16, 17]), policy=ACS_SPM_DEVELOPMENT_POLICY + ) + assert len(groups(result)) == 3 + assert result.membership.independent_minor_role.tolist() == [True, True, True] + assert result.membership.role_source.iloc[1:].eq("modeled_assumption").all() + assert ( + result.membership.role_rule.iloc[1:] + .eq("minor_singleton_unit_reference_v1") + .all() + ) + result.require_resolved() + + +@requires_assembler +@pytest.mark.parametrize("partner_role", [True, False]) +def test_development_minor_partner_sensitivity_preserves_membership(partner_role): + result = assemble( + roster([20, 22, 25, 35], [45, 16, 16, 17]), + policy=ACS_SPM_DEVELOPMENT_POLICY, + minor_partner_role=partner_role, + ) + assert len(groups(result)) == 1 + assert result.membership.independent_minor_role.tolist() == [ + True, + partner_role, + False, + False, + ] + assert result.membership.role_source.iloc[1:].eq("modeled_assumption").all() + assert result.provenance["minor_partner_role"] is partner_role + result.require_resolved() + + +@requires_assembler +def test_development_accepted_parent_precedes_unknown_parent_pooling(): + result = assemble( + roster([20, 34, 36], [45, 35, 8]), + policy=ACS_SPM_DEVELOPMENT_POLICY, + links=[AcsSpmLink(103, 102, "parent", "accepted_parent_v1")], + ) + assert groups(result) == {frozenset([101]), frozenset([102, 103])} + assert ( + result.membership.parent_link_status.iloc[2] == "approved_inferred_parent_link" + ) + assert result.membership.partition_assumption.iloc[2] == "accepted_sharing_links" + result.require_resolved() + + +@requires_assembler +def test_development_unique_accepted_minor_parent_gets_modeled_reference_role(): + result = assemble( + roster([20, 34, 36], [45, 16, 0]), + policy=ACS_SPM_DEVELOPMENT_POLICY, + links=[AcsSpmLink(103, 102, "parent", "accepted_parent_v1")], + ) + assert groups(result) == {frozenset([101]), frozenset([102, 103])} + assert result.membership.independent_minor_role.tolist() == [True, True, False] + assert ( + result.membership.role_rule.iloc[1] == "accepted_parent_component_reference_v1" + ) + assert result.membership.role_source.iloc[1] == "modeled_assumption" + result.require_resolved() + + +@requires_assembler +def test_development_does_not_promote_under15_parent(): + persons = roster([20, 34, 36], [45, 14, 0]) + kwargs = { + "policy": ACS_SPM_DEVELOPMENT_POLICY, + "links": [AcsSpmLink(103, 102, "parent", "accepted_parent_v1")], + } + result = assemble(persons, **kwargs) + assert len(groups(result)) == 1 + assert result.membership.independent_minor_role.tolist() == [True, False, False] + assert result.membership.parent_link_status.iloc[1] == "unknown" + assert ( + result.membership.partition_assumption.iloc[1] + == "accepted_sharing_and_parent_unknown_reference_pooling" + ) + assert result.links[["person_id", "relative_id"]].values.tolist() == [[103, 102]] + assert result.provenance["modeled_residual_child_attachment_people"] == 1 + with pytest.raises(ValueError, match="15–17"): + assemble( + persons, + **kwargs, + role_decisions=[AcsSpmRoleDecision(102, True, "invalid_promotion")], + ) + + +@requires_assembler +def test_development_two_possible_minor_parent_references_need_explicit_roles(): + result = assemble( + roster([20, 34, 36, 36], [45, 16, 17, 0]), + policy=ACS_SPM_DEVELOPMENT_POLICY, + links=[ + AcsSpmLink(104, 102, "parent", "accepted_parent_v1"), + AcsSpmLink(104, 103, "parent", "accepted_parent_v1"), + ], + ) + assert groups(result) == {frozenset([101]), frozenset([102, 103, 104])} + assert result.membership.independent_minor_role.iloc[1:3].isna().all() + with pytest.raises(ValueError, match="role"): + result.require_resolved() + + +@requires_assembler +def test_development_adult_parent_reference_does_not_promote_minor_coparent(): + result = assemble( + roster([20, 34, 36, 36], [45, 35, 17, 0]), + policy=ACS_SPM_DEVELOPMENT_POLICY, + links=[ + AcsSpmLink(104, 102, "parent", "accepted_parent_v1"), + AcsSpmLink(104, 103, "parent", "accepted_parent_v1"), + ], + ) + assert not result.membership.independent_minor_role.iloc[2] + result.require_resolved() diff --git a/packages/microcosm-build/tests/test_us_acs_spm_scope.py b/packages/microcosm-build/tests/test_us_acs_spm_scope.py new file mode 100644 index 000000000..0e8552069 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_spm_scope.py @@ -0,0 +1,427 @@ +"""Invented constructor evidence only: no issuers, source files or country runtime.""" + +import copy +import hashlib +import json +from dataclasses import FrozenInstanceError, replace + +import pytest + +from microcosm.build.acs_spm_scope import ( + ACSAnalysisProfile, + ACSSPMScopeError, + classify_acs_spm_scope, + count_acs_spm_authorities, +) + + +def profile(**changes): + return ACSAnalysisProfile( + **{ + "year": 2024, + "source_vintage": "acs_2024_1yr", + "admit_modeled": False, + "admit_approved_inference": False, + **changes, + } + ) + + +def member(pid, household, unit, **changes): + return { + "person_id": pid, + "person_household_id": household, + "old_spm_unit_id": household + 1000, + "SPORDER": 1, + "TYPEHUGQ": 1, + "proposed_spm_unit_id": unit, + "independent_minor_role": True, + "role_source": "observed_relationship_rule", + "role_rule": "acs_relshipp_reference_head_or_spouse", + "partition_assumption": "reference_relationship_rule", + "secondary_link_status": "not_required", + "measurement_status": "included_acs_household", + **changes, + } + + +def table(records, columns=None): + columns = columns or list(records[0]) + return { + "columns": columns, + "dtypes": ["object"] * len(columns), + "index": list(range(len(records))), + "rows": [[row[column] for column in columns] for row in records], + } + + +def fixture_evidence(members=None, links=(), *, partner=False): + """JSON-shaped invented evidence, never an authenticated preparation owner.""" + if members is None: + members = [ + member(1, 10, "u-observed"), + member( + 2, + 10, + "u-observed", + SPORDER=2, + independent_minor_role=False, + role_source="age_not_role_sensitive", + role_rule="canonical_role_age_scope", + ), + member( + 3, + 20, + "u-modeled", + independent_minor_role=partner, + role_source="modeled_assumption", + role_rule="minor_reference_partner_role_sensitivity_v1", + ), + member(4, 30, "u-inferred", role_source="approved_inference"), + member(5, 40, "u-uncertain-head"), + member( + 6, + 40, + "u-uncertain-secondary", + SPORDER=2, + secondary_link_status="unassessed", + partition_assumption="parent_unknown_residual_separation", + ), + member( + 7, + 50, + "u-missing", + independent_minor_role=None, + role_source="age_not_role_sensitive", + ), + *[ + member( + pid, + hh, + name, + TYPEHUGQ=kind, + independent_minor_role=None, + role_source="outside_acs_household_universe", + role_rule="not_classified", + partition_assumption="preserved_gq_membership", + measurement_status="outside_acs_household_universe", + ) + for pid, hh, name, kind in ((8, 60, "u-gq2", 2), (9, 70, "u-gq3", 3)) + ], + ] + units = {} + for row in members: + units.setdefault(row["proposed_spm_unit_id"], []).append(row) + registry = [] + crosswalk = [] + unit_evidence = [] + uncertain = { + row["person_household_id"] + for row in members + if row["secondary_link_status"] in {"unassessed", "ambiguous"} + } + for number, (canonical, rows) in enumerate(sorted(units.items()), 100): + native_keys = sorted( + (f"2024TEST{row['person_household_id']}", row["SPORDER"]) for row in rows + ) + component = hashlib.sha256( + json.dumps(native_keys, separators=(",", ":")).encode() + ).hexdigest() + for row in rows: + registry.append( + { + "serialno": f"2024TEST{row['person_household_id']}", + "sporder": row["SPORDER"], + "person_id": row["person_id"], + "household_id": row["person_household_id"], + "old_spm_unit_id": row["old_spm_unit_id"], + "new_spm_unit_id": number, + "component_sha256": component, + "source_record_sha256": "a" * 64, + } + ) + crosswalk.append( + { + "person_household_id": rows[0]["person_household_id"], + "old_spm_unit_id": rows[0]["old_spm_unit_id"], + "proposed_spm_unit_id": canonical, + "person_count": len(rows), + } + ) + unit_evidence.append( + { + "proposed_spm_unit_id": canonical, + "role_sources": sorted({row["role_source"] for row in rows}), + "partition_assumptions": sorted( + {row["partition_assumption"] for row in rows} + ), + "secondary_link_statuses": sorted( + {row["secondary_link_status"] for row in rows} + ), + "household_source_uncertain": rows[0]["person_household_id"] + in uncertain, + "outside_acs_household_universe": rows[0]["TYPEHUGQ"] != 1, + } + ) + options = { + "policy": "acs_spm_development_reconstruction_v1", + "minor_partner_role": partner, + } + return { + "protocol": "microcosm.acs-spm-native-construction-evidence.v1", + "source_binding": "actual_construction_from_owner_captured_archives", + "implementation": {"options": options}, + "construction_receipt": {"options": options}, + "partition_provenance": options, + "registry": { + "options": options, + "id_ceiling": 10000, + "assembler_sha256": "b" * 64, + "entries": registry, + }, + "partition": { + "membership": table(members), + "links": table( + list(links), ["person_id", "relative_id", "kind", "source", "rule_id"] + ), + "crosswalk": table(crosswalk), + "regrouping": table([], ["field", "action"]), + }, + "native_crosswalk": table(registry), + "unit_evidence": table(unit_evidence), + "engine_role_delivered": False, + "annual_universe_declared": False, + "release_eligible": False, + } + + +def set_cell(evidence, table_name, row, column, value): + target = evidence["partition"][table_name] + target["rows"][row][target["columns"].index(column)] = value + + +@pytest.mark.parametrize( + "modeled,inference", [(False, False), (True, False), (False, True), (True, True)] +) +def test_exact_profile_decisions_preserve_labels_roles_and_whole_household_uncertainty( + modeled, inference +): + evidence = fixture_evidence() + original = copy.deepcopy(evidence) + result = classify_acs_spm_scope( + evidence, profile(admit_modeled=modeled, admit_approved_inference=inference) + ) + by_label = {row.canonical_id: row for row in result.units} + assert by_label["u-observed"].status == "INCLUDED" + assert by_label["u-modeled"].status == ("INCLUDED" if modeled else "UNRESOLVED") + assert by_label["u-modeled"].authority == "modeled_assumption" + assert by_label["u-inferred"].status == ("INCLUDED" if inference else "UNRESOLVED") + assert by_label["u-inferred"].authority == "approved_inference" + for name in ("u-uncertain-head", "u-uncertain-secondary"): + assert by_label[name].status == "UNRESOLVED" + assert by_label[name].authority == "unresolved" + assert by_label[name].reason == "source_membership_or_authority_unresolved" + assert by_label["u-missing"].reason == "source_role_missing" + for name in ("u-gq2", "u-gq3"): + assert by_label[name].status == "OUTSIDE" + assert by_label[name].authority == "outside_acs_household_universe" + assert [row.value for row in result.roles] == [ + True, + False, + False, + True, + True, + True, + None, + None, + None, + ] + assert count_acs_spm_authorities(result.units) == { + "observed_relationship_rule": 2, + "modeled_assumption": 1, + "approved_inference": 1, + "unresolved": 2, + "outside_acs_household_universe": 2, + } + assert evidence == original + assert result.profile.year == 2024 + assert result.minor_partner_role is False + with pytest.raises(FrozenInstanceError): + result.units[0].status = "INCLUDED" + + +@pytest.mark.parametrize( + "source,expected", + [ + ("source_observed", "observed_relationship_rule"), + ("approved_inference", "approved_inference"), + ], +) +def test_internal_link_authority_uses_reviewed_order(source, expected): + members = [member(1, 10, "u"), member(2, 10, "u", SPORDER=2)] + link = { + "person_id": 2, + "relative_id": 1, + "kind": "parent", + "source": source, + "rule_id": "invented_rule", + } + result = classify_acs_spm_scope( + fixture_evidence(members, [link]), profile(admit_approved_inference=True) + ) + assert result.units[0].authority == expected + assert result.units[0].status == "INCLUDED" + + +def test_row_reordering_and_partner_sensitivity_do_not_change_identity_or_authority(): + evidence = fixture_evidence() + before = classify_acs_spm_scope(evidence, profile()) + for value in ( + *evidence["partition"].values(), + evidence["native_crosswalk"], + evidence["unit_evidence"], + ): + value["rows"].reverse() + value["index"].reverse() + evidence["registry"]["entries"].reverse() + assert classify_acs_spm_scope(evidence, profile()) == before + after = classify_acs_spm_scope(fixture_evidence(partner=True), profile()) + assert after.units == before.units + assert after.minor_partner_role is True + assert after.roles[2] == replace(before.roles[2], value=True) + assert after.roles[:2] + after.roles[3:] == before.roles[:2] + before.roles[3:] + + +@pytest.mark.parametrize( + "change", + [ + {"year": 2025}, + {"year": True}, + {"source_vintage": "acs_2025_1yr"}, + {"admit_modeled": 1}, + {"admit_approved_inference": None}, + {"name": "all_acs"}, + {"methodology": "other"}, + {"lineage": "other"}, + ], +) +def test_profile_is_explicit_and_never_carries_to_another_year(change): + with pytest.raises(ACSSPMScopeError): + profile(**change) + + +@pytest.mark.parametrize( + "defect", + [ + "row_width", + "duplicate_column", + "dtype_count", + "index_count", + "duplicate_person", + "role_integer", + "unknown_authority", + "unknown_assumption", + "unknown_secondary", + "mixed_household_kind", + "outside_observed_role", + "native_membership", + "native_id_type", + "registry_coverage", + "canonical_merge", + "unit_evidence", + "partition_count", + "partner_option", + "scope_already_declared", + "missing_column", + ], +) +def test_malformed_or_inconsistent_construction_evidence_refuses(defect): + evidence = fixture_evidence() + membership = evidence["partition"]["membership"] + if defect == "row_width": + membership["rows"][0].pop() + elif defect == "duplicate_column": + membership["columns"][1] = membership["columns"][0] + elif defect == "dtype_count": + membership["dtypes"].pop() + elif defect == "index_count": + membership["index"].pop() + elif defect == "duplicate_person": + set_cell(evidence, "membership", 1, "person_id", 1) + elif defect == "role_integer": + set_cell(evidence, "membership", 0, "independent_minor_role", 1) + elif defect == "unknown_authority": + set_cell(evidence, "membership", 0, "role_source", "guessed_observation") + elif defect == "unknown_assumption": + set_cell(evidence, "membership", 0, "partition_assumption", "default") + elif defect == "unknown_secondary": + set_cell(evidence, "membership", 0, "secondary_link_status", "default") + elif defect == "mixed_household_kind": + set_cell(evidence, "membership", 1, "TYPEHUGQ", 2) + elif defect == "outside_observed_role": + set_cell(evidence, "membership", 7, "independent_minor_role", False) + elif defect == "native_membership": + row = evidence["native_crosswalk"] + row["rows"][0][row["columns"].index("new_spm_unit_id")] += 1 + elif defect == "native_id_type": + row = evidence["native_crosswalk"] + position = row["columns"].index("new_spm_unit_id") + row["rows"][0][position] = float(row["rows"][0][position]) + elif defect == "registry_coverage": + evidence["registry"]["entries"].pop() + elif defect == "canonical_merge": + for row in evidence["registry"]["entries"]: + row["new_spm_unit_id"] = 100 + target = evidence["native_crosswalk"] + for row in target["rows"]: + row[target["columns"].index("new_spm_unit_id")] = 100 + elif defect == "unit_evidence": + target = evidence["unit_evidence"] + target["rows"][0][target["columns"].index("outside_acs_household_universe")] = ( + False + ) + elif defect == "partition_count": + set_cell(evidence, "crosswalk", 0, "person_count", 99) + elif defect == "partner_option": + evidence["implementation"] = { + "options": { + **evidence["implementation"]["options"], + "minor_partner_role": True, + } + } + elif defect == "scope_already_declared": + evidence["annual_universe_declared"] = True + elif defect == "missing_column": + position = membership["columns"].index("old_spm_unit_id") + for key in ("columns", "dtypes"): + membership[key].pop(position) + for row in membership["rows"]: + row.pop(position) + with pytest.raises(ACSSPMScopeError): + classify_acs_spm_scope(evidence, profile()) + + +@pytest.mark.parametrize( + "defect", ["missing_person", "cross_household", "unknown_source"] +) +def test_links_cannot_reference_unknown_people_or_other_households(defect): + members = [member(1, 10, "u1"), member(2, 20, "u2")] + link = { + "person_id": 1, + "relative_id": 2, + "kind": "parent", + "source": "approved_inference", + "rule_id": "invented_rule", + } + if defect == "missing_person": + link["relative_id"] = 100 + elif defect == "unknown_source": + members[1] = member(2, 10, "u1", SPORDER=2) + link["source"] = "guess" + code = { + "missing_person": "LINK_PERSON", + "cross_household": "LINK_HOUSEHOLD", + "unknown_source": "LINK_SOURCE", + }[defect] + with pytest.raises(ACSSPMScopeError, match=code): + classify_acs_spm_scope(fixture_evidence(members, [link]), profile()) diff --git a/packages/microcosm-build/tests/test_us_acs_spm_source_assembly.py b/packages/microcosm-build/tests/test_us_acs_spm_source_assembly.py new file mode 100644 index 000000000..b23c4f969 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_spm_source_assembly.py @@ -0,0 +1,508 @@ +"""Invented source rosters only; no population files or country calculation.""" + +import hashlib +from dataclasses import replace +from pathlib import Path +from zipfile import ZipFile + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import acs_spm_partition as partition +from microcosm.build.acs_spm_source_assembly import ( + RECEIPT_KEY, + AcsSpmSourceAssemblyOptions, + assemble_acs_spm_source, +) +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +OPTIONS = AcsSpmSourceAssemblyOptions(partition.ACS_SPM_DEVELOPMENT_POLICY, True) +CEILING = 10_000 +requires_assembler = pytest.mark.xfail( + not partition.probe_acs_spm_assembler().supported, + raises=partition.UnsupportedAssembler, + strict=True, + reason="Explicit source construction requires the canonical assembler capability", +) + + +def source_frame(relations=(20, 34, 36), ages=(45, 30, 16), *, kind=1, offset=0): + count = len(relations) + people = pd.DataFrame( + { + "person_id": np.arange(count, dtype=np.int64) + 100 + offset, + "person_household_id": np.full(count, 10 + offset, dtype=np.int64), + "person_spm_unit_id": np.full(count, 20 + offset, dtype=np.int64), + "person_tax_unit_id": np.arange(count, dtype=np.int64) + 30 + offset, + "person_family_id": np.full(count, 40 + offset, dtype=np.int64), + "person_marital_unit_id": np.arange(count, dtype=np.int64) + 50 + offset, + "SPORDER": pd.Series([str(i + 1) for i in range(count)], dtype="string"), + "RELSHIPP": relations, + "AGEP": ages, + "MAR": [5] * count, + "is_household_head": [value == 20 for value in relations], + "tax_unit_role_input": ["HEAD"] * count, + "is_related_to_head_or_spouse": [value == 20 for value in relations], + "source_year": [2024] * count, + "native_income": pd.Series([1.5] * count, dtype="Float64"), + } + ) + tables = {"person": people} + for entity in US_SCHEMA.group_entities: + tables[entity] = pd.DataFrame( + {f"{entity}_id": sorted(set(people[f"person_{entity}_id"]))}, dtype="int64" + ) + tables["tax_unit"]["filing_status"] = "SINGLE" + tables["household"]["SERIALNO"] = f"2024TEST{offset}" + tables["household"]["TYPEHUGQ"] = kind + tables["household"]["NP"] = count + tables["household"]["TEN"] = 3 if kind == 1 else pd.NA + tables["household"]["geography_unmodified"] = "06001" + return Frame( + tables, + US_SCHEMA, + {"household": Weights([7.25], WeightKind.DESIGN)}, + pd.Series(["acs_2024_1yr"] * count), + metadata={"original": {"kept": True}}, + ) + + +def rebuild(frame, *, tables=None, weights=None): + return Frame( + tables or {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + weights or dict(frame._weights), + frame.strata, + metadata=frame.metadata, + mass_log=frame.mass_log, + ) + + +def assemble(frame, **kwargs): + return assemble_acs_spm_source( + frame, + options=kwargs.pop("options", OPTIONS), + id_ceiling=kwargs.pop("id_ceiling", CEILING), + **kwargs, + ) + + +def assert_preserved(before, after): + for entity in before.entities: + if entity == "spm_unit": + continue + columns = [ + name for name in before.table(entity) if name != "person_spm_unit_id" + ] + pd.testing.assert_frame_equal( + before.table(entity)[columns], + after.table(entity)[columns], + check_exact=True, + ) + pd.testing.assert_series_equal(before.strata, after.strata, check_exact=True) + np.testing.assert_array_equal( + before.weights_for("household").values, after.weights_for("household").values + ) + assert after.weights_for("household").kind is before.weights_for("household").kind + assert before.mass_log == after.mass_log + assert before.metadata["original"] == after.metadata["original"] + assert "is_spm_independent_minor_role" not in after.person + assert "spm_unit_spm_universe_status" not in after.table("spm_unit") + + +@requires_assembler +def test_split_preserves_all_tax_inputs_and_returns_separate_development_evidence(): + frame = source_frame() + original = rebuild(frame) + result = assemble(frame) + assert result.frame.n("spm_unit") == 3 + assert_preserved(frame, result.frame) + for entity in frame.entities: + pd.testing.assert_frame_equal( + frame.table(entity), original.table(entity), check_exact=True + ) + assert frame.metadata == original.metadata + assert RECEIPT_KEY not in frame.metadata + assert result.receipt["source_file_authentication"] is False + assert result.receipt["engine_role_delivered"] is False + assert result.receipt["annual_universe_declared"] is False + assert result.receipt["unit_amounts_copied"] is False + assert result.receipt["units_with_household_source_uncertainty"] > 0 + assert result.unit_evidence.household_source_uncertain.all() + assert "modeled_assumption" in set(result.partition.membership.role_source) + assert result.receipt["assembler"]["supported"] is True + assert ( + result.receipt["implementation_file_sha256"]["partition_helper"] + == hashlib.sha256(Path(partition.__file__).read_bytes()).hexdigest() + ) + assert result.registry.id_ceiling == CEILING + assert ( + result.frame.metadata[RECEIPT_KEY]["registry_sha256"] == result.registry.sha256 + ) + + +@requires_assembler +@pytest.mark.parametrize( + "relations,ages,count", + [ + ((20, 25), (45, 12), 1), + ((20, 36), (45, 12), 1), + ((20, 34), (45, 30), 2), + ((20, 36), (45, 21), 2), + ((20, 35), (45, 21), 1), + ((20, 35), (45, 22), 2), + ], +) +def test_native_rules_and_foster_boundary(relations, ages, count): + frame = source_frame(relations, ages) + result = assemble(frame) + assert result.frame.n("spm_unit") == count + assert_preserved(frame, result.frame) + if count == 1: + assert result.frame.person.person_spm_unit_id.eq(20).all() + if relations == (20, 36) and ages == (45, 12): + assert result.partition.membership.partition_assumption.eq( + "parent_unknown_reference_pooling" + ).any() + assert result.partition.links.empty + + +@requires_assembler +@pytest.mark.parametrize("kind,relation", [(2, 37), (3, 38)]) +def test_gq_retains_membership_and_missing_roles(kind, relation): + frame = source_frame((relation,), (30,), kind=kind) + result = assemble(frame) + pd.testing.assert_frame_equal(frame.person, result.frame.person, check_exact=True) + assert result.partition.membership.independent_minor_role.isna().all() + assert result.unit_evidence.outside_acs_household_universe.all() + assert_preserved(frame, result.frame) + + +@requires_assembler +def test_partner_sensitivity_changes_only_returned_role_evidence(): + frame = source_frame((20, 22), (45, 16)) + yes = assemble(frame) + no = assemble(frame, options=replace(OPTIONS, minor_partner_role=False)) + for entity in frame.entities: + pd.testing.assert_frame_equal( + yes.frame.table(entity), no.frame.table(entity), check_exact=True + ) + assert yes.partition.membership.independent_minor_role.iloc[1] + assert not no.partition.membership.independent_minor_role.iloc[1] + assert yes.registry.sha256 != no.registry.sha256 + + +def paired_frame(): + first, second = source_frame(), source_frame((20, 25), (40, 7), offset=1000) + tables = { + entity: pd.concat( + [first.table(entity), second.table(entity)], ignore_index=True + ) + for entity in first.entities + } + return Frame( + tables, + US_SCHEMA, + {"household": Weights([7.25, 11.5], WeightKind.DESIGN)}, + pd.Series(["acs_2024_1yr"] * 5), + metadata=first.metadata, + ) + + +@requires_assembler +def test_one_native_registry_applies_to_shuffle_and_complete_household_chunks(): + frame = paired_frame() + whole = assemble(frame) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["person"] = tables["person"].sample(frac=1, random_state=3) + shuffled = Frame( + tables, + US_SCHEMA, + dict(frame._weights), + frame.strata.reindex(tables["person"].index), + metadata=frame.metadata, + ) + replay = assemble(shuffled, registry=whole.registry) + pd.testing.assert_series_equal( + whole.frame.person.set_index("person_id").person_spm_unit_id.sort_index(), + replay.frame.person.set_index("person_id").person_spm_unit_id.sort_index(), + ) + assert assemble(shuffled).registry == whole.registry + for household_id in frame.table("household").household_id: + chunk = frame.select( + frame.person.person_household_id.eq(household_id).to_numpy() + ) + applied = assemble(chunk, registry=whole.registry) + assert applied.receipt["registry_persons"] == 5 + assert applied.registry == whole.registry + expected = whole.frame.select( + whole.frame.person.person_household_id.eq(household_id).to_numpy() + ) + for entity in expected.entities: + pd.testing.assert_frame_equal( + applied.frame.table(entity), expected.table(entity), check_exact=True + ) + + +@requires_assembler +def test_ids_remain_exact_above_float_integer_precision_and_selection_is_bound(): + frame = source_frame(offset=2**53) + result = assemble(frame, id_ceiling=np.iinfo(np.int64).max) + assert result.frame.table("spm_unit").spm_unit_id.dtype == np.dtype("int64") + assert result.crosswalk.person_id.tolist() == frame.person.person_id.tolist() + other = source_frame() + with pytest.raises(ValueError, match="REGISTRY_ROSTER_DOMAIN"): + assemble(other, registry=result.registry, id_ceiling=np.iinfo(np.int64).max) + + +@pytest.mark.parametrize("value", [True, np.bool_(False), 3.0, -1, 2**63]) +def test_ceiling_requires_exact_int64(value): + with pytest.raises(ValueError, match="EXACT_INTEGER"): + assemble(source_frame(), id_ceiling=value) + + +@requires_assembler +def test_registry_overflow_cannot_wrap_or_collide(): + with pytest.raises(ValueError, match="REGISTRY_ID_OVERFLOW"): + assemble(source_frame(), id_ceiling=21) + result = assemble(source_frame()) + entries = list(result.registry.entries) + entries[1] = replace(entries[1], new_spm_unit_id=entries[0].new_spm_unit_id) + with pytest.raises(ValueError, match="REGISTRY_ASSIGNMENT"): + assemble( + source_frame(), registry=replace(result.registry, entries=tuple(entries)) + ) + + +@requires_assembler +@pytest.mark.parametrize("change", ["source", "assembler", "options", "native_key"]) +def test_registry_rejects_changed_domain_source_or_contract(change): + frame = source_frame() + registry = assemble(frame).registry + if change == "source": + frame.person.loc[0, "AGEP"] += 1 + elif change == "assembler": + registry = replace(registry, assembler_sha256="0" * 64) + elif change == "options": + registry = replace(registry, options=replace(OPTIONS, minor_partner_role=False)) + else: + frame.table("household").loc[0, "SERIALNO"] = "DIFFERENT_NATIVE_DOMAIN" + with pytest.raises(ValueError, match="REGISTRY_"): + assemble(frame, registry=registry) + + +@pytest.mark.parametrize( + "field", ["childcare", "receives_housing_assistance", "spm_unit_source_id"] +) +def test_existing_spm_values_are_never_regrouped_or_copied(field): + frame = source_frame() + frame.table("spm_unit")[field] = 0 + with pytest.raises(ValueError, match="ID_ONLY_SPM_TABLE"): + assemble(frame) + + +@pytest.mark.parametrize( + "defect", + [ + "partial_np", + "duplicate_native", + "bad_code", + "gq_count", + "universe_conflict", + "engine_role", + "nonacs", + "wrong_weight", + ], +) +def test_source_refusals_are_not_hidden_by_unsupported_runtime(defect): + frame = source_frame() + if defect == "partial_np": + frame.table("household").loc[0, "NP"] += 1 + elif defect == "duplicate_native": + frame.person.loc[1, "SPORDER"] = "1" + elif defect == "bad_code": + frame.person.loc[0, "RELSHIPP"] = 99 + elif defect == "gq_count": + frame.table("household")["TYPEHUGQ"] = 2 + elif defect == "universe_conflict": + frame.person["TYPEHUGQ"] = 2 + elif defect == "engine_role": + frame.person["is_spm_independent_minor_role"] = False + elif defect == "nonacs": + frame.person["PERIDNUM"] = "asec-person" + elif defect == "wrong_weight": + frame = rebuild( + frame, weights={"household": Weights([7.25], WeightKind.IMPORTANCE)} + ) + with pytest.raises(ValueError): + assemble(frame) + + +@pytest.mark.parametrize( + "defect", + ["duplicate_person", "orphan_spm", "duplicate_household", "missing_native_key"], +) +def test_mutated_frame_identity_refuses_before_assembler(defect): + frame = source_frame() + if defect == "duplicate_person": + frame.person.loc[1, "person_id"] = frame.person.loc[0, "person_id"] + elif defect == "orphan_spm": + frame.person.loc[1, "person_spm_unit_id"] = 999 + elif defect == "duplicate_household": + frame._tables["household"] = pd.concat( + [frame.table("household")] * 2, ignore_index=True + ) + else: + frame.table("household").drop(columns="SERIALNO", inplace=True) + with pytest.raises(ValueError): + assemble(frame) + + +def test_options_refuse_implicit_or_unknown_choices(): + with pytest.raises(ValueError, match="PARTNER_ROLE_BOOL"): + AcsSpmSourceAssemblyOptions(partition.ACS_SPM_DEVELOPMENT_POLICY, 1) + with pytest.raises(ValueError, match="POLICY"): + AcsSpmSourceAssemblyOptions("invented-policy", True) + + +def test_explicit_request_never_falls_back_when_assembler_is_unavailable(monkeypatch): + probe = partition.AcsSpmAssemblerProbe(False, "assembler_unavailable", None, None) + monkeypatch.setattr(partition, "probe_acs_spm_assembler", lambda: probe) + with pytest.raises(partition.UnsupportedAssembler): + assemble(source_frame()) + + +def invented_archives(tmp_path): + from microcosm.build.us_runtime.acs_pums import AcsPumsSource + + household = { + "SERIALNO": "2024TEST001", + "ST": "06", + "PUMA": "00100", + "WGTP": 10, + "NP": 3, + "ADJHSG": 1_000_000, + "TEN": 3, + "RNTP": 100, + "GRNTP": 150, + "TAXAMT": 0, + "TYPEHUGQ": 1, + } + persons = [] + for order, relation, age in ((1, 20, 45), (2, 34, 30), (3, 36, 16)): + persons.append( + { + "SERIALNO": household["SERIALNO"], + "SPORDER": order, + "RELSHIPP": relation, + "AGEP": age, + "SEX": 1, + "MAR": 5, + "ADJINC": 1_000_000, + "WAGP": 50_000, + "SEMP": 0, + "SSP": 0, + "SSIP": 0, + "RETP": 0, + "INTP": 0, + "PWGTP": 10, + } + ) + for name, member, rows in ( + ("csv_hus.zip", "psam_husa.csv", [household]), + ("csv_pus.zip", "psam_pusa.csv", persons), + ): + with ZipFile(tmp_path / name, "w") as archive: + archive.writestr(member, pd.DataFrame(rows).to_csv(index=False)) + return AcsPumsSource(tmp_path / "csv_hus.zip", tmp_path / "csv_pus.zip") + + +@requires_assembler +def test_real_constructor_maps_tenure_on_every_split_and_housing_only_to_head(tmp_path): + from microcosm.build.us_runtime.acs_inputs import map_acs_native_inputs + from microcosm.build.us_runtime.acs_pums import build_acs_pums_unit_frame + from microcosm.build.us_runtime.housing_participation import route_participation + from microcosm.build.us_runtime.operator_column_contracts import ( + PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID, + ) + + source = invented_archives(tmp_path) + old, old_receipt = build_acs_pums_unit_frame(source, chunksize=1) + new, receipt = build_acs_pums_unit_frame( + source, chunksize=1, spm_construction=OPTIONS + ) + # The constructor does not change unrelated frame metadata or inputs. + for entity in old.entities: + if entity == "spm_unit": + continue + columns = [name for name in old.table(entity) if name != "person_spm_unit_id"] + pd.testing.assert_frame_equal( + old.table(entity)[columns], new.table(entity)[columns], check_exact=True + ) + assert old.n("spm_unit") == 1 and new.n("spm_unit") == 3 + assert RECEIPT_KEY not in old_receipt + assert ( + receipt[RECEIPT_KEY]["registry_id_ceiling"] + == PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID + ) + mapped = map_acs_native_inputs(new).frame + assert mapped.table("spm_unit").spm_unit_tenure_type.eq("RENTER").all() + mapped.person["is_household_head"] = mapped.person.RELSHIPP.eq(20) + flags = route_participation( + mapped, pd.Series(True, index=mapped.table("household").household_id) + ) + head_spm = mapped.person.loc[ + mapped.person.RELSHIPP.eq(20), "person_spm_unit_id" + ].iloc[0] + assert flags.receives_housing_assistance.sum() == 1 + assert bool(flags.loc[head_spm, "receives_housing_assistance"]) + assert flags.takes_up_housing_assistance_if_eligible.equals( + flags.receives_housing_assistance + ) + + +def test_constructor_refuses_before_source_io_and_default_does_not_probe( + tmp_path, monkeypatch +): + from microcosm.build.us_runtime import acs_pums + + source = invented_archives(tmp_path) + probe = partition.AcsSpmAssemblerProbe( + False, "incompatible_parent_link_order", "invented", "0" * 64 + ) + calls = [] + + def unavailable(): + calls.append(True) + return probe + + monkeypatch.setattr(partition, "probe_acs_spm_assembler", unavailable) + old, receipt = acs_pums.build_acs_pums_unit_frame(source) + assert calls == [] and old.n("spm_unit") == 1 and RECEIPT_KEY not in receipt + + def forbidden(*args, **kwargs): + raise AssertionError("Source reader reached after unsupported capability") + + monkeypatch.setattr(acs_pums, "load_acs_pums_tables", forbidden) + with pytest.raises(partition.UnsupportedAssembler): + acs_pums.build_acs_pums_unit_frame(source, spm_construction=OPTIONS) + assert calls == [True] + + +@requires_assembler +def test_registry_component_and_duplicate_native_key_corruption_refuse(): + frame = source_frame() + registry = assemble(frame).registry + changed = replace(registry.entries[0], component_sha256="0" * 64) + with pytest.raises(ValueError, match="REGISTRY_COMPONENT_IDENTITY"): + assemble( + frame, registry=replace(registry, entries=(changed, *registry.entries[1:])) + ) + with pytest.raises(ValueError, match="REGISTRY_NATIVE_KEYS"): + assemble( + frame, + registry=replace( + registry, entries=(*registry.entries, registry.entries[-1]) + ), + ) diff --git a/packages/microcosm-build/tests/test_us_acs_transfer.py b/packages/microcosm-build/tests/test_us_acs_transfer.py index e9e615bec..707af9224 100644 --- a/packages/microcosm-build/tests/test_us_acs_transfer.py +++ b/packages/microcosm-build/tests/test_us_acs_transfer.py @@ -2,6 +2,7 @@ import os import stat +from dataclasses import replace from pathlib import Path from types import SimpleNamespace @@ -11,6 +12,7 @@ import microcosm.build.us_runtime.acs_transfer as acs_transfer_module import microcosm.build.us_runtime.acs_transfer_bank as acs_transfer_bank_module +import microcosm.build.us_runtime.immigration as immigration_module from microcosm.build.frame_checkpoint import write_frame_checkpoint from microcosm.build.serialization_dtypes import CANONICAL_STRING_DTYPE from microcosm.build.us_runtime.acs_transfer import ( @@ -22,6 +24,7 @@ ACS_PERSON_TRANSFER_PREDICTORS, AcsTransferResult, acs_adult_care_qualifying_rows, + acs_transfer_donor_requirements, declared_acs_transfer_target_families, default_acs_transfer_target_families, transfer_acs_inputs, @@ -29,9 +32,15 @@ from microcosm.build.us_runtime.acs_transfer_bank import ( AcsTransferTargetBankStore, ) +from microcosm.build.us_runtime.immigration import ( + HumanitarianDraw, + ImmigrationControls, + UndocumentedControls, +) from microcosm.build.us_runtime.puf_support import clone_us_frame_for_puf_support from microcosm.build.us_runtime.spine_assembly import assemble_spines -from microcosm.fit import Regime +from microcosm.fit import QRF, Regime +from microcosm.fit.qrf import _index_identity from microcosm.frame import US_SCHEMA, EntitySchema, Frame, WeightKind, Weights from microcosm.frame.adapters.policyengine_us import ( PolicyEngineUSVariableMetadataIndex, @@ -84,6 +93,9 @@ def _donor_frame() -> Frame: ), "age": [45.0, 43.0, 30.0, 8.0, 68.0, 66.0, 39.0, 17.0], "is_female": [False, True, True, False, False, True, True, False], + "PRCITSHP": [1, 1, 1, 1, 1, 1, 5, 5], + "PENATVTY": [57, 57, 57, 57, 57, 57, 164, 373], + "PEINUSYR": [0, 0, 0, 0, 0, 0, 28, 24], "is_household_head": [True, False, True, False, True, False, True, False], "employment_income_before_lsr": [ 80_000.0, @@ -208,6 +220,9 @@ def _recipient_frame() -> Frame: ), "age": [41.0, 40.0, 27.0, 72.0, 69.0, 15.0], "is_female": [False, True, True, False, True, False], + "CIT": [1, 1, 5, 1, 1, 5], + "POBP": [6, 6, 164, 36, 36, 373], + "YOEP": [np.nan, np.nan, 2023, np.nan, np.nan, 2015], "is_household_head": [True, False, True, True, False, False], "employment_income_before_lsr": [ 65_000.0, @@ -263,6 +278,90 @@ def _recipient_frame() -> Frame: ) +def _mixed_immigration_recipient() -> Frame: + """One immutable ASEC row followed by ACS rows with paired null targets.""" + + base = _recipient_frame() + tables = {entity: base.table(entity).copy() for entity in base.entities} + person = tables["person"] + for column in ("PRCITSHP", "PENATVTY", "PEINUSYR"): + person[column] = np.nan + immutable_index = person.index[0] + person.loc[immutable_index, ["CIT", "POBP", "YOEP"]] = np.nan + person.loc[immutable_index, ["PRCITSHP", "PENATVTY", "PEINUSYR"]] = [ + 5, + 200, + 27, + ] + person["ssn_card_type"] = pd.Series( + ["OTHER_NON_CITIZEN", *([np.nan] * (len(person) - 1))], + index=person.index, + dtype=object, + ) + person["immigration_status_str"] = pd.Series( + ["PAROLED_ONE_YEAR", *([np.nan] * (len(person) - 1))], + index=person.index, + dtype=object, + ) + return Frame( + tables, + base.schema, + {entity: base.weights_for(entity) for entity in base.weighted_entities}, + base.strata, + mass_log=base.mass_log, + ) + + +def _small_immigration_controls( + *, + ukraine_target: float = 60.0, +) -> ImmigrationControls: + source = "https://example.com/immigration-control" + return ImmigrationControls( + undocumented=UndocumentedControls( + workers=1.0, + students=1.0, + population_anchor=1.0, + sources={ + "undocumented_workers": source, + "undocumented_students": source, + "undocumented_population_anchor": source, + }, + ), + humanitarian=( + HumanitarianDraw( + category="paroled_one_year", + origin="afghanistan", + status="PAROLED_ONE_YEAR", + target=50.0, + source=source, + ), + HumanitarianDraw( + category="paroled_one_year", + origin="ukraine", + status="PAROLED_ONE_YEAR", + target=ukraine_target, + source=source, + ), + HumanitarianDraw( + category="tps", + origin="venezuela", + status="TPS", + target=70.0, + source=source, + ), + ), + ) + + +def _no_humanitarian_controls() -> ImmigrationControls: + controls = _small_immigration_controls() + return ImmigrationControls( + undocumented=controls.undocumented, + humanitarian=(), + ) + + def _replace_column( frame: Frame, entity: str, @@ -489,6 +588,33 @@ def predict(self, frame: pd.DataFrame) -> pd.DataFrame: ) +class _MeanChainQRF(_MeanQRF): + """Constant invented draws with genuine checkpoint state, without fitting.""" + + def start_chain(self, frame, predictors, targets, *, weights): + # The maintained initializer only binds identity and RNG state; no fit. + return QRF(n_estimators=self.n_estimators, seed=self.seed).start_chain( + frame, predictors, targets, weights=weights + ) + + def fit_draw_next(self, frame, recipient, priors, *, state, weights): + assert weights == state.weight_kind + assert tuple(priors.columns) == state.completed_targets + target = state.next_target + assert target is not None + mean = float(frame.table(state.entity)[target].mean()) + return SimpleNamespace( + target=target, + raw_draw=np.full(len(recipient), mean, dtype=np.float64), + state=replace( + state, + completed_targets=(*state.completed_targets, target), + recipient_index=_index_identity(recipient.index), + ), + weight_kind=state.weight_kind, + ) + + def _run_bank_fixture( target_bank: AcsTransferTargetBankStore | None = None, ) -> AcsTransferResult: @@ -677,7 +803,6 @@ def test_explicit_declared_plan_preserves_deferred_geography( lambda: plan, ) monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) - result = transfer_acs_inputs( _recipient_frame(), _donor_frame(), @@ -745,6 +870,59 @@ def test_declared_families_are_independent_of_release_coverage_surface() -> None assert "receives_wic" in production_declared["person"]["model_required_boolean"] +def test_donor_requirements_include_immigration_source_evidence_triplet() -> None: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["CITIZEN"] * 6 + ["NONE", "NONE"], + "immigration_status_str": ["CITIZEN"] * 6 + + ["UNDOCUMENTED", "UNDOCUMENTED"], + }, + ) + plan = { + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + } + + requirements = acs_transfer_donor_requirements(donor, plan) + + assert {"PRCITSHP", "PENATVTY", "PEINUSYR"}.issubset(requirements["person"]) + + acs_tables = {entity: donor.table(entity).copy() for entity in donor.entities} + acs_person = acs_tables["person"].rename( + columns={"PRCITSHP": "CIT", "PENATVTY": "POBP"} + ) + acs_person = acs_person.drop(columns=["PEINUSYR"]) + acs_person["YOEP"] = [np.nan] * 6 + [2023, 2015] + acs_tables["person"] = acs_person + acs_donor = Frame( + acs_tables, + donor.schema, + {entity: donor.weights_for(entity) for entity in donor.weighted_entities}, + donor.strata, + mass_log=donor.mass_log, + ) + acs_requirements = acs_transfer_donor_requirements(acs_donor, plan) + assert {"CIT", "POBP", "YOEP"}.issubset(acs_requirements["person"]) + + tables = {entity: donor.table(entity).copy() for entity in donor.entities} + tables["person"] = tables["person"].drop(columns=["PEINUSYR"]) + incomplete = Frame( + tables, + donor.schema, + {entity: donor.weights_for(entity) for entity in donor.weighted_entities}, + donor.strata, + mass_log=donor.mass_log, + ) + with pytest.raises(ValueError, match="incomplete donor evidence triplet"): + acs_transfer_donor_requirements(incomplete, plan) + + def test_declared_plan_carries_the_23_stage_base_surface() -> None: """The M/N/O donor repairs must reach the ACS spine through the plan.""" @@ -871,6 +1049,451 @@ def test_explicit_transfer_adds_requested_model_inputs( and "immigration_status_str" not in call["targets"] for call in joint_calls ) + assert { + "__acs_transfer_is_us_citizen", + "__acs_transfer_birth_country_code", + "__acs_transfer_arrival_year", + }.issubset(joint_calls[0]["predictors"]) + + +def test_joint_immigration_codec_never_emits_constrained_baseline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + constrained = [ + "PAROLED_ONE_YEAR", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "TPS", + "CUBAN_HAITIAN_ENTRANT", + "DACA", + "REFUGEE", + ] + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["OTHER_NON_CITIZEN"] * 8, + "immigration_status_str": constrained, + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _no_humanitarian_controls, + ) + + result = transfer_acs_inputs( + _recipient_frame(), + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + n_estimators=1, + ) + + assert set(result.frame.person["immigration_status_str"]) <= { + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + } + encoding = acs_transfer_module._target_encodings( + donor.person, + targets=("ssn_card_type", "immigration_status_str"), + )["immigration_status_str"] + assert not {pair[1] for pair in encoding.categories}.intersection(constrained) + + +def test_transfer_rederives_entrant_and_daca_only_on_compatible_acs_cohorts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["NON_CITIZEN_VALID_EAD"] * 8, + "immigration_status_str": [ + "CUBAN_HAITIAN_ENTRANT", + "DACA", + ] + * 4, + }, + ) + recipient = _with_columns( + _recipient_frame(), + "person", + { + "age": [50.0, 50.0, 29.0, 50.0, 40.0, 70.0], + "CIT": [5, 5, 5, 5, 1, 5], + "POBP": [327, 164, 312, 312, 6, 332], + "YOEP": [2000, 2020, 2005, 2020, np.nan, 1970], + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _no_humanitarian_controls, + ) + + result = transfer_acs_inputs( + recipient, + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + n_estimators=1, + ) + + assert result.frame.person["immigration_status_str"].tolist() == [ + "CUBAN_HAITIAN_ENTRANT", + "LEGAL_PERMANENT_RESIDENT", + "DACA", + "LEGAL_PERMANENT_RESIDENT", + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + ] + receipt = next( + item.reconciliation + for item in result.imputed_inputs + if item.column == "immigration_status_str" + ) + assert receipt is not None + assert receipt["special_status_assignments"] == 2 + + +def _captured_controls_transfer_inputs(): + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["OTHER_NON_CITIZEN"] * 8, + "immigration_status_str": ["LEGAL_PERMANENT_RESIDENT"] * 8, + "fixture_observation_year": [2025] * 8, + }, + ) + recipient = _with_metadata( + _with_columns( + _mixed_immigration_recipient(), + "person", + {"fixture_observation_year": [2025] + [2024] * 5}, + ), + {"fixture_context": {"retained": "original recipient"}}, + ) + options = { + "target_families": { + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + "donor_channel": None, + "seed": 19, + "n_estimators": 1, + "observation_year_column": "fixture_observation_year", + } + return donor, recipient, options + + +def test_captured_controls_match_default_and_survive_bank_recovery( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + """The real common reconciler consumes the captured object on every route.""" + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanChainQRF) + donor, recipient, options = _captured_controls_transfer_inputs() + controls = _small_immigration_controls() + loader_calls = [] + + def load(): + loader_calls.append(controls) + return controls + + monkeypatch.setattr(immigration_module, "us_immigration_controls", load) + default = transfer_acs_inputs(recipient, donor, **options) + assert loader_calls == [controls] + + def forbid_reload(): + raise AssertionError("captured controls must not reload") + + monkeypatch.setattr(immigration_module, "us_immigration_controls", forbid_reload) + consumed = [] + real_reconcile = acs_transfer_module.reconcile_us_immigration_humanitarian_transfer + + def observe(person, **kwargs): + assert kwargs["controls"] is controls + consumed.append(kwargs["controls"]) + return real_reconcile(person, **kwargs) + + monkeypatch.setattr( + acs_transfer_module, "reconcile_us_immigration_humanitarian_transfer", observe + ) + + def run(bank=None): + return transfer_acs_inputs( + recipient, donor, **options, target_bank=bank, immigration_controls=controls + ) + + ordinary = run() + cold = run(_bank_store(tmp_path / "cold")) + warm_bank = _bank_store(tmp_path / "cold") + warm = run(warm_bank) + assert warm_bank.receipt()["targets"]["0"]["source"] == "checkpoint" + assert len(consumed) == 3 + + interrupted = _bank_store(tmp_path / "interrupted") + write = interrupted.write_target + + def interrupt_after_write(checkpoint): + write(checkpoint) + raise RuntimeError("invented interruption after durable paired target") + + monkeypatch.setattr(interrupted, "write_target", interrupt_after_write) + with pytest.raises(RuntimeError, match="invented interruption"): + run(interrupted) + assert len(consumed) == 3 # no post-step after an incomplete transfer + recovered_bank = _bank_store(tmp_path / "interrupted") + recovered = run(recovered_bank) + assert recovered_bank.receipt()["targets"]["0"]["source"] == "checkpoint" + assert len(consumed) == 4 + + for result in (ordinary, cold, warm, recovered): + _assert_transfer_results_exact(result, default) + assert result.frame.metadata == recipient.metadata + assert result.frame.mass_log == recipient.mass_log + for entity in recipient.weighted_entities: + expected = recipient.weights_for(entity) + actual = result.frame.weights_for(entity) + assert actual.kind is expected.kind + np.testing.assert_array_equal(actual.values, expected.values) + for entity in recipient.entities: + columns = recipient.table(entity).columns.difference( + ["ssn_card_type", "immigration_status_str"] + ) + pd.testing.assert_frame_equal( + result.frame.table(entity)[columns], recipient.table(entity)[columns] + ) + records = [r for r in result.imputed_inputs if r.reconciliation is not None] + assert len(records) == 2 + assert records[0].reconciliation is records[1].reconciliation + assert records[0].reconciliation["observation_years"]["row_counts"] == { + "2024": 5, + "2025": 1, + } + assert ( + result.frame.person.iloc[0]["immigration_status_str"] == "PAROLED_ONE_YEAR" + ) + + +def test_captured_controls_retain_candidate_shortfall(monkeypatch): + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + donor, recipient, options = _captured_controls_transfer_inputs() + with pytest.raises( + ValueError, match="candidate shortfall.*paroled_one_year:ukraine" + ): + transfer_acs_inputs( + recipient, + donor, + **options, + immigration_controls=_small_immigration_controls(ukraine_target=61.0), + ) + + +@pytest.mark.parametrize( + "case", ["empty", "complete", "complete_with_other_target", "custom_family"] +) +@pytest.mark.parametrize("explicit", [False, True]) +def test_captured_controls_do_not_activate_inactive_reconciliation( + monkeypatch, case, explicit +): + donor, recipient, options = _captured_controls_transfer_inputs() + options.pop("observation_year_column") + if case == "empty": + options["target_families"] = {} + elif case in {"complete", "complete_with_other_target"}: + recipient = _with_columns( + recipient, + "person", + { + "ssn_card_type": ["OTHER_NON_CITIZEN"] * 6, + "immigration_status_str": ["LEGAL_PERMANENT_RESIDENT"] * 6, + }, + ) + if case == "complete_with_other_target": + options["target_families"]["person"]["fixture_tax_detail"] = ( + "qualified_dividend_income", + ) + else: + options["target_families"] = { + "person": {"fixture_custom": ("ssn_card_type", "immigration_status_str")} + } + if explicit: + options["immigration_controls"] = _small_immigration_controls() + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + + def refuse(*args, **kwargs): + raise AssertionError("inactive immigration must not consume controls") + + monkeypatch.setattr(immigration_module, "us_immigration_controls", refuse) + monkeypatch.setattr( + acs_transfer_module, "reconcile_us_immigration_humanitarian_transfer", refuse + ) + result = transfer_acs_inputs(recipient, donor, **options) + assert all(record.reconciliation is None for record in result.imputed_inputs) + if case == "complete_with_other_target": + assert [record.column for record in result.imputed_inputs] == [ + "qualified_dividend_income" + ] + + +@pytest.mark.parametrize("case", ["incomplete_pair", "unequal_masks"]) +def test_captured_controls_preserve_paired_target_refusals(monkeypatch, case): + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + donor, recipient, options = _captured_controls_transfer_inputs() + options.pop("observation_year_column") + if case == "incomplete_pair": + options["target_families"] = { + "person": {"source_operator_immigration": ("ssn_card_type",)} + } + message = "missing paired target" + else: + recipient.person.loc[recipient.person.index[1], "ssn_card_type"] = "CITIZEN" + message = "exactly the same recipient rows" + with pytest.raises(ValueError, match=message): + transfer_acs_inputs( + recipient, + donor, + **options, + immigration_controls=_small_immigration_controls(), + ) + + +def test_humanitarian_reconciliation_uses_residual_targets_and_records_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["OTHER_NON_CITIZEN"] * 8, + "immigration_status_str": ["LEGAL_PERMANENT_RESIDENT"] * 8, + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _small_immigration_controls, + ) + + result = transfer_acs_inputs( + _mixed_immigration_recipient(), + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + seed=19, + n_estimators=1, + ) + + person = result.frame.person + assert person["immigration_status_str"].tolist() == [ + "PAROLED_ONE_YEAR", # immutable Afghanistan contribution + "CITIZEN", # source CIT repairs the imputed baseline pair + "PAROLED_ONE_YEAR", # compatible Ukrainian residual + "CITIZEN", + "CITIZEN", + "TPS", # compatible Venezuelan residual + ] + assert person.loc[person.index[0], "ssn_card_type"] == "OTHER_NON_CITIZEN" + reconciled = [ + item + for item in result.imputed_inputs + if item.column in {"ssn_card_type", "immigration_status_str"} + ] + assert len(reconciled) == 2 + assert reconciled[0].reconciliation == reconciled[1].reconciliation + receipt = reconciled[0].reconciliation + assert receipt is not None + assert receipt["mutable_rows"] == 5 + assert receipt["immutable_rows"] == 1 + draws = receipt["draws"] + assert draws["paroled_one_year:afghanistan"] == { + "target": 50.0, + "immutable_population": 50.0, + "residual_target": 0.0, + "eligible_recipient_population": 0.0, + "selected_recipient_population": 0.0, + "achieved_population": 50.0, + "absolute_error": 0.0, + "selection_threshold_tie_population": 0.0, + "residual_selection_error": 0.0, + "within_residual_discrete_weight_bound": True, + "immutable_overshoot": 0.0, + "within_discrete_weight_bound": True, + } + assert draws["paroled_one_year:ukraine"]["residual_target"] == 60.0 + assert draws["paroled_one_year:ukraine"]["achieved_population"] == 60.0 + assert draws["tps:venezuela"]["achieved_population"] == 70.0 + assert all(draw["within_discrete_weight_bound"] for draw in draws.values()) + + +def test_humanitarian_reconciliation_fails_on_candidate_shortfall( + monkeypatch: pytest.MonkeyPatch, +) -> None: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["OTHER_NON_CITIZEN"] * 8, + "immigration_status_str": ["LEGAL_PERMANENT_RESIDENT"] * 8, + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + lambda: _small_immigration_controls(ukraine_target=61.0), + ) + + with pytest.raises( + ValueError, + match="candidate shortfall.*paroled_one_year:ukraine", + ): + transfer_acs_inputs( + _mixed_immigration_recipient(), + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + seed=19, + n_estimators=1, + ) def test_large_target_family_is_split_to_bound_retained_qrf_forests( @@ -1158,6 +1781,11 @@ def test_target_bank_resumes_joint_immigration_codec_as_one_model_target( tmp_path: Path, ) -> None: _lock_bank_fixture_threads(monkeypatch) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _no_humanitarian_controls, + ) donor = _with_columns( _donor_frame(), "person", @@ -1467,9 +2095,8 @@ def test_pregnancy_draws_once_per_eligible_source_person_and_fans_to_clones( ) person = result.frame.person - eligible = ( - person["is_female"].astype(bool) - & person["age"].between(15, 44, inclusive="both") + eligible = person["is_female"].astype(bool) & person["age"].between( + 15, 44, inclusive="both" ) assert person.loc[eligible, "is_pregnant"].all() assert not person.loc[~eligible, "is_pregnant"].any() @@ -1490,7 +2117,9 @@ def test_pregnancy_draws_once_per_eligible_source_person_and_fans_to_clones( .first() .sum() ) - record = next(item for item in result.imputed_inputs if item.column == "is_pregnant") + record = next( + item for item in result.imputed_inputs if item.column == "is_pregnant" + ) receipt = record.structural_receipt assert receipt is not None assert sum(pattern.recipient_rows for pattern in record.patterns) == ( @@ -1576,12 +2205,8 @@ def test_pregnancy_partial_clone_fanout_receipt_categories_are_disjoint( record = result.imputed_inputs[0] receipt = record.structural_receipt assert receipt is not None - assert receipt["preexisting_value_fanout_rows"] == int( - (missing & eligible).sum() - ) - assert receipt["ineligible_rows_assigned_false"] == int( - (missing & ~eligible).sum() - ) + assert receipt["preexisting_value_fanout_rows"] == int((missing & eligible).sum()) + assert receipt["ineligible_rows_assigned_false"] == int((missing & ~eligible).sum()) assert ( receipt["preexisting_value_fanout_rows"] + receipt["ineligible_rows_assigned_false"] @@ -2468,7 +3093,12 @@ def test_strict_leaf_audit_reports_missing_us_extra( from microcosm.frame.adapters import policyengine_us as adapter_module class _MissingMetadataIndex: - def __init__(self) -> None: + # Mirrors PolicyEngineUSVariableMetadataIndex's keyword-only surface. + # The audit path constructs it twice: puf_support._formula_owned_engine + # passes include_consumers=False before the strict-leaf branch + # constructs it bare, and both must raise the absent-extra ImportError + # rather than a signature TypeError. + def __init__(self, *, include_consumers: bool = True) -> None: raise ImportError("policyengine-us is absent") monkeypatch.setattr( diff --git a/packages/microcosm-build/tests/test_us_asec_2024_native_population.py b/packages/microcosm-build/tests/test_us_asec_2024_native_population.py new file mode 100644 index 000000000..79b14b6ea --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_2024_native_population.py @@ -0,0 +1,692 @@ +"""Real closed loaders over invented source bytes; no genuine source admission.""" + +import csv +import hashlib +import importlib.util +import json +import sys +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.frame_checkpoint import ( + load_frame_checkpoint, + write_frame_checkpoint, +) +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.us_runtime import asec_2024_native_population as native +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_current_money_source as money +from microcosm.build.us_runtime import asec_household_observations as household_owner +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import native_household_origin as origin +from microcosm.frame import Frame, WeightKind, Weights + + +def _helpers(): + spec = importlib.util.spec_from_file_location( + "native_2024_invented_fixture", + Path(__file__).with_name("test_us_asec_coverage_authentication.py"), + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _rebuild_observations( + tmp_path, + monkeypatch, + persons, + *, + extra_household, + legacy_weights, + missing_money=None, + current_predictor_money=None, +): + """Build larger invented real checkpoints and the actual observation attachment.""" + parent_path = tmp_path / "invented-original-v4.h5" + household_path = tmp_path / "invented-household-attachment.h5" + loaded = load_frame_checkpoint(parent_path) + tables = {e: loaded.frame.table(e).copy(deep=True) for e in loaded.frame.entities} + strata = loaded.frame.strata.copy(deep=True) + if extra_household: + extra = tables["person"].iloc[-2:].copy(deep=True) + extra["person_id"] = np.array([107, 108], dtype=np.int64) + extra["PERIDNUM"] = pd.array([str(i).zfill(22) for i in (7, 8)], dtype="string") + extra["source_household_id"] = 8 + extra["PH_SEQ"] = 4 + for entity in loaded.frame.schema.group_entities: + extra[f"person_{entity}_id"] = 4 + group = tables[entity].iloc[-1:].copy(deep=True) + group[f"{entity}_id"] = 4 + tables[entity] = pd.concat([tables[entity], group], ignore_index=True) + tables["person"] = pd.concat([tables["person"], extra]) + strata = pd.concat([strata, strata.iloc[-2:]]) + cohort_path = tmp_path / "invented-cohort-2024.h5" + cohort = pd.read_hdf(cohort_path, key="household") + extra_cohort = cohort.iloc[-1:].copy(deep=True) + extra_cohort["H_SEQ"] = 8 + pd.concat([cohort, extra_cohort], ignore_index=True).to_hdf( + cohort_path, key="household", format="fixed", mode="w" + ) + + def add_people(rows): + extra_rows = [r.copy() for r in rows[1:]] + for row in extra_rows: + row[0] = str(int(row[0]) + 2).zfill(22) + row[1] = "8" + rows.extend(extra_rows) + + _helpers()._rewrite(persons[2024], add_people) + if missing_money is not None: + year, field = missing_money + assert (year, field) in ((2024, "ANN_VAL"), (2022, "WSAL_VAL")) + person = tables["person"] + position = np.flatnonzero(person.source_year.to_numpy() == year)[0] + # Alter only invented source construction, before identities, attachments + # and normal issuance. Never mutate an already authenticated parent. + person.iloc[position, person.columns.get_loc(field)] = np.nan + if current_predictor_money is not None: + # Alter only invented original source construction before checkpoint + # identities, observation attachments, private pins and fresh issuance. + person = tables["person"] + positions = np.flatnonzero(person.source_year.to_numpy() == 2024) + assert set(current_predictor_money) == { + "WSAL_VAL", + "SEMP_VAL", + "INT_VAL", + "DIV_VAL", + "CAP_VAL", + } + for field, observations in current_predictor_money.items(): + observations = np.asarray(observations) + assert observations.dtype == np.dtype("float64") + assert ( + observations.shape == (len(positions),) + and np.isfinite(observations).all() + ) + person.iloc[positions, person.columns.get_loc(field)] = observations + values = ( + legacy_weights + if legacy_weights is not None + else ([1, 2, 3, 9] if extra_household else [1, 2, 3]) + ) + frame = Frame( + tables, + loaded.frame.schema, + {"household": Weights(np.array(values), WeightKind.DESIGN)}, + strata, + ) + metadata = loaded.metadata + for key in ("identity", "source_construction_identity"): + metadata[key] = frame_identity(frame).to_payload() + sources = [] + for year in (2022, 2023, 2024): + path = tmp_path / f"invented-cohort-{year}.h5" + sources.append( + household_owner.AsecHouseholdObservationSource( + year, path, hashlib.sha256(path.read_bytes()).hexdigest() + ) + ) + metadata["source_receipt"]["sources"] = [ + {"year": source.year, "path": str(source.path), "sha256": source.sha256} + for source in sources + ] + write_frame_checkpoint(parent_path, frame, metadata=metadata) + parent_sha = hashlib.sha256(parent_path.read_bytes()).hexdigest() + attachment = household_owner.with_asec_household_observations( + frame, + checkpoint_metadata=metadata, + checkpoint_sha256=parent_sha, + sources=sources, + ) + write_frame_checkpoint( + household_path, + attachment.frame, + metadata={ + "schema_version": 1, + "artifact_kind": "microcosm.asec_household_observations_source", + "parent_checkpoint_sha256": parent_sha, + "household_observations": attachment.receipt, + }, + ) + monkeypatch.setattr( + money, + "_SOURCE_PINS", + ( + parent_sha, + hashlib.sha256(household_path.read_bytes()).hexdigest(), + tuple((s.year, s.sha256) for s in sources), + ), + ) + _helpers()._pin(persons, monkeypatch) + + +def _fixture( + tmp_path, + monkeypatch, + *, + household_rows=None, + extra_household=False, + legacy_weights=None, + missing_money=None, + current_predictor_money=None, + **changes, +): + helpers = _helpers() + _, persons = helpers._fixtures(tmp_path, monkeypatch, **changes) + if ( + extra_household + or legacy_weights is not None + or missing_money is not None + or current_predictor_money is not None + ): + _rebuild_observations( + tmp_path, + monkeypatch, + persons, + extra_household=extra_household, + legacy_weights=legacy_weights, + missing_money=missing_money, + current_predictor_money=current_predictor_money, + ) + # Both real owners must pin the same final invented original member bytes. + monkeypatch.setattr(restoration, "_MEMBER_PINS", coverage._MEMBER_PINS) + parent = tmp_path / "invented-original-v4.h5" + household = tmp_path / "invented-household-attachment.h5" + output = tmp_path / "restored" + restoration.restore_asec_person_income_source( + parent, household, member_paths=persons, output_dir=output + ) + member = tmp_path / "hhpub25.csv" + rows = ( + household_rows + if household_rows is not None + else [ + ["00007", "1", "000255212", "6", "1", "2"], + ["00008", "2", "", "0", "0", "0"], + ] + ) + if extra_household and household_rows is None: + rows[1] = ["00008", "1", "00000000", "6", "1", "2"] + with member.open("w", newline="") as handle: + writer = csv.writer(handle) + writer.writerow( + ["H_SEQ", "H_HHTYPE", "HSUP_WGT", "HRHTYPE", "H_LIVQRT", "H_NUMPER"] + ) + writer.writerows(rows) + data = member.read_bytes() + monkeypatch.setattr( + origin, + "_ASEC_MEMBER_PINS", + ( + origin.AsecNativeMemberPin( + income_year=2024, + survey_year=2025, + canonical_member_id="invented/2025/household/hhpub25.csv", + member_name="hhpub25.csv", + archive_sha256=origin.ASEC_EDUCATION_ASSISTANCE_ARCHIVES[ + 2024 + ].zip_sha256, + member_sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + rows=len(rows), + ), + ), + ) + return dict( + parent_path=parent, + household_attachment_path=household, + person_income_attachment_path=output / restoration.CHECKPOINT_FILENAME, + person_member_paths=persons, + household_member_path=member, + ) + + +def test_real_three_cohort_loader_selects_2024_original_anchors(tmp_path, monkeypatch): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + frame, receipt = result.frame, result.receipt + assert frame.person.source_year.tolist() == [2024, 2024] + assert frame.person.person_id.tolist() == [105, 106] + assert frame.person.A_AGE.tolist() == [55, 14] + assert frame.person.asec_PTOTVAL.tolist() == [-9999, 0] + assert frame.weights_for("household").kind is WeightKind.DESIGN + assert frame.weights_for("household").values.tolist() == [2552.12] + assert dict(frame.metadata) == {} and frame.mass_log == () + assert json.loads(result.context)["entities"]["person"]["rows"] == 2 + assert receipt["parent_custody"]["cohorts"] == [2022, 2023, 2024] + assert receipt["parent_custody"]["population_cohorts"] == [2024] + assert receipt["households"][0]["HSUP_WGT"] == "000255212" + assert receipt["households"][0]["fraction"] == [63803, 25] + assert receipt["households"][0]["native_key"] == [2024, 7] + assert not receipt["release_eligible"] + assert not receipt["selection"]["inclusion_probabilities_known"] + assert {r["A_LINENO"] for r in receipt["persons"]} == {1, 2} + for entity in frame.schema.group_entities: + assert len(frame.table(entity)) == 1 + # Real parent remains separate, complete and sealed after all child borrows. + state = native._ISSUED[id(result)][2] + assert state.parent.frame.person.source_year.tolist() == [ + 2022, + 2022, + 2023, + 2023, + 2024, + 2024, + ] + assert state.parent.frame.weights_for("household").values.tolist() == [1, 2, 3] + state.parent.validate() + + +def test_exact_selection_candidate_reconstruction_and_value_equal_copy_refusal( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + candidate = result.to_bytes() + reconstructed = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),), candidate=candidate + ) + assert reconstructed == result and reconstructed is not result + reconstructed.validate() + for copy in ( + replace(result), + native.AuthenticatedAsec2024NativePopulation(candidate), + ): + assert copy == result + with pytest.raises(native.AsecNativePopulationError, match="UNISSUED"): + copy.validate() + with pytest.raises(native.AsecNativePopulationError, match="CANDIDATE_MISMATCH"): + native.load_authenticated_asec_2024_native_population(**paths, candidate=b"{}") + + +@pytest.mark.parametrize( + "selection", + [ + (), + ((2023, 7),), + ((2024, 7), (2024, 7)), + ((2024, True),), + [(2024, 7)], + ((2024, 8),), + ], +) +def test_non_exact_or_absent_selection_refuses(tmp_path, monkeypatch, selection): + paths = _fixture(tmp_path, monkeypatch) + with pytest.raises(native.AsecNativePopulationError, match="SELECTION"): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=selection + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("H_NUMPER", "1"), + ("H_NUMPER", ""), + ("H_SEQ", "9"), + ("HSUP_WGT", "2552.12"), + ("HSUP_WGT", ""), + ("HSUP_WGT", "-1"), + ("HSUP_WGT", "0"), + ("H_HHTYPE", "2"), + ], +) +def test_source_household_roster_or_unusable_anchor_refuses( + tmp_path, monkeypatch, field, value +): + names = ["H_SEQ", "H_HHTYPE", "HSUP_WGT", "HRHTYPE", "H_LIVQRT", "H_NUMPER"] + row = ["00007", "1", "255212", "6", "1", "2"] + row[names.index(field)] = value + paths = _fixture(tmp_path, monkeypatch, household_rows=[row]) + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population(**paths) + + +def test_absent_interview_household_not_silently_lost(tmp_path, monkeypatch): + paths = _fixture( + tmp_path, + monkeypatch, + household_rows=[ + ["7", "1", "100", "6", "1", "2"], + ["8", "1", "200", "6", "1", "1"], + ], + ) + with pytest.raises( + native.AsecNativePopulationError, match="HOUSEHOLD_MEMBER_COUNT" + ): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "age", + "unrelated", + "group", + "weight", + "strata", + "strata_flags", + "table_flags", + "attrs", + "columns", + "metadata", + "membership", + ], +) +def test_full_receiving_frame_mutation_refuses(tmp_path, monkeypatch, mutation): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + frame = result.frame + if mutation == "age": + frame.person.loc[:, "A_AGE"] += 1 + elif mutation == "unrelated": + frame.person.loc[:, "unrelated_raw_observation"] += 1 + elif mutation == "group": + frame.table("household").loc[:, "state_fips"] += 1 + elif mutation == "weight": + vector = frame.weights_for("household").values + vector.setflags(write=True) + vector[0] += 1 + vector.setflags(write=False) + elif mutation == "strata": + frame.strata.iloc[0] = "other" + elif mutation == "strata_flags": + frame.strata.flags.allows_duplicate_labels = False + elif mutation == "table_flags": + frame.table("household").flags.allows_duplicate_labels = False + elif mutation == "attrs": + frame.person.attrs["changed"] = True + elif mutation == "columns": + frame.person.columns.name = "changed" + elif mutation == "metadata": + frame._metadata = {"changed": True} + elif mutation == "membership": + frame.person.iloc[0, frame.person.columns.get_loc("person_tax_unit_id")] = 999 + with pytest.raises(native.AsecNativePopulationError): + result.validate() + + +@pytest.mark.parametrize( + "which", + [ + "parent_path", + "household_attachment_path", + "person_income_attachment_path", + "household_member_path", + "older_person", + ], +) +def test_changed_original_source_refuses_existing_borrow(tmp_path, monkeypatch, which): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + path = ( + paths["person_member_paths"][2022] if which == "older_person" else paths[which] + ) + with path.open("ab") as handle: + handle.write(b"changed") + with pytest.raises(native.AsecNativePopulationError, match="SOURCE_FILE"): + result.validate() + + +def test_capsule_payload_rehash_and_live_owner_changes_refuse(tmp_path, monkeypatch): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + state = native._ISSUED[id(result)][2] + payload = state.anchors.payload + b" " + object.__setattr__(state.anchors, "payload", payload) + object.__setattr__( + state.anchors, "_issued_sha256", hashlib.sha256(payload).hexdigest() + ) + with pytest.raises(native.AsecNativePopulationError): + result.validate() + original = restoration._implementation + monkeypatch.setattr(restoration, "_implementation", lambda: original()) + with pytest.raises(native.AsecNativePopulationError, match="PRODUCER_CODE_CHANGED"): + native.load_authenticated_asec_2024_native_population(**paths) + + +@pytest.mark.parametrize("legacy", [[100, 200, 17], [0.001, 10_000, 0.0001]]) +def test_original_anchor_ignores_legacy_pooled_weights(tmp_path, monkeypatch, legacy): + paths = _fixture(tmp_path, monkeypatch, legacy_weights=legacy) + result = native.load_authenticated_asec_2024_native_population(**paths) + assert result.frame.weights_for("household").values.tolist() == [2552.12] + assert result.receipt["households"][0]["fraction"] == [63803, 25] + assert ( + native._ISSUED[id(result)][2] + .parent.frame.weights_for("household") + .values.tolist() + == legacy + ) + + +def test_mixed_zero_complete_households_and_selected_zero_refusal( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch, extra_household=True) + result = native.load_authenticated_asec_2024_native_population(**paths) + assert result.frame.person.person_id.tolist() == [105, 106, 107, 108] + assert result.frame.person.A_AGE.tolist() == [55, 14, 55, 14] + assert result.frame.weights_for("household").values.tolist() == [2552.12, 0] + assert [r["fraction"] for r in result.receipt["households"]] == [ + [63803, 25], + [0, 1], + ] + selected = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + assert selected.frame.person.person_id.tolist() == [105, 106] + for entity in selected.frame.schema.group_entities: + assert selected.frame.table(entity)[f"{entity}_id"].tolist() == [3] + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 8),) + ) + + +def test_late_mutation_during_final_file_verification_refuses(tmp_path, monkeypatch): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + frame = result.frame + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + frame.person.loc[:, "A_AGE"] += 1 + return trace + + sys.settrace(trace) + try: + with pytest.raises(native.AsecNativePopulationError, match="FRAME_CHANGED"): + result.validate() + finally: + sys.settrace(None) + assert fired + + +def test_real_loaders_run_without_prepared_population_or_money_execution( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch) + called = set() + + def trace(stack, event, argument): + if event == "call": + module = stack.f_globals.get("__name__", "") + name = stack.f_code.co_name + assert module not in ( + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.engine", + ) + assert name not in ( + "classify_asec_money", + "prepare_asec_current_money_population", + ) + if module.startswith("microcosm.build.us_runtime."): + called.add(name) + + sys.setprofile(trace) + try: + native.load_authenticated_asec_2024_native_population(**paths) + finally: + sys.setprofile(None) + assert { + "load_authenticated_restored_current_money_source", + "authenticate_asec_coverage", + "load_authenticated_asec_household_weights", + "load_authenticated_asec_household_coverage_fields", + } <= called + + +@pytest.mark.parametrize( + "mutation", ["missing", "duplicate", "substitute", "split", "age"] +) +def test_original_person_mutation_refuses_reconstruction( + tmp_path, monkeypatch, mutation +): + paths = _fixture(tmp_path, monkeypatch) + + def change(rows): + if mutation == "missing": + rows.pop() + elif mutation == "duplicate": + rows[2] = rows[1].copy() + elif mutation == "substitute": + rows[1][0] = "9" * 22 + elif mutation == "split": + rows[1][1] = "8" + elif mutation == "age": + rows[1][3] = "54" + + _helpers()._rewrite(paths["person_member_paths"][2024], change) + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population(**paths) + + +def test_external_state_and_rehashed_population_cannot_be_rewritten( + tmp_path, monkeypatch +): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + with pytest.raises(AttributeError): + object.__setattr__(state, "frame_identity", "a" * 64) + changed = json.loads(result.payload) + changed["frame_sha256"] = "a" * 64 + object.__setattr__(result, "payload", native._encode(changed)) + with pytest.raises(native.AsecNativePopulationError, match="UNISSUED_OR_CHANGED"): + result.validate() + + +@pytest.mark.parametrize("owner", ["anchors", "fields", "coverage"]) +def test_equal_unissued_source_evidence_refuses(tmp_path, monkeypatch, owner): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + evidence = getattr(state, owner) + copy = object.__new__(type(evidence)) + # Copy/deserialization cannot inherit the reader's external issuance seal. + for name in evidence.__slots__: + if name != "__weakref__": + object.__setattr__(copy, name, getattr(evidence, name)) + if owner != "coverage": + assert copy == evidence + verifier = { + "anchors": native.anchor_owner.verify_asec_household_weights_source, + "fields": native.field_owner.verify_asec_household_coverage_fields, + "coverage": lambda value: native.coverage_owner.verify_asec_coverage_parent( + value, state.parent + ), + }[owner] + with pytest.raises(ValueError): + verifier(copy) + result.validate() + + +@pytest.mark.parametrize( + "interface", ["validate", "frame", "context", "receipt", "to_bytes"] +) +def test_late_population_payload_mutation_refuses_every_interface( + tmp_path, monkeypatch, interface +): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + object.__setattr__(result, "payload", b'{"release_eligible":true}') + return trace + + sys.settrace(trace) + try: + with pytest.raises(native.AsecNativePopulationError): + value = getattr(result, interface) + if callable(value): + value() + finally: + sys.settrace(None) + assert fired + + +@pytest.mark.parametrize("owner", ["anchors", "fields", "coverage", "parent"]) +def test_late_attached_evidence_mutation_refuses(tmp_path, monkeypatch, owner): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + if owner == "parent": + object.__setattr__( + state.parent.source, "evidence", state.parent.source.evidence + b" " + ) + elif owner == "coverage": + object.__setattr__(state.coverage, "_body", state.coverage._body + b"x") + else: + source = getattr(state, owner) + object.__setattr__(source, "payload", source.payload + b" ") + + return_trace = trace + + def tracing(stack, event, argument): + return_trace(stack, event, argument) + return tracing + + sys.settrace(tracing) + try: + with pytest.raises(native.AsecNativePopulationError): + result.to_bytes() + finally: + sys.settrace(None) + assert fired diff --git a/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py b/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py new file mode 100644 index 000000000..d22eed814 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py @@ -0,0 +1,352 @@ +"""Immutable-leaf reuse of the ASEC records identity plus actual issuer controls. + +Supplied records exercise private value helpers only; they are never treated +as an authenticated source catalogue. The last controls use real issuance over +the maintained invented preparation fixture. +""" + +import sys + +import pytest +from test_us_survey_population_preparation import fixture + +from microcosm.build.us_runtime import asec_population_catalogue as owner +from microcosm.build.us_runtime import survey_population_domains as domains +from microcosm.build.us_runtime import survey_population_preparation as preparation + + +def _key(native_id="1"): + return domains.HouseholdKey(domains.Source.ASEC, 2024, 2025, native_id) + + +def _records(): + key = _key() + person = domains.AsecPerson("2024000000000000000001", "1", "37", "1", "1", key) + household = domains.AsecHousehold(key, "1", "1", "1", "1", "1234567", (person,)) + ledger = owner.UnrepresentedAsecHousehold(_key("2"), "2", "9", "1", "0", "7654321") + return (household,), (ledger,) + + +def _calls(action): + calls = [] + + def trace(frame, event, arg): + if event == "call" and frame.f_code is owner._records_identity.__code__: + calls.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = action() + finally: + sys.setprofile(previous) + return result, len(calls) + + +def _issued(households, ledger): + (identity, memo), calls = _calls( + lambda: owner._records_identity(households, ledger, memo=True) + ) + assert calls == 1 + return identity, memo + + +def _memoized(households, ledger, memo, identity): + return _calls( + lambda: owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + ) + + +def test_same_issued_leaves_reuse_exact_identity_without_another_encode(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + assert identity == owner._records_identity(households, ledger) + assert type(memo) is owner._RecordsMemo + assert memo.households is households and memo.ledger is ledger + assert memo.identity == identity + assert memo.household_leaves == (owner._household_leaves(households[0]),) + assert memo.ledger_leaves == (owner._ledger_leaves(ledger[0]),) + result, calls = _calls( + lambda: [ + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + for _ in range(3) + ] + ) + assert result == [identity] * 3 and calls == 0 + + +def test_equal_replacement_roots_miss_without_changing_identity_or_memo(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + replacement = (tuple(list(households)), tuple(list(ledger))) + assert replacement == (households, ledger) + assert replacement[0] is not households and replacement[1] is not ledger + result, calls = _memoized(*replacement, memo, identity) + assert result == identity and calls == 1 + assert memo.households is households and memo.ledger is ledger + + +@pytest.mark.parametrize( + "target", + ["household", "person", "household_key", "person_key", "ledger", "reason"], +) +def test_in_place_leaf_change_misses_and_changes_identity(target): + households, ledger = _records() + identity, memo = _issued(households, ledger) + household, person = households[0], households[0].persons[0] + if target == "household": + object.__setattr__(household, "hsup_wgt", "0000000") + elif target == "person": + object.__setattr__(person, "age", "38") + elif target == "household_key": + object.__setattr__(household.key, "native_id", "9") + elif target == "person_key": + object.__setattr__(person, "household_key", _key("9")) + elif target == "ledger": + object.__setattr__(ledger[0], "h_numper", "1") + else: + object.__setattr__(ledger[0], "reason", "other") + result, calls = _memoized(households, ledger, memo, identity) + assert calls == 1 and result != identity + assert result == owner._records_identity(households, ledger) + + +@pytest.mark.parametrize("target", ["persons", "key"]) +def test_replaced_container_misses_even_when_equal(target): + households, ledger = _records() + identity, memo = _issued(households, ledger) + household = households[0] + original = getattr(household, target) + replacement = ( + tuple(list(original)) if target == "persons" else _key(original.native_id) + ) + assert replacement == original and replacement is not original + object.__setattr__(household, target, replacement) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 1 + + +def test_persons_replaced_by_a_list_takes_the_refusing_path(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + object.__setattr__(households[0], "persons", list(households[0].persons)) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + + +def test_deleted_leaf_misses_instead_of_raising_inside_the_memo(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + object.__delattr__(households[0].persons[0], "age") + with pytest.raises(AttributeError): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + + +@pytest.mark.parametrize( + "kind", + [ + "str_subclass_literal", + "str_subclass_person", + "str_subclass_reason", + "int_subclass_year", + "str_subclass_native_id", + ], +) +def test_subclass_leaves_yield_no_memo_and_keep_the_full_identity_path(kind): + class StrSubclass(str): + pass + + class IntSubclass(int): + pass + + households, ledger = _records() + household, person = households[0], households[0].persons[0] + if kind == "str_subclass_literal": + object.__setattr__(household, "h_hhtype", StrSubclass("1")) + elif kind == "str_subclass_person": + object.__setattr__(person, "prpertyp", StrSubclass("1")) + elif kind == "str_subclass_reason": + object.__setattr__(ledger[0], "reason", StrSubclass("x")) + elif kind == "int_subclass_year": + object.__setattr__(household.key, "source_year", IntSubclass(2024)) + else: + object.__setattr__(ledger[0].key, "native_id", StrSubclass("2")) + identity, memo = _issued(households, ledger) + assert memo is None + assert identity == owner._records_identity(households, ledger) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 1 + + +def test_memo_requires_agreement_with_the_owner_identity(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + wrong = (*identity[:3], identity[3] + 1) + result, calls = _memoized(households, ledger, memo, wrong) + assert result == identity and calls == 1 + drifted = memo._replace(identity=wrong) + result, calls = _memoized(households, ledger, drifted, identity) + assert result == identity and calls == 1 + plain = tuple(memo) + assert plain == memo + result, calls = _memoized(households, ledger, plain, identity) + assert result == identity and calls == 1 + + +def test_swapped_memo_entry_misses(): + households, ledger = _records() + identity, _ = _issued(households, ledger) + other_households, other_ledger = _records() + object.__setattr__(other_households[0], "hsup_wgt", "0000001") + other_identity, other_memo = _issued(other_households, other_ledger) + assert other_identity != identity + result, calls = _memoized(households, ledger, other_memo, identity) + assert result == identity and calls == 1 + result, calls = _memoized(households, ledger, other_memo, other_identity) + assert result == identity and calls == 1 + + +def test_crafted_memo_over_lists_still_refuses(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + rows = list(households) + crafted = owner._RecordsMemo( + rows, ledger, memo.household_leaves, memo.ledger_leaves, identity + ) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + rows, ledger, crafted, expected_identity=identity + ) + + +def test_wrong_source_member_or_foreign_types_yield_no_memo(): + households, ledger = _records() + foreign = domains.HouseholdKey(domains.Source.ACS, 2024, 2025, "1") + object.__setattr__(ledger[0], "key", foreign) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_SOURCE$"): + owner._records_identity(households, ledger, memo=True) + assert owner._eligible_leaves(ledger[0], "unrepresented") is None + assert owner._eligible_leaves(households[0], "unrepresented") is None + assert owner._eligible_leaves(ledger[0], "household") is None + + +def test_source_member_value_tampering_misses_and_the_full_path_reflects_it(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + member = domains.Source.ASEC + original = member._value_ + object.__setattr__(member, "_value_", "tampered") + try: + assert member.value == "tampered" + result, calls = _memoized(households, ledger, memo, identity) + assert calls == 1 and result != identity + assert result == owner._records_identity(households, ledger) + finally: + object.__setattr__(member, "_value_", original) + assert member.value == original + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 0 + + +@pytest.mark.parametrize("target", ["household", "person", "key", "ledger"]) +def test_class_reassignment_misses_and_the_full_path_refuses(target): + class SwappedHousehold(domains.AsecHousehold): + __slots__ = () + + class SwappedPerson(domains.AsecPerson): + __slots__ = () + + class SwappedKey(domains.HouseholdKey): + __slots__ = () + + class SwappedLedger(owner.UnrepresentedAsecHousehold): + __slots__ = () + + households, ledger = _records() + identity, memo = _issued(households, ledger) + victim, swapped = { + "household": (households[0], SwappedHousehold), + "person": (households[0].persons[0], SwappedPerson), + "key": (households[0].key, SwappedKey), + "ledger": (ledger[0], SwappedLedger), + }[target] + original = type(victim) + object.__setattr__(victim, "__class__", swapped) + try: + assert type(victim) is swapped + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + finally: + object.__setattr__(victim, "__class__", original) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 0 + + +@pytest.fixture(scope="module") +def actual_preparation(tmp_path_factory): + with pytest.MonkeyPatch.context() as monkeypatch: + arguments = fixture(tmp_path_factory.mktemp("asec-records-memo"), monkeypatch) + result = preparation.prepare_authenticated_survey_population(**arguments) + state = preparation._ISSUED[id(result)][2] + catalogue = state.catalogues[1] + catalogue_state = owner._ISSUED[id(catalogue)][2] + assert type(catalogue_state.records_memo) is owner._RecordsMemo + assert catalogue_state.records_memo.identity == catalogue_state.records_identity + yield result, catalogue, catalogue_state + result.validate() + catalogue.validate() + + +def test_actual_borrows_reuse_the_issued_identity(actual_preparation): + result, catalogue, _ = actual_preparation + _, calls = _calls(catalogue.validate) + assert calls == 0 + _, calls = _calls(result.checked_view) + assert calls == 0 + + +def test_actual_household_leaf_change_refuses_every_borrow(actual_preparation): + result, catalogue, catalogue_state = actual_preparation + row = catalogue_state.households[0] + original = row.hsup_wgt + object.__setattr__(row, "hsup_wgt", original + "0") + try: + with pytest.raises(owner.AsecSourceCatalogueError, match="RECORDS_CHANGED"): + catalogue.validate() + with pytest.raises( + preparation.SurveyPopulationPreparationError, + match="NESTED_EVIDENCE_CHANGED", + ): + result.checked_view() + finally: + object.__setattr__(row, "hsup_wgt", original) + _, calls = _calls(catalogue.validate) + assert calls == 0 + + +def test_actual_person_key_change_refuses_every_borrow(actual_preparation): + result, catalogue, catalogue_state = actual_preparation + person = catalogue_state.households[0].persons[0] + original = person.household_key + object.__setattr__(person, "household_key", _key(original.native_id + "0")) + try: + with pytest.raises(owner.AsecSourceCatalogueError, match="RECORDS_CHANGED"): + catalogue.validate() + with pytest.raises( + preparation.SurveyPopulationPreparationError, + match="NESTED_EVIDENCE_CHANGED", + ): + result.checked_view() + finally: + object.__setattr__(person, "household_key", original) + result.validate() diff --git a/packages/microcosm-build/tests/test_us_asec_checkpoint.py b/packages/microcosm-build/tests/test_us_asec_checkpoint.py index aa2a1338a..3496eb470 100644 --- a/packages/microcosm-build/tests/test_us_asec_checkpoint.py +++ b/packages/microcosm-build/tests/test_us_asec_checkpoint.py @@ -7,6 +7,7 @@ import pandas as pd import pytest +import microcosm.build.us_runtime.asec_checkpoint as checkpoint_module from microcosm.build.frame_checkpoint import write_frame_checkpoint from microcosm.build.outer_stage_runtime import ( OUTER_STAGE_CONTEXT_SCHEMA_VERSION, @@ -26,6 +27,396 @@ ) from microcosm.frame import US_SCHEMA, EntitySchema, Frame, WeightKind, Weights + +def _v4_frame() -> Frame: + frame = _raw_us_frame() + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + frame.table("person")[column] = np.asarray([1, 2], dtype=np.int64) + return frame + + +def _v4_binding(frame: Frame) -> dict: + metadata = _raw_binding(frame) + metadata["schema_version"] = 4 + metadata["source_receipt"]["sources"].append( + dict(metadata["source_receipt"]["sources"][0], year=2023) + ) + pins = checkpoint_module.ASEC_EDUCATION_ASSISTANCE_ARCHIVES + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + metadata["raw_source_mappings"][column] = { + "column": column, + "entity": "person", + "operation": "exact_source_join", + "join_keys": ["source_year", "PERIDNUM"], + "source_pins": [ + { + "income_year": year, + "locator": pins[year].zip_url, + "member": pins[year].member, + "sha256": pins[year].zip_sha256, + "member_sha256": pins[year].member_sha256, + } + for year in (2022, 2023) + ], + # Audit describes the full official member, not this two-row subset. + "audit": { + str(year): { + "rows": pins[year].rows, + "yes_rows": 1, + "no_rows": pins[year].rows - 1, + "weighted_yes_share": 0.25, + } + for year in (2022, 2023) + }, + } + return metadata + + +def test_v4_roundtrip_requires_explicit_codec(tmp_path: Path) -> None: + frame = _v4_frame() + path = tmp_path / "v4.h5" + _write_checkpoint(path, frame, metadata=_v4_binding(frame)) + loaded, binding = checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + assert binding["schema_version"] == 4 + assert loaded.table("person")["NOW_MCAID"].tolist() == [1, 2] + with pytest.raises(ValueError, match="unsupported raw-stage schema version"): + load_asec_raw_stage_checkpoint(path) + path3 = tmp_path / "v3.h5" + legacy = _raw_us_frame() + _write_checkpoint(path3, legacy, metadata=_raw_binding(legacy)) + with pytest.raises(ValueError, match="unsupported raw-stage schema version"): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path3) + + +def test_restored_raw_observations_carry_to_input_leaves_without_engine() -> None: + from microcosm.build.us_runtime.cps_carried import derive_us_cps_carried_inputs + + frame = _raw_us_frame() + person = frame.table("person") + person["NOW_GRP"] = [1, 2] + person["NOW_MRK"] = [2, 1] + person["OI_VAL"] = [0.0, 0.0] + person["OI_OFF"] = [0, 0] + sidecar = person[["source_year", "PERIDNUM"]].copy() + sidecar["PH_SEQ"] = [101, 102] + sidecar["P_SEQ"] = [1, 1] + sidecar["A_LINENO"] = [1, 1] + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + sidecar[column] = [1, 2] + result = derive_us_cps_carried_inputs(frame, reported_coverage_source=sidecar) + assert "NOW_MCAID" not in frame.table("person") + assert result.table("person")[ + "has_medicaid_health_coverage_at_interview" + ].tolist() == [True, False] + assert result.table("person")["has_esi"].tolist() == [True, False] + assert result.table("person")[ + "has_marketplace_health_coverage_at_interview" + ].tolist() == [False, True] + assert result.weights_for("household").kind is WeightKind.DESIGN + pd.testing.assert_frame_equal( + result.table("person"), derive_us_cps_carried_inputs(result).table("person") + ) + + +@pytest.mark.parametrize( + "with_mass_history,forward_metadata", + [(False, False), (True, False), (True, True)], + ids=["empty_log", "declared_mass_change", "metadata_forwarding"], +) +def test_v4_restoration_producer_authenticates_source_and_writes_new_bundle( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + with_mass_history: bool, + forward_metadata: bool, +) -> None: + import hashlib + from dataclasses import replace + + from microcosm.build.us_runtime import asec_raw_stage_v4 as restoration + from microcosm.build.us_runtime import reported_coverage_source + from microcosm.build.us_runtime.asec_raw_stage_v4 import restore_asec_raw_stage_v4 + from microcosm.frame import MassChange + + legacy = _raw_us_frame() + if with_mass_history: + original = legacy + weights = original.weights_for("household") + legacy = original.with_weights( + "household", + Weights(weights.values * 2.0, weights.kind), + mass=MassChange( + factor=2.0, reason="Invented raw-source design-weight rescaling" + ), + ) + assert original.mass_log == () + assert len(legacy.mass_log) == 1 + record = legacy.mass_log[0] + assert record.entity == "household" + assert record.old_total == float(weights.values.sum()) + assert record.new_total == 2.0 * record.old_total + assert record.declared_factor == 2.0 + assert record.reason == "Invented raw-source design-weight rescaling" + expected_mass_log = legacy.mass_log + assert bool(expected_mass_log) is with_mass_history + binding = _raw_binding(legacy) + binding["source_receipt"]["sources"].append( + dict(binding["source_receipt"]["sources"][0], year=2023) + ) + input_path = tmp_path / "v3.h5" + _write_checkpoint(input_path, legacy, metadata=binding) + loaded_input, _ = checkpoint_module.load_asec_raw_stage_checkpoint(input_path) + assert loaded_input.mass_log == expected_mass_log + np.testing.assert_array_equal( + loaded_input.weights_for("household").values, + legacy.weights_for("household").values, + ) + input_sha = hashlib.sha256(input_path.read_bytes()).hexdigest() + pins = {} + paths = {} + for year, key in ( + (2022, "0000000000000000000001"), + (2023, "0000000000000000000002"), + ): + path = tmp_path / f"person-{year}.csv" + row = { + "PERIDNUM": key, + "PH_SEQ": 1, + "P_SEQ": 1, + "A_LINENO": 1, + "A_FNLWGT": 100.0, + } + row.update( + { + column: 1 if year == 2022 else 2 + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS + } + ) + pd.DataFrame([row]).to_csv(path, index=False) + pins[year] = replace( + checkpoint_module.ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year], + rows=1, + member=path.name, + member_size_bytes=path.stat().st_size, + member_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), + ) + paths[year] = path + monkeypatch.setattr(checkpoint_module, "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", pins) + monkeypatch.setattr( + reported_coverage_source, "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", pins + ) + forwarded = [] + if forward_metadata: + # The checkpoint codec does not persist Frame.metadata. Supply a + # descriptive value only after the real v3 loader has validated its + # input, then observe the actual converter at the real writer boundary. + supplied_metadata = {"forwarding_probe": {"purpose": "invented observation"}} + real_load = checkpoint_module.load_asec_raw_stage_checkpoint + real_write = restoration.write_frame_checkpoint + + def load_with_descriptive_metadata(path): + loaded, binding = real_load(path) + assert not loaded.metadata + return ( + Frame( + {entity: loaded.table(entity) for entity in loaded.entities}, + loaded.schema, + { + entity: loaded.weights_for(entity) + for entity in loaded.weighted_entities + }, + loaded.strata, + mass_log=loaded.mass_log, + metadata=supplied_metadata, + ), + binding, + ) + + def observe_real_write(path, frame, **kwargs): + assert set(frame.metadata) == {"forwarding_probe"} + assert ( + dict(frame.metadata["forwarding_probe"]) + == supplied_metadata["forwarding_probe"] + ) + assert frame.mass_log == expected_mass_log + forwarded.append(True) + return real_write(path, frame, **kwargs) + + monkeypatch.setattr( + checkpoint_module, + "load_asec_raw_stage_checkpoint", + load_with_descriptive_metadata, + ) + monkeypatch.setattr(restoration, "write_frame_checkpoint", observe_real_write) + output_dir = tmp_path / "restored" + receipt = restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=output_dir, + ) + assert forwarded == ([True] if forward_metadata else []) + assert receipt["input_sha256"] == input_sha + output = output_dir / "asec_raw_stage.checkpoint.h5" + assert receipt["output_sha256"] == hashlib.sha256(output.read_bytes()).hexdigest() + assert hashlib.sha256(input_path.read_bytes()).hexdigest() == input_sha + restored, metadata = checkpoint_module.load_asec_raw_stage_checkpoint_v4(output) + assert restored.mass_log == expected_mass_log + # The codec does not reconstruct the separately supplied Frame metadata. + assert not restored.metadata + assert restored.weights_for("household").kind is WeightKind.DESIGN + np.testing.assert_array_equal( + restored.weights_for("household").values, + loaded_input.weights_for("household").values, + ) + assert restored.table("person")["NOW_MCAID"].tolist() == [1, 2] + assert metadata["source_receipt"] == binding["source_receipt"] + assert json.loads((output_dir / "restoration.receipt.json").read_text()) == receipt + with pytest.raises(FileExistsError): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=output_dir, + ) + with pytest.raises(ValueError, match="SHA-256"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256="f" * 64, + coverage_paths=paths, + output_dir=tmp_path / "bad", + ) + with pytest.raises(ValueError, match="local coverage paths"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths={2022: paths[2022]}, + output_dir=tmp_path / "missing", + ) + paths[2022].write_text("tampered") + with pytest.raises(ValueError, match="byte length mismatch"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=tmp_path / "tampered", + ) + assert not (tmp_path / "bad").exists() + assert not (tmp_path / "missing").exists() + assert not (tmp_path / "tampered").exists() + + +@pytest.mark.parametrize( + "mutation", + ( + "pin_sha", + "pin_member", + "pin_year_missing", + "pin_year_duplicate", + "audit_year", + "audit_rows", + "audit_counts", + "audit_nan", + "audit_bool", + "receipt_year", + "missing_mapping", + "float_version", + ), +) +def test_v4_refuses_inconsistent_attestation(tmp_path: Path, mutation: str) -> None: + frame = _v4_frame() + metadata = _v4_binding(frame) + mapping = metadata["raw_source_mappings"]["NOW_MCAID"] + if mutation == "pin_sha": + mapping["source_pins"][0]["sha256"] = "f" * 64 + elif mutation == "pin_member": + mapping["source_pins"][0]["member"] = "other.csv" + elif mutation == "pin_year_missing": + mapping["source_pins"].pop() + elif mutation == "pin_year_duplicate": + mapping["source_pins"].append(dict(mapping["source_pins"][0])) + elif mutation == "audit_year": + del mapping["audit"]["2023"] + elif mutation == "audit_rows": + mapping["audit"]["2022"]["rows"] = 2 + elif mutation == "audit_counts": + mapping["audit"]["2022"]["no_rows"] = 0 + elif mutation == "audit_nan": + mapping["audit"]["2022"]["weighted_yes_share"] = float("nan") + elif mutation == "audit_bool": + mapping["audit"]["2022"]["yes_rows"] = True + elif mutation == "receipt_year": + metadata["source_receipt"]["sources"].pop() + elif mutation == "missing_mapping": + del metadata["raw_source_mappings"]["NOW_MCAID"] + elif mutation == "float_version": + metadata["schema_version"] = 4.0 + path = tmp_path / "invalid.h5" + # Exercise the same metadata validators directly for NaN because canonical + # checkpoint serialization correctly refuses it before the loader does. + if mutation == "audit_nan": + with pytest.raises(ValueError, match="audit counts/share"): + checkpoint_module._validate_coverage_v4_binding(frame, metadata, path=path) + else: + _write_checkpoint(path, frame, metadata=metadata) + with pytest.raises(ValueError): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +@pytest.mark.parametrize("value", (0, 3, 1.5, float("inf"), float("nan"), True)) +def test_v4_refuses_bad_measured_coverage(tmp_path: Path, value: object) -> None: + frame = _v4_frame() + frame.table("person")["NOW_MCAID"] = [value, value] + path = tmp_path / "bad-code.h5" + _write_checkpoint(path, frame, metadata=_v4_binding(frame)) + with pytest.raises(ValueError, match="NOW_MCAID must be complete integer recodes"): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +@pytest.mark.parametrize( + "mutation", ("missing_column", "peridnum", "duplicate", "weights") +) +def test_v4_refuses_wrong_source_boundary(tmp_path: Path, mutation: str) -> None: + frame = _v4_frame() + person = frame.table("person") + if mutation == "missing_column": + person.drop(columns="NOW_IHSFLG", inplace=True) + elif mutation == "peridnum": + person["PERIDNUM"] = ["1", "2"] + elif mutation == "duplicate": + person["source_year"] = [2022, 2022] + person["PERIDNUM"] = ["0" * 22, "0" * 22] + elif mutation == "weights": + frame = Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + {"household": Weights(np.asarray([2.0, 3.0]), WeightKind.CALIBRATED)}, + frame.strata, + ) + metadata = _v4_binding(frame) + if mutation == "duplicate": + metadata["source_receipt"]["sources"].pop() + path = tmp_path / "boundary.h5" + _write_checkpoint(path, frame, metadata=metadata) + with pytest.raises(ValueError): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +def test_v4_duplicate_guard_compares_normalized_identity() -> None: + frame = _v4_frame() + person = frame.table("person") + person["source_year"] = pd.Series([2022, "2022"], dtype=object) + person["PERIDNUM"] = pd.Series(["0" * 22, b"0" * 22], dtype=object) + metadata = _v4_binding(frame) + metadata["source_receipt"]["sources"].pop() + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + metadata["raw_source_mappings"][column]["source_pins"].pop() + del metadata["raw_source_mappings"][column]["audit"]["2023"] + with pytest.raises(ValueError, match="repeats a source-year/PERIDNUM"): + checkpoint_module._validate_coverage_v4_binding( + frame, metadata, path=Path("fixture.h5") + ) + + _OUTER_STAGE_ARTIFACT_KIND = "populace_outer_stage_frame" diff --git a/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py b/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py new file mode 100644 index 000000000..faca38aae --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py @@ -0,0 +1,948 @@ +"""Invented closed source-parent fixtures only; no genuine preparation.""" + +import _csv +import csv +import hashlib +import importlib.util +import json +import os +import struct +import subprocess +import sys +import textwrap +import traceback +from dataclasses import FrozenInstanceError, replace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.frame_checkpoint import ( + load_frame_checkpoint, + write_frame_checkpoint, +) +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_current_money_source as money +from microcosm.build.us_runtime import asec_person_coverage_source as literal + + +def _helper(name): + spec = importlib.util.spec_from_file_location( + "coverage_invented_" + name, Path(__file__).with_name(name + ".py") + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pin(paths, monkeypatch): + pins = [] + for year, member, archive, _, _, _ in coverage._MEMBER_PINS: + value = paths[year].read_bytes() + with paths[year].open(newline="") as handle: + rows = list(csv.reader(handle)) + pins.append( + (year, member, archive, coverage._sha(value), len(rows) - 1, len(value)) + ) + monkeypatch.setattr(coverage, "_MEMBER_PINS", tuple(pins)) + + +def _rewrite(path, change): + with path.open(newline="") as handle: + rows = list(csv.reader(handle)) + change(rows) + with path.open("w", newline="") as handle: + csv.writer(handle).writerows(rows) + + +def _changed_parent(parent, attachment, monkeypatch, changes): + for path in (parent, attachment): + loaded = load_frame_checkpoint(path) + for column, values in changes.items(): + loaded.frame.person[column] = values + identity = frame_identity(loaded.frame) + for key in ("identity", "source_construction_identity"): + if key in loaded.metadata: + loaded.metadata[key] = identity.to_payload() + if path == parent: + parent_identity = identity + else: + digest = coverage._sha(parent.read_bytes()) + loaded.metadata["parent_checkpoint_sha256"] = digest + loaded.metadata["household_observations"]["input_checkpoint_sha256"] = ( + digest + ) + loaded.metadata["household_observations"][ + "input_structural_identity_sha256" + ] = parent_identity.sha256 + write_frame_checkpoint(path, loaded.frame, metadata=loaded.metadata) + monkeypatch.setattr( + money, + "_SOURCE_PINS", + ( + coverage._sha(parent.read_bytes()), + coverage._sha(attachment.read_bytes()), + money._SOURCE_PINS[2], + ), + ) + + +def _fixtures( + tmp_path, + monkeypatch, + tokens=("1", "2", "3", "", "-4", "NA"), + *, + native_ids=None, + person_changes=None, + raw_ages=None, +): + parent, attachment, paths = _helper( + "test_us_asec_person_income_source" + ).invented_sources(tmp_path, monkeypatch, missing=False) + if native_ids is not None: + _helper("test_us_asec_demographic_source")._repoint_published_households( + parent, attachment, monkeypatch, native_ids + ) + if person_changes: + _changed_parent(parent, attachment, monkeypatch, person_changes) + source = money.load_authenticated_current_money_source(parent, attachment) + for path in paths.values(): + + def change(rows): + rows[0].append("PRPERTYP") + for row in rows[1:]: + position = int(row[0]) - 1 + row.append(tokens[position % len(tokens)]) + if native_ids is not None: + row[1] = str(native_ids[position]) + if raw_ages is not None: + row[3] = str(raw_ages[position]) + if person_changes and "PERIDNUM" in person_changes: + row[0] = person_changes["PERIDNUM"][position] + + _rewrite(path, change) + _pin(paths, monkeypatch) + return source, paths + + +def _read(source, paths, **kwargs): + return coverage.authenticate_asec_coverage(source, member_paths=paths, **kwargs) + + +def test_closed_complete_source_exact_parent_and_defensive_views(tmp_path, monkeypatch): + source, paths = _fixtures(tmp_path, monkeypatch) + before = money._frame_signature(source.frame) + tables = {e: source.frame.table(e).copy(deep=True) for e in source.frame.entities} + result = _read(source, paths) + coverage.verify_asec_coverage_parent(result, source) + view = result.table() + assert view.PRPERTYP.tolist() == ["1", "2", "3", "", "-4", "NA"] + assert view.PRPERTYP_state.tolist() == ["observed_code"] * 3 + [ + "blank_unresolved", + "unlabelled_in_range", + "malformed_token", + ] + assert view.PERIDNUM.tolist() == list(source.scope.person_native_keys) + assert view.person_household_id.tolist() == [1, 1, 2, 2, 3, 3] + raw, original_receipt = literal.read_asec_person_coverage_source( + paths, person_roster=source.frame.person + ) + receipt = result.receipt + assert receipt["literal_reader_receipt"] == original_receipt + # Native key cells retain their exact strings; the artifact view declares + # one non-nullable vocabulary independently of the parent's string storage. + pd.testing.assert_frame_equal(view[list(raw)], raw.astype({"PERIDNUM": "string"})) + assert receipt["source_authenticated"] is True + assert receipt["named_parent_binding_authenticated"] is True + for name in ( + "release_eligible", + "domain_authority", + "period_harmonized", + "cross_survey_coverage_equivalence_established", + "original_design_weight_semantics_established", + "archive_bytes_read", + ): + assert receipt[name] is False + assert receipt["literal_reader_receipt"]["source_authenticated"] is False + assert ( + receipt["literal_reader_receipt"]["population_binding_authenticated"] is False + ) + assert receipt["age_relation"]["compared_rows"] == 6 + assert ( + receipt["age_relation"]["relation"] == "original_csv_A_AGE_equals_parent_A_AGE" + ) + assert receipt["household_partitions"]["households"] == 3 + assert [p["rows"] for p in receipt["sources"]] == [2, 2, 2] + assert str(tmp_path) not in json.dumps(receipt) + assert "0000000000000000000001" not in json.dumps(receipt) + pin = result.content_sha256 + view.loc[0, "PRPERTYP"] = "changed" + view.index = pd.Index([99, 1, 2, 3, 4, 5]) + receipt["parent"]["frame_sha256"] = "changed" + assert result.content_sha256 == pin and result.table().PRPERTYP.iloc[0] == "1" + assert money._frame_signature(source.frame) == before + for entity, expected in tables.items(): + pd.testing.assert_frame_equal( + source.frame.table(entity), expected, check_exact=True + ) + with pytest.raises(FrozenInstanceError): + result._body = b"changed" + with pytest.raises(coverage.AsecCoverageAuthenticationError, match="CONSTRUCTOR"): + coverage.AuthenticatedAsecCoverage(result._header, result._body) + with pytest.raises(coverage.AsecCoverageAuthenticationError, match="CONSTRUCTOR"): + replace(result) + + +@pytest.mark.parametrize( + "token,state", + [ + ("", "blank_unresolved"), + ("\0", "malformed_token"), + ("1\0x", "malformed_token"), + (" 1", "malformed_token"), + ("1.0", "malformed_token"), + ("NA", "malformed_token"), + ('a"b', "malformed_token"), + ("a,b\r\n\t😀", "malformed_token"), + ("-4", "unlabelled_in_range"), + ("4", "out_of_range"), + ("3", "observed_code"), + ], +) +def test_literal_utf8_roundtrip_without_coercion(tmp_path, monkeypatch, token, state): + source, paths = _fixtures(tmp_path, monkeypatch, (token,)) + result = _read(source, paths) + assert result.table().PRPERTYP.tolist() == [token] * 6 + assert result.table().PRPERTYP_state.tolist() == [state] * 6 + assert not result.table().PRPERTYP.isna().any() + + +@pytest.mark.parametrize("kind", ["frame", "receipt", "hash"]) +def test_no_plain_parent_can_issue_authority(tmp_path, monkeypatch, kind): + source, paths = _fixtures(tmp_path, monkeypatch) + candidate = { + "frame": source.frame, + "receipt": json.loads(source.source.identity), + "hash": money._frame_signature(source.frame), + }[kind] + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="AUTHENTICATED_PARENT" + ): + _read(candidate, paths) + with pytest.raises(TypeError): + coverage.authenticate_asec_coverage(source, member_paths=paths, member_pins=()) + + +@pytest.mark.parametrize( + "column,value", + [ + ("A_AGE", "82"), + ("A_AGE", ""), + ("A_AGE", "55.0"), + ("A_AGE", "\0"), + ("PH_SEQ", "8"), + ("A_LINENO", "8"), + ("PERIDNUM", "9" * 22), + ], +) +def test_authenticated_raw_coordinate_conflicts_refuse( + tmp_path, monkeypatch, column, value +): + source, paths = _fixtures(tmp_path, monkeypatch) + _rewrite( + paths[2022], lambda rows: rows[1].__setitem__(rows[0].index(column), value) + ) + _pin(paths, monkeypatch) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="SOURCE_REFUSAL" + ): + _read(source, paths) + + +@pytest.mark.parametrize( + "column", + [ + "A_AGE", + "PERIDNUM", + "person_household_id", + "unrelated_raw_observation", + "WSAL_VAL", + ], +) +def test_same_ids_mutated_parent_cannot_reuse_binding(tmp_path, monkeypatch, column): + source, paths = _fixtures(tmp_path, monkeypatch) + result = _read(source, paths) + source.frame.person.loc[:, column] = "9" * 22 if column == "PERIDNUM" else 1 + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="PARENT_REFUSAL" + ): + coverage.verify_asec_coverage_parent(result, source) + + +def test_distinct_authenticated_parent_with_same_ids_cannot_reuse_artifact( + tmp_path, monkeypatch +): + first = tmp_path / "first" + second = tmp_path / "second" + first.mkdir() + second.mkdir() + source, paths = _fixtures(first, monkeypatch) + result = _read(source, paths) + other, other_paths = _fixtures( + second, + monkeypatch, + person_changes={"unrelated_raw_observation": np.arange(6, dtype=np.int64) + 1}, + ) + assert other.scope.person_ids == source.scope.person_ids + # Restore the producer's fixture pins so this specifically checks parent + # identity rather than the independent source-pin implementation guard. + _pin(paths, monkeypatch) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="PARENT_CHANGED" + ): + coverage.verify_asec_coverage_parent(result, other) + assert set(other_paths) == {2022, 2023, 2024} + + +def test_merged_native_households_refuse_even_with_matching_per_row_coordinates( + tmp_path, monkeypatch +): + # The current-money parent already refuses this shape, before coverage + # issuance. Keep that layering and also test the wrapper's own fence. + with pytest.raises(ValueError, match="SOURCE_CONTRACT_REFUSAL"): + _fixtures(tmp_path, monkeypatch, native_ids=[7, 8, 7, 7, 7, 7]) + person = pd.DataFrame( + { + "source_year": [2022, 2022], + "source_household_id": [7, 8], + "person_household_id": [1, 1], + }, + dtype="int64", + ) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_MERGE" + ): + coverage._partitions(person) + + +@pytest.mark.parametrize("parent_age,raw_age", [(80, 80), (82, 80), (0, "")]) +def test_raw_age_identity_never_substitutes_model_age( + tmp_path, monkeypatch, parent_age, raw_age +): + source, paths = _fixtures( + tmp_path, + monkeypatch, + person_changes={ + "A_AGE": np.array([parent_age, 14] * 3, dtype=np.int64), + }, + raw_ages=[raw_age, 14] * 3, + ) + before = money._frame_signature(source.frame) + if parent_age == raw_age: + result = _read(source, paths) + assert result.table().A_AGE.tolist() == [80, 14] * 3 + assert result.receipt["age_relation"]["model_age_used"] is False + else: + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="SOURCE_REFUSAL" + ): + _read(source, paths) + assert money._frame_signature(source.frame) == before + + +def test_parent_authority_refuses_model_age_before_source_issuance( + tmp_path, monkeypatch +): + # Canonical model age is already forbidden by the raw parent boundary. + # Keep that refusal; do not loosen the parent merely to exercise coverage. + with pytest.raises(ValueError, match="SOURCE_CONTRACT_REFUSAL"): + _fixtures( + tmp_path, + monkeypatch, + person_changes={ + "A_AGE": np.array([80, 14] * 3, dtype=np.int64), + "age": [82.0, 14.0] * 3, + }, + raw_ages=[80, 14] * 3, + ) + + +def test_same_native_person_keys_across_cohorts_remain_distinct(tmp_path, monkeypatch): + keys = [str(i).zfill(22) for i in (1, 2)] * 3 + source, paths = _fixtures(tmp_path, monkeypatch, person_changes={"PERIDNUM": keys}) + result = _read(source, paths) + assert result.table().PERIDNUM.tolist() == keys + assert result.table().PRPERTYP.tolist() == ["1", "2", "3", "", "-4", "NA"] + + +def test_partition_bijection_refuses_splits_and_cross_cohort_merges(): + person = pd.DataFrame( + { + "source_year": [2022, 2022], + "source_household_id": [7, 7], + "person_household_id": [1, 2], + }, + dtype="int64", + ) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_SPLIT" + ): + coverage._partitions(person) + person.source_year = [2022, 2023] + person.person_household_id = [1, 1] + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_MERGE" + ): + coverage._partitions(person) + + +@pytest.mark.parametrize( + "kind", + [ + "missing_cohort", + "duplicate_cohort", + "duplicate_row", + "missing_row", + "swapped_cohort", + ], +) +def test_complete_cohort_authority_refuses_incomplete_or_duplicate_inputs( + tmp_path, monkeypatch, kind +): + source, paths = _fixtures(tmp_path, monkeypatch) + if kind == "missing_cohort": + paths.pop(2023) + elif kind == "duplicate_cohort": + pins = coverage._MEMBER_PINS + monkeypatch.setattr(coverage, "_MEMBER_PINS", (pins[0], pins[0], pins[2])) + elif kind == "swapped_cohort": + paths[2022], paths[2023] = paths[2023], paths[2022] + else: + _rewrite( + paths[2022], + lambda rows: ( + rows.__setitem__(1, rows[2].copy()) + if kind == "duplicate_row" + else rows.pop() + ), + ) + _pin(paths, monkeypatch) + with pytest.raises(coverage.AsecCoverageAuthenticationError): + _read(source, paths) + + +def test_candidate_self_rehash_is_not_authority_and_reconstruction_runs_first( + tmp_path, monkeypatch +): + source, paths = _fixtures(tmp_path, monkeypatch) + result = _read(source, paths) + candidate = tmp_path / "candidate.bin" + candidate.write_bytes(result.to_bytes()) + assert ( + _read(source, paths, candidate_path=candidate).to_bytes() == result.to_bytes() + ) + body = result._body.replace(b"observed_code", b"invented_code", 1) + header = json.loads(result._header) + header["body_sha256"] = coverage._sha(body) + header = coverage._json(header) + payload = coverage.MAGIC + struct.pack(" UsPumaLadder: tract_overlap_puma=puma.copy(), tract_overlap_tract=np.asarray([1_001_000_100, 1_003_000_100], dtype=np.int64), tract_overlap_population=population.copy(), + joint_overlap_puma=puma.copy(), + joint_overlap_tract=np.asarray([1_001_000_100, 1_003_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([101, 102], dtype=np.int64), + joint_overlap_population=np.asarray([40, 60], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py index 02597f63c..90bcd0972 100644 --- a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import hashlib import importlib.util import json from collections import Counter @@ -26,6 +27,7 @@ SpecResolutionError, resolve_cross_references, ) +from microcosm.build.spec_engine.yaml12 import load_yaml12_file Mutation = Callable[[dict[str, Any]], None] ROOT = Path(__file__).resolve().parents[3] @@ -108,6 +110,109 @@ def _walk_values(value: object, path: str = "") -> list[tuple[str, object]]: return rows +def test_source_only_generator_reproduces_packaged_source_authority() -> None: + """Source JSON/YAML drift must fail even without the optional country engine.""" + from tools import generate_us_bundle_from_constants as generator + + documents = generator.build_documents(kinds=("sources",)) + assert set(documents) == {"sources.yaml"} + generated = documents["sources.yaml"] + path = generator.DEFAULT_OUTPUT_DIR / "sources.yaml" + authored = load_yaml12_file(path) + raw = (generator.US_PACKAGE_ROOT / "source_stages.json").read_bytes() + assert {**authored["stage_manifest"], "stages": authored["stages"]} == ( + json.loads(raw) + ) + assert authored == generated + assert path.read_bytes() == generator.render_yaml("sources.yaml", generated) + assert generated["stage_asset"]["sha256"] == hashlib.sha256(raw).hexdigest() + assert ( + generated["stage_asset"]["sha256"] + == (generator.FROZEN_LEGACY_RESOURCE_SHA256["source_stages.json"]) + ) + + +def _immigration_operation(sources: dict[str, Any]) -> dict[str, Any]: + return next( + operation + for stage in sources["stages"] + for operation in stage["operations"] + if operation["kind"] == "derive_immigration_status" + ) + + +def test_humanitarian_stock_values_are_normative_and_citations_operational() -> None: + sources = build_sources() + before, _, surfaces = _source_surface_hashes(sources) + normative = _immigration_operation(surfaces["normative"]) + operational = next( + operation + for stage in surfaces["operational"]["stages"] + for operation in stage.get("operations", []) + if "humanitarian_status_stocks" in operation + ) + stocks = _immigration_operation(sources)["humanitarian_status_stocks"] + assert normative["humanitarian_status_stocks"]["refugee"] == { + "target": stocks["refugee"]["target"] + } + assert operational["humanitarian_status_stocks"]["refugee"] == { + "source": stocks["refugee"]["source"] + } + stocks["refugee"]["source"] += "#citation-only-change" + assert _source_surface_hashes(sources)[0] == before + stocks["refugee"]["target"] += 1 + assert _source_surface_hashes(sources)[0] != before + + +def test_humanitarian_stock_schema_matches_runtime_category_and_origin_rosters() -> ( + None +): + from microcosm.build.us_runtime import immigration + + schema = load_schema_registry().schema("sources.schema.json") + operation = next( + value + for value in schema["$defs"]["source_parameter_record"]["oneOf"] + if value["properties"]["kind"].get("const") == "derive_immigration_status" + ) + stocks = operation["properties"]["humanitarian_status_stocks"] + assert ( + set(stocks["properties"]) + == set(stocks["required"]) + == set(immigration.HUMANITARIAN_STATUS_CATEGORIES) + ) + for category, origins in immigration._PER_ORIGIN_HUMANITARIAN_CATEGORIES.items(): + block = stocks["properties"][category] + assert set(block["properties"]) == set(block["required"]) == set(origins) + + +@pytest.mark.parametrize( + "mutation", + [ + lambda stocks: stocks.pop("refugee"), + lambda stocks: stocks.update({"unknown_status": {"target": 0, "source": "x"}}), + lambda stocks: stocks["tps"].pop("nepal"), + lambda stocks: stocks["tps"].update( + {"unknown_origin": {"target": 0, "source": "x"}} + ), + lambda stocks: stocks["refugee"].update({"target": -1}), + lambda stocks: stocks["refugee"].update({"target": True}), + lambda stocks: stocks["refugee"].update({"source": ""}), + lambda stocks: stocks["refugee"].update({"source": 123}), + lambda stocks: stocks["refugee"].pop("target"), + lambda stocks: stocks["refugee"].pop("source"), + lambda stocks: stocks["refugee"].update({"unreviewed": True}), + ], +) +def test_humanitarian_stock_schema_refuses_malformed_controls( + mutation: Mutation, +) -> None: + sources = build_sources() + mutation(_immigration_operation(sources)["humanitarian_status_stocks"]) + with pytest.raises(SpecValidationError): + load_schema_registry().validate(sources, "sources.schema.json") + + def test_source_surface_classification_is_complete() -> None: sources = build_sources() _, _, surfaces = _source_surface_hashes(sources) @@ -129,7 +234,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "7935d891ed16eac618f04813a7b8a3ceb9a971c8b4c4273683594ba61e7dd631", + "sha256": "b3ad16b1adb77a2cac6deae6c4d0f1779e4cf36a3f18820d9887a5434daaff1a", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/test_us_child_property_income_graph.py b/packages/microcosm-build/tests/test_us_child_property_income_graph.py new file mode 100644 index 000000000..06d5cf74f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_child_property_income_graph.py @@ -0,0 +1,777 @@ +"""Invented country graph values: these pure projections issue no authority.""" + +import json +import sys +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_child_property_income as graph +from microcosm.fit import joint_empirical as empirical +from microcosm.frame import EntitySchema, Frame, MassChangeRecord, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + ArtifactValue, + Graph, + KernelContext, + Node, + Numeric, + NumericScope, + Owned, + SourceRef, + StructuralDelta, + WeightTransition, + compile_graph, +) + +ORDER_TYPE = ArtifactType("test.child_property_ordering", 1) +ORDER_BYTES = b"invented checked receiving parent" + + +def _options(*, zero=False, only_positive=False): + one = empirical.SupportThreshold(1, 1, 1.0, 1.0) + return graph.ChildPropertyOptions( + "invented-zero-stress" if zero else "invented-reference", + "test", + empirical.SupportRequirements( + "invented-minimum-one", + "test", + one, + one, + one, + (3,) if only_positive else (0, 1, 2, 3), + ), + empirical.JointTransport(0.0 if zero else 1.0, (1.0, 1.0)), + ("sha256-u53-v1", "invented-child-law", 0, 7), + ) + + +def _qualified(*, no_children=False): + donor_keys = tuple( + ("asec", 2024, 2025, str(i + 1).zfill(5), str(2**53 + i + 1), "1") + for i in range(4) + ) + donors = pd.DataFrame( + { + graph.child.TARGETS[0]: [0.0, 10.0, 0.0, 3.0], + graph.child.TARGETS[1]: [0.0, 0.0, 20.0, 7.0], + graph.WEIGHT: [1.0, 2.0, 3.0, 4.0], + "donor_key": donor_keys, + "household_key": [key[:4] for key in donor_keys], + }, + index=pd.Index( + np.arange(4, dtype=np.int64) + 2**53 + 1, name="native_person_id" + ), + ) + diagnostic = pd.DataFrame( + { + "cohort": [True] * 4, + "ordinary_known": [True] * 4, + "dividend_known": [True] * 4, + }, + index=donors.index, + ) + descriptor = pd.DataFrame( + {"literal": ["unallocated", "allocated", "topcoded", "ordinary_zero"]}, + index=donors.index, + ) + ids = np.array([2**53 + 101, 2**53 + 102, 2**53 + 103], dtype=np.int64) + keys = [ + ( + source, + 2024, + 2025 if source == "asec" else 2024, + str(i + 7).zfill(5), + str(2**53 + 501 + i), + "1", + ) + for i, source in enumerate(("asec", "acs", "asec")) + ] + ages = [20, 20, 40] if no_children else [14, 10, 40] + recipients = pd.DataFrame( + { + "recipient_key": keys, + "source_age": ages, + "source": [key[0] for key in keys], + "ordinary_source_known": [False] * 3, + "dividend_source_known": [False] * 3, + "eligible": [age < 15 for age in ages], + "reason": [ + "explicit_unmeasured_child" + if age < 15 + else "outside_recipient_age_band" + for age in ages + ], + }, + index=pd.Index(ids, name="person_id"), + ) + origins = pd.DataFrame( + [dict(zip(graph.KEY_COLUMNS, key, strict=True)) for key in keys], + index=recipients.index, + ) + origins["selected_receiving_person_id"] = ids + 10000 + origins["selected_receiving_household_id"] = ids + 20000 + origins["household_id"] = ids + 30000 + projection = graph.child.ChildPropertyDonorProjection( + donors, diagnostic, descriptor.copy(), descriptor.copy() + ) + return graph.child.QualifiedChildPropertySources( + projection, + recipients, + { + "catalogue_sha256": "a" * 64, + "preparation_sha256": "b" * 64, + "fixture_only": True, + }, + ), origins + + +def _receiving_frame(qualified, origins): + """Invented exact two-clone parent; no country host or source issuance.""" + home_ids = tuple(dict.fromkeys(origins.household_id.tolist())) + home_map = { + (int(home), clone): 2**53 + 50000 + 2 * i + clone + for i, home in enumerate(home_ids) + for clone in (0, 1) + } + rows = [] + for i, origin in enumerate(origins.itertuples()): + original = origin.Index + for clone in (0, 1): + home = home_map[int(origin.household_id), clone] + rows.append( + { + "person_id": 2**53 + 100000 + 2 * i + clone, + "person_household_id": home, + "person_tax_unit_id": home, + graph.provenance.support_source_id_column("person"): int(original), + graph.provenance.support_clone_index_column("person"): clone, + graph.provenance.spine_source_id_column("person"): int( + origin.selected_receiving_person_id + ), + graph.provenance.support_channel_column("person"): origin.source, + graph.child.TARGETS[0]: np.nan + if qualified.recipients.loc[original, "eligible"] + else 31.0, + graph.child.TARGETS[1]: np.nan + if qualified.recipients.loc[original, "eligible"] + else 41.0, + "unrelated_amount": -12.5, + "source_amount_known": False, + "census_block": "012345678901234", + } + ) + people = pd.DataFrame(rows) + # The invented CREATE must declare the graph's nullable string dtype; + # pandas inference can instead choose the distinct NaN-based str dtype. + for column in ( + graph.provenance.support_channel_column("person"), + "census_block", + ): + people[column] = pd.array( + people[column], dtype=graph.populations.dtype_for_token("string") + ) + people["nullable_source"] = pd.array([pd.NA, True] * len(origins), dtype="boolean") + homes = pd.DataFrame( + { + "household_id": list(home_map.values()), + graph.provenance.support_source_id_column("household"): [ + key[0] for key in home_map + ], + graph.provenance.support_clone_index_column("household"): [ + key[1] for key in home_map + ], + "unrelated_household": 71.0, + } + ) + return Frame( + { + "person": people, + "household": homes, + "tax_unit": pd.DataFrame( + {"tax_unit_id": homes.household_id, "unrelated_tax_unit": 89.0} + ), + }, + EntitySchema(group_entities=("household", "tax_unit")), + { + "household": Weights( + np.arange(len(homes), dtype=np.float64) + 1.5, WeightKind.DESIGN + ) + }, + metadata={"invented_parent": {"purpose": "whole-state preservation"}}, + ) + + +def test_receiving_fixture_declares_strings_without_changing_other_values(): + qualified, origins = _qualified() + people = _receiving_frame(qualified, origins).person + channel = graph.provenance.support_channel_column("person") + canonical_string = pd.StringDtype(storage="python", na_value=pd.NA) + for column, values in ( + (channel, ["asec", "asec", "acs", "acs", "asec", "asec"]), + ("census_block", ["012345678901234"] * 6), + ): + pd.testing.assert_series_equal( + people[column], pd.Series(values, name=column, dtype=canonical_string) + ) + assert people[column].dtype.storage == "python" + assert people[column].dtype.na_value is pd.NA + + source_id = graph.provenance.support_source_id_column("person") + spine_id = graph.provenance.spine_source_id_column("person") + for column, offsets in ( + ("person_id", [100000, 100001, 100002, 100003, 100004, 100005]), + (source_id, [101, 101, 102, 102, 103, 103]), + (spine_id, [10101, 10101, 10102, 10102, 10103, 10103]), + ("person_household_id", [50000, 50001, 50002, 50003, 50004, 50005]), + ("person_tax_unit_id", [50000, 50001, 50002, 50003, 50004, 50005]), + ): + pd.testing.assert_series_equal( + people[column], + pd.Series( + [2**53 + offset for offset in offsets], name=column, dtype="int64" + ), + ) + pd.testing.assert_series_equal( + people["nullable_source"], + pd.Series([pd.NA, True] * 3, name="nullable_source", dtype="boolean"), + ) + pd.testing.assert_series_equal( + people["source_amount_known"], + pd.Series([False] * 6, name="source_amount_known", dtype="bool"), + ) + for column, values in ( + (graph.child.TARGETS[0], [np.nan] * 4 + [31.0, 31.0]), + (graph.child.TARGETS[1], [np.nan] * 4 + [41.0, 41.0]), + ("unrelated_amount", [-12.5] * 6), + ): + pd.testing.assert_series_equal( + people[column], pd.Series(values, name=column, dtype="float64") + ) + + +def _ordering(producer="test.parent"): + edge = ArtifactInput("ordering", producer, "ordering", ORDER_TYPE) + producer_key = "c" * 64 + value = ArtifactValue( + ORDER_BYTES, + ORDER_TYPE, + graph.opaque_artifact_key(producer_key, "ordering"), + producer_key, + NumericScope(), + ) + pins = { + edge.name: { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": graph._sha(value.payload), + } + } + return edge, value, pins + + +def _case(*, zero=False, no_children=False): + qualified, origins = _qualified(no_children=no_children) + parent = graph.populations.Population.from_frame( + _receiving_frame(qualified, origins), "test.allocated" + ) + edge, value, pins = _ordering() + options = _options(zero=zero) + nodes = graph.child_property_nodes( + qualified, origins, parent, options=options, host_edges=(edge,), host_pins=pins + ) + expected, result, payloads, donors, recipients = graph._reconstruct( + qualified, origins, parent, options, nodes + ) + return ( + qualified, + origins, + parent, + nodes, + expected, + result, + payloads, + donors, + recipients, + value, + pins, + ) + + +def _support_kernel_case(kind, tmp_path, on_final): + """Invented boundary for kernel result custody only; no source authority.""" + qualified, origins, parent, nodes, _, _, _, donors, recipients, value, pins = ( + _case() + ) + selected = next(node for node in nodes if node.id == kind) + boundary = SimpleNamespace( + nodes=nodes, + origins=origins, + entry=(None, None, SimpleNamespace(root=tmp_path)), + host_pins=pins, + options_bytes=graph._json(_options().document()), + revoked=False, + calls=0, + pure_calls=0, + ) + + def current(): + graph.require(not boundary.revoked, "TEST_BOUNDARY_REVOKED") + boundary.calls += 1 + if boundary.calls == 2: + on_final() + return qualified + + def pure(): + boundary.pure_calls += 1 + + boundary._current, boundary._pure = current, pure + context = KernelContext( + selected, + {}, + {}, + pd.Series([], dtype="string"), + selected.params, + np.random.default_rng(0), + sources={graph.SOURCE_NAME: tmp_path}, + artifacts={"ordering": value}, + ) + return ( + graph._ChildKernel(boundary, selected.kernel), + context, + boundary, + donors if kind == graph.DONOR else recipients, + parent, + ) + + +def _run_captured_support_kernel(kernel, context, captured): + # Observe the actual created Frame without replacing any production callable. + previous = sys.getprofile() + + def observe(frame, event, result): + if event == "return" and frame.f_code is graph._support_frame.__code__: + captured.append(result) + + sys.setprofile(observe) + try: + return kernel.run(context) + finally: + sys.setprofile(previous) + + +@pytest.mark.parametrize("kind", [graph.DONOR, graph.RECIPIENT]) +def test_private_support_create_preserves_exact_frame_through_final_callback( + kind, tmp_path +): + captured = [] + kernel, context, boundary, table, parent = _support_kernel_case( + kind, tmp_path, lambda: None + ) + parent_stamp = graph.child.physical._population_stamp(parent) + result = _run_captured_support_kernel(kernel, context, captured) + assert len(captured) == 1 and result.frame is captured[0] + assert boundary.calls == 2 and boundary.pure_calls == 1 + assert not boundary.revoked + graph.replay.same_replayed_frame(graph._support_frame(table), result.frame) + pd.testing.assert_frame_equal(result.frame.person[list(table)], table) + assert result.frame.schema == EntitySchema(group_entities=("household",)) + assert not result.frame.links and not result.frame._link_tables + # A support stamp neither coerces the Frame into US_SCHEMA nor changes that + # separate owner's strict source-population admission rule. + with pytest.raises(ValueError, match="^FRAME_TYPE$"): + graph.child.source._frame_identity(result.frame) + assert graph.child.physical._population_stamp(parent) == parent_stamp + + +@pytest.mark.parametrize("kind", [graph.DONOR, graph.RECIPIENT]) +@pytest.mark.parametrize( + "surface", + [ + "person_cell", + "household_cell", + "dtype", + "index", + "table_flags", + "weights", + "weight_kind", + "strata", + "metadata", + "mass_log", + "schema", + "extra_table", + "missing_table", + "hidden_link", + "hidden_weight", + ], +) +def test_private_support_final_callback_mutation_refuses_without_changing_parent( + kind, surface, tmp_path +): + captured, mutations = [], [] + + def mutate(): + assert len(captured) == 1 + frame = captured[0] + mutations.append(surface) + if surface == "person_cell": + frame.person.loc[0, "raw_native_person_id"] = "changed" + elif surface == "household_cell": + frame.table("household").loc[0, "household_id"] += 1 + elif surface == "dtype": + frame.person["source_year"] = frame.person.source_year.astype("int32") + elif surface == "index": + frame.person.index = pd.Index(frame.person.index.to_numpy(), name="changed") + elif surface == "table_flags": + frame.person.flags.allows_duplicate_labels = False + elif surface == "weights": + frame._weights["household"] = Weights( + frame.weights_for("household").values + 1.0, WeightKind.DESIGN + ) + elif surface == "weight_kind": + frame._weights["household"] = Weights( + frame.weights_for("household").values, WeightKind.IMPORTANCE + ) + elif surface == "strata": + frame.strata.iloc[0] = "changed" + elif surface == "metadata": + frame._metadata = {"changed": True} + elif surface == "mass_log": + frame._mass_log = (MassChangeRecord("household", 1.0, 2.0, 2.0, "test"),) + elif surface == "schema": + frame._schema = EntitySchema(group_entities=("household", "tax_unit")) + elif surface == "extra_table": + frame._tables["unexpected"] = pd.DataFrame({"unexpected_id": [1]}) + elif surface == "missing_table": + del frame._tables["household"] + elif surface == "hidden_link": + frame._link_tables["unexpected"] = pd.DataFrame({"person_id": [1]}) + else: + frame._weights["unexpected"] = Weights(np.ones(1), WeightKind.DESIGN) + + kernel, context, boundary, _, parent = _support_kernel_case(kind, tmp_path, mutate) + parent_stamp = graph.child.physical._population_stamp(parent) + with pytest.raises( + ValueError, match="FINAL_KERNEL_RESULT_CHANGED|SUPPORT_FRAME_TYPE" + ): + _run_captured_support_kernel(kernel, context, captured) + assert mutations == [surface] and boundary.calls == 2 + assert boundary.revoked and boundary.pure_calls == 0 + assert graph.child.physical._population_stamp(parent) == parent_stamp + with pytest.raises(ValueError, match="TEST_BOUNDARY_REVOKED"): + kernel.run(context) + + +def test_child_nodes_compile_with_structural_ids_and_source_identity_reads(): + _, _, parent, nodes, *_ = _case() + frame = parent.frame + create = Node( + "test.parent", + "test.child.receiving@1", + sources=(graph.SOURCE_NAME,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned( + entity, + column, + graph.populations.token_for_dtype(frame.table(entity)[column].dtype), + ) + for entity in frame.entities + for column in frame.table(entity) + if column != frame.schema.entity_id_column(entity) + and not ( + entity == frame.schema.person_entity + and column + in { + frame.schema.membership_column(group) + for group in frame.schema.group_entities + } + ) + ), + artifact_outputs=(ArtifactOutput("ordering", ORDER_TYPE),), + ) + allocated = Node( + parent.version, + "test.child.allocate@1", + base=create.id, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "importance", "conserve"), + mass="conserve", + ) + compiled = compile_graph( + Graph( + "us", + (SourceRef(graph.SOURCE_NAME, "frame-store"),), + (create, allocated, *nodes), + ) + ) + assert graph.ATTACH in compiled.predecessors[graph.VERIFY] + for node in nodes[-2:]: + reads = {s.entity: set(s.columns) for s in node.inputs} + for entity in frame.entities: + expected = set(frame.table(entity)) + expected.remove(frame.schema.entity_id_column(entity)) + if entity == frame.schema.person_entity: + expected.difference_update( + frame.schema.membership_column(group) + for group in frame.schema.group_entities + ) + if node.id == graph.VERIFY: + expected.update((graph.STATUS, graph.IMPUTED)) + assert reads[entity] == expected + assert graph.provenance.support_source_id_column("person") in reads["person"] + assert graph.provenance.spine_source_id_column("person") in reads["person"] + + +@pytest.mark.parametrize( + "zero,no_children", [(False, False), (True, False), (False, True)] +) +def test_real_empirical_reconstruction_preserves_complete_parent_and_clone_pairs( + zero, no_children +): + q, origins, parent, nodes, expected, result, payloads, donors, recipients, _, _ = ( + _case(zero=zero, no_children=no_children) + ) + assert len(nodes) == 6 + assert nodes[3].params["eligibility_column"] == "eligible" + assert len(recipients) == len(q.recipients) + assert len(graph._support_frame(recipients).person) == 3 + assert len(donors) == 4 # Original donor support, never two cloned copies. + assert not nodes[0].inputs and nodes[0].outputs + assert graph.SOURCE_NAME in nodes[0].sources + for entity in parent.frame.entities: + names = [ + name + for name in parent.frame.table(entity) + if entity != "person" or name not in graph.child.TARGETS + ] + pd.testing.assert_frame_equal( + expected.frame.table(entity)[names], + parent.frame.table(entity)[names], + check_exact=True, + ) + for name in graph.child.TARGETS: + original = expected.frame.person[ + graph.provenance.support_source_id_column("person") + ] + for identity, group in expected.frame.person.groupby(original): + assert len(group) == 2 and group[name].nunique(dropna=False) == 1 + if q.recipients.loc[identity, "eligible"]: + assert group[graph.IMPUTED].all() + if zero: + assert group[name].eq(0.0).all() + assert group[graph.STATUS].eq("imputed_zero").all() + replayed = graph.populations.patch(parent, nodes[4], result) + graph.replay.same_replayed_population(expected, replayed) + completion = json.loads(payloads[graph.ATTACH, "completion"]) + assert completion["source_knownness_changed"] is False + assert ( + completion["receiving_imputed_count"] + == 2 * completion["original_recipient_count"] + ) + + +@pytest.mark.parametrize( + "defect", + [ + "original_float", + "native_id", + "clone", + "household", + "channel", + "known_o", + "known_d", + ], +) +def test_exact_source_clone_join_and_known_incumbent_refuse(defect): + q, origins = _qualified() + frame = _receiving_frame(q, origins) + people = frame.person + if defect == "original_float": + people[graph.provenance.support_source_id_column("person")] = people[ + graph.provenance.support_source_id_column("person") + ].astype(float) + elif defect == "native_id": + people.loc[0, graph.provenance.spine_source_id_column("person")] += 1 + elif defect == "clone": + people.loc[0, graph.provenance.support_clone_index_column("person")] = 1 + elif defect == "household": + people.loc[0, "person_household_id"] = people.loc[2, "person_household_id"] + elif defect == "channel": + people.loc[0, graph.provenance.support_channel_column("person")] = "acs" + else: + people.loc[0, graph.child.TARGETS[defect == "known_d"]] = 0.0 + parent = graph.populations.Population.from_frame(frame, "test.allocated") + edge, _, pins = _ordering() + with pytest.raises(ValueError): + nodes = graph.child_property_nodes( + q, origins, parent, options=_options(), host_edges=(edge,), host_pins=pins + ) + graph._reconstruct(q, origins, parent, _options(), nodes) + + +@pytest.mark.parametrize( + "defect", + ["amount", "knownness", "unrelated", "group", "weights", "owners", "ledger"], +) +def test_independent_whole_population_comparison_refuses_unowned_or_completed_drift( + defect, +): + _, _, parent, nodes, expected, result, *_ = _case() + changed = graph.populations.patch(parent, nodes[4], result) + if defect == "amount": + changed.frame.person.loc[0, graph.child.TARGETS[0]] += 1.0 + elif defect == "knownness": + changed.frame.person.loc[0, "source_amount_known"] = True + elif defect == "unrelated": + changed.frame.person.loc[0, "unrelated_amount"] += 1.0 + elif defect == "group": + changed.frame.table("tax_unit").loc[0, "unrelated_tax_unit"] += 1.0 + elif defect == "weights": + altered = changed.design_weights["household"].copy() + altered[0] += 1.0 + changed = replace(changed, design_weights={"household": altered}) + elif defect == "owners": + changed = replace( + changed, owners={**changed.owners, ("person", graph.STATUS): "forged"} + ) + else: + # A different ledger tuple type is invalid even when empty. + object.__setattr__(changed, "mass_ledger", []) + with pytest.raises(ValueError): + graph.replay.same_replayed_population(expected, changed) + + +@pytest.mark.parametrize( + "defect", ["model", "draws", "scenario", "completion", "producer", "host"] +) +def test_exact_reconstructed_artifact_bindings_refuse_changed_values(defect): + *_, nodes, expected, result, payloads, donors, recipients, order, pins = _case() + del expected, result, donors, recipients + node = nodes[-1] + values = {} + for edge in node.artifact_inputs: + if edge.name == "ordering": + values[edge.name] = order + else: + producer = graph._sha(edge.producer.encode()) + values[edge.name] = ArtifactValue( + payloads[edge.producer, edge.artifact], + edge.type, + graph.opaque_artifact_key(producer, edge.artifact), + producer, + NumericScope( + Numeric.PLATFORM_BITWISE, platform=graph.platform_fingerprint() + ), + ) + if defect == "producer": + value = values["model_metadata"] + producer = "f" * 64 + values["model_metadata"] = replace( + value, + producer_key=producer, + key=graph.opaque_artifact_key(producer, "model_metadata"), + ) + else: + name = "ordering" if defect == "host" else defect + values[name] = replace(values[name], payload=values[name].payload + b" ") + with pytest.raises(ValueError): + graph._artifacts(node, values, payloads, pins) + + +def test_target_sampling_digest_does_not_change_full_donor_projection_or_model(): + q, origins = _qualified() + donor_payload = graph._projections(q, origins)[2] + changed = replace(q, evidence={**q.evidence, "preparation_sha256": "f" * 64}) + assert graph._projections(changed, origins)[2] == donor_payload + assert graph._projections(changed, origins)[3] != graph._projections(q, origins)[3] + models = [] + for qualified in (q, changed): + table = graph._projections(qualified, origins)[0] + models.append( + empirical.fit_joint_empirical( + table, + targets=graph.child.TARGETS, + donor_keys=tuple( + table[list(graph.KEY_COLUMNS)].itertuples(index=False, name=None) + ), + household_keys=tuple( + table[list(graph.KEY_COLUMNS[:4])].itertuples( + index=False, name=None + ) + ), + support=_options().support, + weights=graph.WEIGHT, + ).to_bytes() + ) + assert models[0] == models[1] + + +@pytest.mark.parametrize( + "document", + [ + {"a": [1.5, 2.0, -0.0, 1e-9], "z": "\u00e9", "n": None, "t": True}, + { + "protocol": "invented/draw", + "rows": [ + { + "support_id": str(3_565_013 - i), + "coordinate": [["str", "acs"], ["int", 2024]], + "values": [i / 7, 0.0], + "pattern": i % 4, + "donor_key": None, + } + for i in range(300) + ], + }, + ], +) +def test_graph_json_is_the_adapter_encoding_at_every_segment_size( + monkeypatch, document +): + """The draw document is a whole-roster stream; its bytes are json.dumps's.""" + whole = graph.adapter._json(document) + assert graph._json(document) == whole + encoder = json.JSONEncoder(sort_keys=True, separators=(",", ":"), allow_nan=False) + longest = max(len(piece.encode()) for piece in encoder.iterencode(document)) + assert longest < len(whole) + for segment in sorted({longest, longest + 1, 4096, len(whole)}): + monkeypatch.setattr(graph, "MAX_ARTIFACT_BYTES", segment) + assert graph._json(document) == whole + monkeypatch.setattr(graph, "MAX_ARTIFACT_BYTES", longest - 1) + with pytest.raises(ValueError, match="GRAPH_ARTIFACT_SIZE"): + graph._json(document) + monkeypatch.setattr(graph, "MAX_ARTIFACT_BYTES", 64 * 1024**2) + monkeypatch.setattr(graph, "MAX_ROSTER_BYTES", len(whole)) + assert graph._json(document) == whole + monkeypatch.setattr(graph, "MAX_ROSTER_BYTES", len(whole) - 1) + with pytest.raises(ValueError, match="GRAPH_ARTIFACT_LIMIT"): + graph._json(document) + + +def test_the_draw_decoder_ceiling_is_the_producers_total(): + assert graph.adapter.MAX_DRAW_BYTES == graph.MAX_ROSTER_BYTES + assert graph.MAX_ROSTER_BYTES == 64 * graph.MAX_ARTIFACT_BYTES + + +def test_original_recipient_roster_ceiling_refuses_at_a_patched_down_value( + monkeypatch, +): + """MAX_ORIGINALS bounds the original recipients, one per stacked person. + + It takes the row-count rule's 15,000,000 from the 3,565,013 stacked persons + a full-source selection carries, and the fit adapter's MAX_RECIPIENTS, + which bounds the same roster one node later, is the same number. + """ + assert graph.MAX_ORIGINALS == 15_000_000 == graph.adapter.MAX_RECIPIENTS + q, origins = _qualified() + monkeypatch.setattr(graph, "MAX_ORIGINALS", len(origins) - 1) + with pytest.raises(ValueError, match="ORIGINAL_RECIPIENT_ROSTER"): + graph._projections(q, origins) + monkeypatch.setattr(graph, "MAX_ORIGINALS", len(origins)) + assert len(graph._projections(q, origins)[0]) > 0 diff --git a/packages/microcosm-build/tests/test_us_child_property_income_graph_owner.py b/packages/microcosm-build/tests/test_us_child_property_income_graph_owner.py new file mode 100644 index 000000000..d788a6c06 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_child_property_income_graph_owner.py @@ -0,0 +1,378 @@ +"""Real small survey owners plus real graph; invented receiving host only. + +No actual country financial/tax/PUF host acceptance is asserted by these tests. +""" + +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +import test_us_child_property_income_source_owner as source_fixture +from test_us_child_property_income_graph import ( + ORDER_BYTES, + ORDER_TYPE, + _options, + _ordering, + _receiving_frame, +) + +from microcosm.build.us_runtime import graph_child_property_income as graph +from microcosm.build.us_runtime import graph_survey_population as survey_graph +from microcosm.frame import WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactValue, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + NumericScope, + Owned, + SourceRef, + StructuralDelta, + WeightTransition, + compile_graph, + run_graph, +) + +actual = source_fixture.actual + + +def _artifact(manifest, store, edge): + receipt = manifest.node(edge.producer) + key = receipt.opaque_artifacts[edge.artifact] + numeric = receipt.capabilities.numeric + return ArtifactValue( + store.load_bytes(key), + edge.type, + key, + receipt.key, + NumericScope( + numeric, + platform=graph.platform_fingerprint() + if numeric is Numeric.PLATFORM_BITWISE + else None, + ), + ) + + +def _run(actual, tmp_path): + preparation = actual.partial + entry = preparation._checked() + origins = graph._origins(entry) + qualified = graph.child.qualify_child_property_sources(preparation) + receiving = _receiving_frame(qualified, origins) + create_id, allocated_id = "test.child.receiving", "test.child.allocated" + allocated_weights = Weights( + receiving.weights_for("household").values, WeightKind.IMPORTANCE + ) + + class Receiving(KernelBase): + ref = "test.child.receiving@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + return KernelResult(frame=receiving, artifacts={"ordering": ORDER_BYTES}) + + class Allocate(KernelBase): + ref = "test.child.allocate@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def run(self, context): + return KernelResult(weights=allocated_weights) + + create = Node( + create_id, + Receiving.ref, + sources=(graph.SOURCE_NAME,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned( + entity, + col, + graph.populations.token_for_dtype(receiving.table(entity)[col].dtype), + ) + for entity in receiving.entities + for col in receiving.table(entity) + if col != receiving.schema.entity_id_column(entity) + and not ( + entity == receiving.schema.person_entity + and col + in { + receiving.schema.membership_column(group) + for group in receiving.schema.group_entities + } + ) + ), + artifact_outputs=(ArtifactOutput("ordering", ORDER_TYPE),), + ) + allocate = Node( + allocated_id, + Allocate.ref, + base=create_id, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "importance", "conserve"), + mass="conserve", + ) + refs = (SourceRef(graph.SOURCE_NAME, survey_graph.SOURCE_CODEC),) + registry = KernelRegistry() + registry.register(Receiving()) + registry.register(Allocate()) + store = ContentStore( + tmp_path / "store", + codecs=survey_graph.survey_population_source_codecs( + snapshot_root=tmp_path / "unused-codec-snapshot" + ), + ) + sources = {graph.SOURCE_NAME: entry[2].root} + captured = {} + prefix = run_graph( + compile_graph(Graph("us", refs, (create, allocate))), + sources=sources, + store=store, + kernels=registry, + _population_observer=lambda name, pop: captured.update({name: pop}), + ) + parent = captured[allocated_id] + stamp = graph.child.physical._population_stamp(parent) + calls = SimpleNamespace(count=0, on_call=None) + + def require_parent(): + calls.count += 1 + preparation._checked() + assert graph.child.physical._population_stamp(parent) == stamp + if calls.on_call is not None: + calls.on_call(calls.count) + return parent + + edge = ArtifactInput("ordering", create_id, "ordering", ORDER_TYPE) + value = _artifact(prefix, store, edge) + pins = { + "ordering": { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": graph._sha(value.payload), + } + } + boundary = graph.ChildPropertyBoundary( + preparation, + parent, + require_parent=require_parent, + options=_options(only_positive=True), + host_edges=(edge,), + host_pins=pins, + ) + for kernel in boundary.kernels(): + registry.register(kernel) + compiled = compile_graph(Graph("us", refs, (create, allocate, *boundary.nodes))) + captured = {} + cold = run_graph( + compiled, + sources=sources, + store=store, + kernels=registry, + _population_observer=lambda name, pop: captured.update({name: pop}), + ) + assert all(not cold.node(node.id).hit for node in boundary.nodes) + + def evidence(manifest, observed): + inputs = { + edge.name: _artifact(manifest, store, edge) + for edge in boundary.nodes[-1].artifact_inputs + } + verification = _artifact( + manifest, + store, + ArtifactInput( + "verification", graph.VERIFY, "verification", graph.VERIFICATION_TYPE + ), + ) + return dict( + population=observed[graph.VERIFY], + artifacts=inputs, + support_populations={ + name: observed[name] for name in (graph.DONOR, graph.RECIPIENT) + }, + verification=verification, + ) + + def check(manifest, observed): + return boundary.verify_materialized(**evidence(manifest, observed)) + + check(cold, captured) + warm_capture = {} + calls_before = calls.count + warm = run_graph( + compiled, + sources=sources, + store=store, + kernels=registry, + resume="require", + _population_observer=lambda name, pop: warm_capture.update({name: pop}), + ) + assert all(item.hit for item in warm.nodes.values()) + assert calls.count == calls_before # Required replay skips every kernel. + check(warm, warm_capture) + assert calls.count > calls_before # Retained host check remains mandatory. + return SimpleNamespace( + boundary=boundary, + cold=cold, + warm=warm, + captured=warm_capture, + store=store, + calls=calls, + check=check, + evidence=evidence, + qualified=qualified, + parent=parent, + ) + + +def test_actual_owner_full_donor_and_actual_graph_complete_identical_child_clones( + actual, tmp_path +): + result = _run(actual, tmp_path) + donor = result.captured[graph.DONOR].frame.person + assert len(donor) == 1 + assert ( + donor[graph.WEIGHT].iloc[0] + == result.qualified.donor_projection.donors[graph.WEIGHT].iloc[0] + ) + assert tuple(result.qualified.donor_projection.donors.donor_key)[0] not in set( + result.qualified.recipients.recipient_key + ) + output = result.captured[graph.VERIFY].frame.person + original = output[graph.provenance.support_source_id_column("person")] + for identity in result.qualified.recipients.index[ + result.qualified.recipients.eligible + ]: + copies = output.loc[original == identity] + assert len(copies) == 2 and copies[graph.IMPUTED].all() + np.testing.assert_array_equal( + copies[list(graph.child.TARGETS)].iloc[0], + copies[list(graph.child.TARGETS)].iloc[1], + ) + for name in result.parent.frame.person: + if name not in graph.child.TARGETS: + pd.testing.assert_series_equal( + result.parent.frame.person[name], output[name], check_exact=True + ) + actual.partial._checked() + + +@pytest.mark.parametrize("when", ["first", "final"]) +def test_actual_owner_callbacks_cannot_change_the_completed_population( + actual, tmp_path, when +): + result = _run(actual, tmp_path) + baseline = result.calls.count + output = result.captured[graph.VERIFY] + count = baseline + (1 if when == "first" else 2) + before = output.frame.person.loc[0, "unrelated_amount"] + + def mutate(n): + if n == count: + output.frame.person.loc[0, "unrelated_amount"] += 1.0 + + result.calls.on_call = mutate + with pytest.raises(ValueError): + result.check(result.warm, result.captured) + assert output.frame.person.loc[0, "unrelated_amount"] == before + 1.0 + assert result.boundary.revoked + result.calls.on_call = None + with pytest.raises(ValueError, match="REVOKED"): + result.check(result.warm, result.captured) + actual.partial._checked() + + +@pytest.mark.parametrize("when", ["first", "final"]) +@pytest.mark.parametrize( + "surface", + [ + "artifact_replace", + "support_add", + "support_remove", + "verification_payload", + "verification_producer", + ], +) +def test_actual_owner_callbacks_cannot_change_materialized_evidence( + actual, tmp_path, when, surface +): + result = _run(actual, tmp_path) + inputs = result.evidence(result.warm, result.captured) + before = graph.child.physical._population_stamp(inputs["population"]) + target_call = result.calls.count + (1 if when == "first" else 2) + mutations = [] + + def mutate(count): + if count != target_call: + return + mutations.append(surface) + if surface == "artifact_replace": + value = inputs["artifacts"]["draws"] + inputs["artifacts"]["draws"] = replace(value, payload=value.payload + b" ") + elif surface == "support_add": + inputs["support_populations"]["unexpected"] = inputs["support_populations"][ + graph.DONOR + ] + elif surface == "support_remove": + del inputs["support_populations"][graph.RECIPIENT] + elif surface == "verification_payload": + value = inputs["verification"] + object.__setattr__(value, "payload", value.payload + b" ") + else: + # Keep the descriptor internally valid: changed provenance must + # still be detected across either original owner borrow. + value = inputs["verification"] + producer = "f" * 64 + assert value.producer_key != producer + object.__setattr__(value, "producer_key", producer) + object.__setattr__( + value, "key", graph.opaque_artifact_key(producer, "verification") + ) + + result.calls.on_call = mutate + with pytest.raises(ValueError, match="EVIDENCE_CHANGED|SUPPORT_POPULATION_ROSTER"): + result.boundary.verify_materialized(**inputs) + assert mutations == [surface] + assert graph.child.physical._population_stamp(inputs["population"]) == before + assert result.boundary.revoked + result.calls.on_call = None + with pytest.raises(ValueError, match="REVOKED"): + # Fresh valid descriptors cannot resurrect the boundary. + result.check(result.warm, result.captured) + actual.partial._checked() + + +def test_copied_source_owner_cannot_construct_the_country_boundary(actual): + preparation = object.__new__(type(actual.partial)) + object.__setattr__(preparation, "payload", actual.partial.payload) + qualified = actual.partial_values + origins = graph._origins(actual.partial._checked()) + parent = graph.populations.Population.from_frame( + _receiving_frame(qualified, origins), "test.allocated" + ) + edge, _, pins = _ordering() + with pytest.raises(ValueError, match="UNISSUED_OR_CHANGED"): + graph.ChildPropertyBoundary( + preparation, + parent, + require_parent=lambda: parent, + options=_options(only_positive=True), + host_edges=(edge,), + host_pins=pins, + ) + actual.partial._checked() diff --git a/packages/microcosm-build/tests/test_us_child_property_income_source.py b/packages/microcosm-build/tests/test_us_child_property_income_source.py new file mode 100644 index 000000000..a27615a30 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_child_property_income_source.py @@ -0,0 +1,292 @@ +"""Pure invented source semantics: no donor fit, native reads or owner issuance.""" + +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_child_property_income_source as child +from microcosm.build.us_runtime import survey_population_domains as domains + + +def literal_inputs(ages=(15, 16, 17, 14), weights=(100, 200, 300, 400)): + rows, homes = [], [] + for index, (age, weight) in enumerate(zip(ages, weights, strict=True), 1): + key = domains.HouseholdKey(domains.Source.ASEC, 2024, 2025, f"{index:05d}") + person = domains.AsecPerson(f"{index:022d}", "1", str(age), "2", "source", key) + homes.append( + domains.AsecHousehold(key, "1", "6", "1", "1", str(weight), (person,)) + ) + raw = { + name: "0" + for name in (*child.interest.READ_COLUMNS, *child.dividend.READ_COLUMNS) + } + raw.update( + PERIDNUM=person.peridnum, + PH_SEQ=str(index), + A_LINENO="1", + A_AGE=str(age), + INT_YN="2" if age >= 15 else "0", + DIV_YN="2" if age >= 15 else "0", + ) + rows.append(raw) + frame = pd.DataFrame( + rows, + index=pd.Index( + np.arange(2**53 + 1, 2**53 + 1 + len(rows), dtype=np.int64), + name="native_person_id", + ), + ) + return ( + frame[list(child.interest.READ_COLUMNS)].copy(), + frame[list(child.dividend.READ_COLUMNS)].copy(), + tuple(homes), + ) + + +def test_full_observed_pair_uses_ordinary_interest_without_retirement_or_rental(): + interest, dividend, homes = literal_inputs() + interest.loc[interest.index[0], ["INT_YN", "INT_VAL", "TRDINT_VAL"]] = [ + "1", + "200", + "100", + ] + dividend.loc[dividend.index[0], ["DIV_YN", "DIV_VAL"]] = ["1", "50"] + result = child.project_child_property_donors(interest, dividend, homes) + assert len(result.donors) == 3 + assert result.donors.iloc[0][child.TARGETS[0]] == 100 # Combined INT_VAL is 200. + assert result.donors.iloc[0][child.TARGETS[1]] == 50 + assert result.interest.RINT_VAL1_amount.isna().all() + assert result.donors.original_household_design_weight.tolist() == [1, 2, 3] + assert result.donors.donor_key.iloc[0] == ( + "asec", + 2024, + 2025, + "00001", + "0000000000000000000001", + "1", + ) + assert result.diagnostics.reason.iloc[-1] == "outside_donor_age_band" + + +def test_observed_ordinary_zero_and_allocation_provenance_are_retained(): + interest, dividend, homes = literal_inputs() + interest.loc[interest.index[0], ["INT_YN", "INT_VAL", "I_INTYN", "TTRDINT_VAL"]] = [ + "1", + "200", + "1", + "1", + ] + dividend.loc[dividend.index[0], ["DIV_YN", "DIV_VAL", "I_DIVYN", "TDIV_VAL"]] = [ + "1", + "50", + "2", + "1", + ] + result = child.project_child_property_donors(interest, dividend, homes) + assert ( + result.interest.TRDINT_VAL_reporting_status.iloc[0] == "observed_zero_component" + ) + assert result.donors.iloc[0][child.TARGETS[0]] == 0 + assert result.donors.iloc[0][child.TARGETS[1]] == 50 + assert result.dividend.TDIV_VAL_code.iloc[0] == 1 + assert result.interest.TTRDINT_VAL_code.iloc[0] == 1 + assert result.dividend.I_DIVYN_code.iloc[0] == 2 + + +def test_zero_weight_observed_donor_is_retained_as_zero_mass_diagnostic(): + interest, dividend, homes = literal_inputs(weights=(0, 200, 300, 400)) + result = child.project_child_property_donors(interest, dividend, homes) + assert len(result.donors) == 3 + assert result.diagnostics.observed_donor.iloc[0] + assert not result.diagnostics.positive_mass_donor.iloc[0] + assert result.diagnostics.reason.iloc[0] == "zero_design_mass" + assert result.diagnostics.observed_zero_pair.iloc[0] + + +@pytest.mark.parametrize( + "change", ["age", "line", "household", "peridnum", "missing", "duplicate"] +) +def test_full_catalogue_coordinate_or_coverage_mismatch_refuses(change): + interest, dividend, homes = literal_inputs() + if change == "missing": + homes = homes[:-1] + elif change == "duplicate": + homes = (*homes, homes[0]) + else: + field, value = { + "age": ("A_AGE", "16"), + "line": ("A_LINENO", "2"), + "household": ("PH_SEQ", "5"), + "peridnum": ("PERIDNUM", "other"), + }[change] + interest.loc[interest.index[0], field] = value + dividend.loc[dividend.index[0], field] = value + with pytest.raises(ValueError, match="CHILD_PROPERTY_"): + child.project_child_property_donors(interest, dividend, homes) + + +@pytest.mark.parametrize("token", ["15.5", " 15", "100", "-1", ""]) +def test_imprecise_or_invalid_source_age_is_never_truncated(token): + interest, dividend, homes = literal_inputs() + interest.loc[interest.index[0], "A_AGE"] = token + dividend.loc[dividend.index[0], "A_AGE"] = token + with pytest.raises(ValueError, match="AGE"): + child.project_child_property_donors(interest, dividend, homes) + + +@pytest.mark.parametrize( + "token,status", + [ + ("", "missing_amount"), + ("bad", "invalid_amount_literal"), + ("100", "contradictory_no_nonzero"), + ], +) +def test_missing_and_source_error_donor_exclusions_remain_distinct(token, status): + interest, dividend, homes = literal_inputs() + interest.loc[interest.index[0], "TRDINT_VAL"] = token + result = child.project_child_property_donors(interest, dividend, homes) + assert result.interest.TRDINT_VAL_reporting_status.iloc[0] == status + assert result.diagnostics.reason.iloc[0] == ( + "unresolved_observed_pair" if token == "" else "source_review_required" + ) + assert interest.index[0] not in result.donors.index + + +def recipient_inputs(): + interest, dividend, homes = literal_inputs(ages=(14, 15, 16, 17)) + projected = child.project_child_property_donors(interest, dividend, homes) + a = projected.interest.iloc[:1].copy() + b = projected.dividend.iloc[:1].copy() + native = int(a.index[0]) + for table in (a, b): + table["native_person_id"] = np.array([native], dtype=np.int64) + table.index = pd.Index([11], dtype="int64", name="person_id") + acs = pd.DataFrame( + dict( + native_person_id=[22], + AGEP=["9"], + INTP=[""], + property_income_status=["outside_universe_blank"], + property_income_adjustment_known=[True], + ), + index=pd.Index([12], dtype="int64", name="person_id"), + ) + columns = [ + "person_id", + "source", + "source_year", + "survey_year", + "raw_native_household_id", + "raw_native_person_id", + "native_line_numeric_original", + "selected_receiving_person_id", + ] + origins = { + "persons": { + "columns": columns, + "rows": [ + [ + 11, + "asec", + 2024, + 2025, + "00001", + "0000000000000000000001", + "1", + native, + ], + [12, "acs", 2024, 2024, "2024HU0000001", "1", "1", 22], + ], + } + } + return origins, a, b, acs + + +def test_only_explicitly_unmeasured_children_are_eligible(): + origins, a, b, acs = recipient_inputs() + before = a.copy(deep=True) + result = child.project_child_property_recipients(origins, a, b, acs) + assert result.eligible.tolist() == [True, True] + assert ( + not result.ordinary_source_known.any() + and not result.dividend_source_known.any() + ) + pd.testing.assert_frame_equal(a, before) + + +@pytest.mark.parametrize( + "change", + [ + "acs_adjustment", + "acs_numeric_zero", + "acs_negative", + "asec_positive", + "asec_malformed", + "known_incumbent", + ], +) +def test_child_source_errors_and_known_incumbents_are_never_imputation_eligibility( + change, +): + origins, a, b, acs = recipient_inputs() + if change == "acs_adjustment": + acs.loc[12, "property_income_adjustment_known"] = False + elif change in ("acs_numeric_zero", "acs_negative"): + acs.loc[12, "INTP"] = "0" if change == "acs_numeric_zero" else "-10" + acs.loc[12, "property_income_status"] = "outside_universe_observation" + elif change == "asec_positive": + a.loc[11, "TRDINT_VAL_published_amount"] = 7 + a.loc[11, "TRDINT_VAL_reporting_status"] = ( + "contradictory_outside_reporting_universe" + ) + elif change == "asec_malformed": + a.loc[11, "TRDINT_VAL_literal_status"] = "malformed" + else: + a.loc[11, "TRDINT_VAL_amount_known"] = True + result = child.project_child_property_recipients(origins, a, b, acs) + target = 12 if change.startswith("acs") else 11 + assert not result.loc[target, "eligible"] + + +def test_complete_private_seal_checks_exact_tuple_coordinates(): + i, d, h = literal_inputs() + donor = child.project_child_property_donors(i, d, h) + origins, a, b, acs = recipient_inputs() + value = child.QualifiedChildPropertySources( + donor, child.project_child_property_recipients(origins, a, b, acs), {} + ) + before = child.child_property_sources_seal(value) + changed = donor.donors.copy(deep=True) + changed.at[changed.index[0], "donor_key"] = ( + "asec", + 2024, + 2025, + "00001", + 2**53 + 1, + "1", + ) + altered = replace(value, donor_projection=replace(donor, donors=changed)) + assert child.child_property_sources_seal(altered) != before + + +def test_recipient_axis_ceiling_refuses_at_a_patched_down_value(monkeypatch): + """MAX_RECIPIENT_ROWS bounds the stacked-person axis the projection classifies. + + MAX_ROWS stays the ASEC source's own row bound. The recipient axis is the + selection's stacked persons, 3,565,013 at full source, so it takes the + row-count rule's 15,000,000. A patched-down value refuses RECIPIENT_AXIS + before any frame is joined, and the value one row higher admits. + """ + assert child.MAX_RECIPIENT_ROWS == 15_000_000 + assert child.MAX_RECIPIENT_ROWS == -(-4 * 3_565_013 // 1_000_000) * 1_000_000 + origins, a, b, acs = recipient_inputs() + rows = len(origins["persons"]["rows"]) + monkeypatch.setattr(child, "MAX_RECIPIENT_ROWS", rows - 1) + with pytest.raises(ValueError, match="RECIPIENT_AXIS"): + child.project_child_property_recipients(origins, a, b, acs) + monkeypatch.setattr(child, "MAX_RECIPIENT_ROWS", rows) + result = child.project_child_property_recipients(origins, a, b, acs) + assert len(result) == rows diff --git a/packages/microcosm-build/tests/test_us_child_property_income_source_owner.py b/packages/microcosm-build/tests/test_us_child_property_income_source_owner.py new file mode 100644 index 000000000..6fe8ab124 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_child_property_income_source_owner.py @@ -0,0 +1,368 @@ +"""Actual bounded source owners on invented inputs; tests authored, not native.""" + +import hashlib +import json +import shutil +import sys +from fractions import Fraction +from types import SimpleNamespace + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_child_property_income_source as child +from microcosm.fit.joint_empirical import ( + SupportRequirements, + SupportThreshold, + fit_joint_empirical, +) + + +def _change_unissued_literals(arguments, monkeypatch, donor, changes): + """Change invented raw literals and rebuild their real attachment before issuance.""" + from microcosm.build.us_runtime import asec_person_income_source as restoration + + folder = arguments["source_dir"] / "asec" + pins, paths = [], {} + for year, member, archive, *_ in child.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + if year == 2024: + selected = raw.PERIDNUM.eq(str(donor - 100).zfill(22)) + assert selected.sum() == 1 + for name, token in {"I_INTYN": "10", **changes}.items(): + raw.loc[selected, name] = token + raw.to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (child.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = folder.parent.parent / "child-restored-money" + restoration.restore_asec_person_income_source( + folder / "parent.h5", + folder / "household-attachment.h5", + member_paths=paths, + output_dir=output, + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + + +def source_arguments(tmp_path, monkeypatch, *, donor_changes=None): + """Place the sole eligible teenager in the unselected household before issuance.""" + import test_us_current_asec_income_routing as routing_fixture + import test_us_current_property_income_sources as property_fixture + import test_us_survey_population_preparation as preparation_fixture + + fraction, seed = Fraction(2, 3), 41 + domain = child.domains.Domain.SHARED_HOUSING.value + chosen = child.source.survey_domain_sample.select_domain_households( + row_ids=("00007", "00008"), + source_channels=("asec", "asec"), + domain_keys=(domain, domain), + cells=((domain, "asec"),), + fraction=fraction, + seed=seed, + )[0] + assert len(chosen.positions) == 1 + selected_adult = (105, 107)[chosen.positions[0]] + donor = 107 if selected_adult == 105 else 105 + ages = {105: 55, 106: 14, 107: 55, 108: 14} + ages[donor] = 15 + monkeypatch.setattr(routing_fixture, "AGES", ages) + original_fixture = preparation_fixture.fixture + + def positive_households(path, patch): + return original_fixture(path, patch, zero=False) + + monkeypatch.setattr(routing_fixture, "fixture", positive_households) + arguments = property_fixture.source_arguments(tmp_path, monkeypatch) + _change_unissued_literals(arguments, monkeypatch, donor, donor_changes or {}) + # Both preparation requests are created before either owner is issued. + source_dir = arguments["source_dir"] + partial = tmp_path / "partial-source" + shutil.copytree(source_dir, partial) + request = json.loads((partial / "selection-request.json").read_bytes()) + request["fraction"] = [fraction.numerator, fraction.denominator] + request["seed"] = seed + (partial / "selection-request.json").write_bytes( + json.dumps(request, sort_keys=True, separators=(",", ":")).encode() + ) + captures = tmp_path / "partial-captures" + captures.mkdir() + full = { + name: arguments[name] + for name in ("source_dir", "snapshot_root", "fraction", "seed") + } + smaller = dict( + source_dir=partial, snapshot_root=captures, fraction=fraction, seed=seed + ) + return full, smaller, donor + + +@pytest.fixture(scope="module") +def actual(tmp_path_factory): + root = tmp_path_factory.mktemp("child-property-source-owner") + with pytest.MonkeyPatch.context() as patch: + full_args, partial_args, donor = source_arguments(root, patch) + full = child.source.prepare_authenticated_survey_population(**full_args) + partial = child.source.prepare_authenticated_survey_population(**partial_args) + full_values = child.qualify_child_property_sources(full) + partial_values = child.qualify_child_property_sources(partial) + yield SimpleNamespace( + full=full, + partial=partial, + values=full_values, + partial_values=partial_values, + donor=donor, + ) + full._checked() + partial._checked() + + +def test_full_source_donor_is_available_when_target_sample_has_no_teenagers(actual): + selected = actual.partial._checked()[2].frame.person + asec = selected.loc[ + selected[child.routing.support_channel_column("person")].eq("asec") + ] + assert not asec.A_AGE.between(15, 17).any() + left, right = actual.values.donor_projection, actual.partial_values.donor_projection + assert left.donors.index.tolist() == right.donors.index.tolist() == [actual.donor] + pd.testing.assert_frame_equal(left.donors, right.donors) + pd.testing.assert_frame_equal(left.diagnostics, right.diagnostics) + homes = actual.full._checked()[2].catalogues[1]._checked()[2].households + assert {home.key.native_id: home.hsup_wgt for home in homes} == { + "00007": "000255212", + "00008": "000010000", + } + assert left.diagnostics.original_household_design_weight.to_dict() == { + 105: 2552.12, + 106: 2552.12, + 107: 100.0, + 108: 100.0, + } + expected_weight = 2552.12 if actual.donor == 105 else 100.0 + assert left.donors.original_household_design_weight.tolist() == [expected_weight] + assert left.interest.loc[actual.donor, "allocation_origin"] == "publisher_allocated" + one = SupportThreshold(1, 1, 1.0, 1.0) + support = SupportRequirements( + "invented-one-source-donor", "test", one, one, one, (3,) + ) + + def fit(projection): + frame = projection.donors + return fit_joint_empirical( + frame, + targets=child.TARGETS, + donor_keys=frame.donor_key.tolist(), + household_keys=frame.household_key.tolist(), + support=support, + weights="original_household_design_weight", + ) + + assert fit(left).to_bytes() == fit(right).to_bytes() + assert ( + actual.values.evidence["preparation_sha256"] + != actual.partial_values.evidence["preparation_sha256"] + ) + + +def test_actual_source_knownness_and_complete_preparation_are_preserved(actual): + before = child.source._frame_identity(actual.full._checked()[2].frame) + value = child.qualify_child_property_sources(actual.full) + assert before == child.source._frame_identity(actual.full._checked()[2].frame) + eligible = value.recipients.loc[value.recipients.eligible] + assert len(eligible) >= 2 + assert ( + not eligible.ordinary_source_known.any() + and not eligible.dividend_source_known.any() + ) + assert value.evidence["full_source_rows"] == 4 + assert value.evidence["donor_sampling_fraction_applied"] is False + assert value.evidence["source_admission_issued"] is False + + +def test_copied_preparation_cannot_supply_full_donor_authority(actual): + forged = object.__new__(type(actual.full)) + object.__setattr__(forged, "payload", actual.full.payload) + with pytest.raises(ValueError): + child.qualify_child_property_sources(forged) + actual.full._checked() + + +@pytest.mark.parametrize("when", ["first", "final"]) +@pytest.mark.parametrize( + "mutation", + [ + "constant", + "dependency_constant", + "dividend_evidence_note", + "function", + "defaults", + "closure", + "source_value", + ], +) +def test_initial_and_final_owner_callback_mutations_refuse_current_borrow( + actual, when, mutation +): + target = 1 if when == "first" else 2 + count = 0 + mutated = False + original_profile = sys.getprofile() + original_ages = child.DONOR_AGES + original_unpublished = child.routing.ALLOCATION_CODE_MEANINGS_UNPUBLISHED + original_conflict_note = child.dividend.ALLOCATION_CONFLICT_NOTE + original_function = child._bad_status + original_defaults = child._age.__defaults__ + closure_cell = child.QualifiedChildPropertySources.__repr__.__closure__[0] + original_closure = closure_cell.cell_contents + source_people = actual.full._checked()[2].source_frames[1].person + source_values = source_people.A_AGE.copy(deep=True) + + def profile(frame, event, argument): + nonlocal count, mutated + if ( + event == "return" + and frame.f_code + is child.source.AuthenticatedSurveyPopulationPreparation._checked.__code__ + and frame.f_back is not None + and frame.f_back.f_code is child.qualify_child_property_sources.__code__ + ): + count += 1 + if count == target: + mutated = True + if mutation == "constant": + child.DONOR_AGES = (16, 17) + elif mutation == "dependency_constant": + child.routing.ALLOCATION_CODE_MEANINGS_UNPUBLISHED = frozenset( + {"I_INTYN"} + ) + elif mutation == "dividend_evidence_note": + child.dividend.ALLOCATION_CONFLICT_NOTE = "changed interpretation" + elif mutation == "function": + child._bad_status = lambda value: False + elif mutation == "defaults": + child._age.__defaults__ = ("15",) + elif mutation == "closure": + closure_cell.cell_contents = object() + else: + source_people.loc[:, "A_AGE"] = source_people.A_AGE + 1 + + sys.setprofile(profile) + try: + with pytest.raises(ValueError): + child.qualify_child_property_sources(actual.full) + finally: + sys.setprofile(original_profile) + child.DONOR_AGES = original_ages + child.routing.ALLOCATION_CODE_MEANINGS_UNPUBLISHED = original_unpublished + child.dividend.ALLOCATION_CONFLICT_NOTE = original_conflict_note + child._bad_status = original_function + child._age.__defaults__ = original_defaults + closure_cell.cell_contents = original_closure + source_people.loc[:, "A_AGE"] = source_values + assert mutated + actual.full._checked() + + +def test_detached_donor_mutation_does_not_change_retained_source(actual): + values = child.qualify_child_property_sources(actual.full) + old = child.child_property_sources_seal(values) + values.donor_projection.donors.loc[:, child.TARGETS[0]] += 11 + assert child.child_property_sources_seal(values) != old + new = child.qualify_child_property_sources(actual.full) + assert child.child_property_sources_seal(new) == old + actual.full._checked() + + +@pytest.mark.parametrize("dependency", ["allocation_set", "dividend_note"]) +def test_preexisting_dependency_interpretation_mutation_refuses( + actual, monkeypatch, dependency +): + if dependency == "allocation_set": + monkeypatch.setattr( + child.routing, + "ALLOCATION_CODE_MEANINGS_UNPUBLISHED", + frozenset({"I_INTYN"}), + ) + else: + monkeypatch.setattr( + child.dividend, "ALLOCATION_CONFLICT_NOTE", "changed interpretation" + ) + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + child.qualify_child_property_sources(actual.full) + + +@pytest.mark.parametrize("changes", [{"TRDINT_VAL": "bad"}, {"INT_YN": "2"}]) +def test_malformed_excluded_teenager_is_refused_from_full_source(tmp_path, changes): + with pytest.MonkeyPatch.context() as patch: + _, arguments, donor = source_arguments(tmp_path, patch, donor_changes=changes) + preparation = child.source.prepare_authenticated_survey_population(**arguments) + selected = preparation._checked()[2].frame.person + asec = selected.loc[ + selected[child.routing.support_channel_column("person")].eq("asec") + ] + assert donor not in set(asec[child.routing.spine_source_id_column("person")]) + assert not asec.A_AGE.between(15, 17).any() + with pytest.raises(ValueError, match="DONOR_SOURCE_REVIEW_REQUIRED"): + child.qualify_child_property_sources(preparation) + preparation._checked() + + +@pytest.mark.parametrize("owner", ["catalogue", "preparation"]) +@pytest.mark.parametrize("surface", ["observation", "result"]) +def test_final_owner_callbacks_cannot_change_already_projected_values( + actual, owner, surface +): + preparation = actual.full + catalogue = preparation._checked()[2].catalogues[1] + code = type(catalogue if owner == "catalogue" else preparation)._checked.__code__ + original_profile = sys.getprofile() + calls, mutated = 0, False + + def profile(frame, event, argument): + nonlocal calls, mutated + caller = frame.f_back + if ( + event == "return" + and frame.f_code is code + and caller is not None + and caller.f_code is child.qualify_child_property_sources.__code__ + ): + calls += 1 + if calls == 2: + # These are only the test's invented, detached in-process + # projections. No value, frame or local is emitted to output. + if surface == "observation": + caller.f_locals["observed_interest"].person.loc[ + :, "allocation_origin" + ] = "changed" + else: + caller.f_locals["result"].recipients.loc[:, "reason"] = "changed" + mutated = True + + sys.setprofile(profile) + try: + expected = ( + "FINAL_OBSERVATIONS_CHANGED" + if surface == "observation" + else "FINAL_VALUES_CHANGED" + ) + with pytest.raises(ValueError, match=expected): + child.qualify_child_property_sources(preparation) + finally: + sys.setprofile(original_profile) + assert mutated + preparation._checked() diff --git a/packages/microcosm-build/tests/test_us_common_frame_export_contract.py b/packages/microcosm-build/tests/test_us_common_frame_export_contract.py new file mode 100644 index 000000000..349e78527 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_common_frame_export_contract.py @@ -0,0 +1,405 @@ +"""Invented supplied-parent comparisons; no calibration or source issuer imitation.""" + +import json + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import frame_checkpoint +from microcosm.build.us_runtime import common_frame_export_contract as contract +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + +def _parent(): + person = pd.DataFrame( + { + "person_id": np.arange(1, 7, dtype=np.int64), + "person_household_id": np.array([10, 10, 20, 20, 30, 30], dtype=np.int64), + "person_tax_unit_id": np.array( + [100, 100, 200, 201, 300, 300], dtype=np.int64 + ), + "money": np.array([-0.0, 2.0, 3.0, 4.0, 5.0, 6.0]), + "hours": pd.Series([10, pd.NA, 30, 40, pd.NA, 60], dtype="Int32"), + "reported": pd.Series( + [True, pd.NA, False, True, pd.NA, False], dtype="boolean" + ), + "source_person_id": np.arange(101, 107, dtype=np.int64), + } + ) + household = pd.DataFrame( + { + "household_id": np.array([10, 20, 30], dtype=np.int64), + "origin": pd.Series(["asec", "acs", "acs"], dtype="string"), + "assigned_block": pd.Series(["001", "002", "003"], dtype="string"), + "county": pd.Series(["A", "A", "B"], dtype="string"), + "clone": np.array([0, 1, 0], dtype=np.int8), + } + ) + tax = pd.DataFrame( + { + "tax_unit_id": np.array([100, 200, 201, 300], dtype=np.int64), + "deduction": np.array([1.0, 2.0, 3.0, 4.0]), + } + ) + return Frame( + {"person": person, "household": household, "tax_unit": tax}, + EntitySchema(group_entities=("household", "tax_unit")), + {"household": Weights(np.array([1.0, 2.0, 3.0]), WeightKind.IMPORTANCE)}, + pd.Series(["a", "a", "b", "b", "b", "b"], name="stratum"), + metadata={"construction": "one-common-parent"}, + ) + + +def _weighted(parent, person_mask, weights): + selected = parent.select(np.asarray(person_mask, dtype=np.bool_)) + return Frame( + {e: selected.table(e).copy(deep=True) for e in selected.entities}, + selected.schema, + { + "household": Weights( + np.asarray(weights, dtype=np.float64), WeightKind.CALIBRATED + ) + }, + selected.strata.copy(), + metadata=selected.metadata, + ) + + +def _arguments(parent): + return { + "parent_reference": "supplied-parent-example-not-an-issued-graph-key", + "ordered_household_ids": parent.table("household") + .household_id.to_numpy() + .copy(), + "calibrated_weights": np.array([0.0, 9.0, 10.0]), + "calibration_specification": b'{"targets":"invented","scope":"national"}', + } + + +@pytest.mark.parametrize("view", ["full", "pruned", "local"]) +def test_one_parent_full_pruned_and_local_preserve_all_inputs(view): + parent = _parent() + args = _arguments(parent) + if view == "full": + candidate = _weighted(parent, [True] * 6, [0, 9, 10]) + args["prune_zero_weight"] = False + elif view == "pruned": + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + else: + candidate = _weighted(parent, [False, False, False, False, True, True], [10]) + args["scope_household_ids"] = np.array([30], dtype=np.int64) + binding = contract.verify_retained_frame_export(parent, candidate, **args) + report = json.loads(binding) + assert report["actual_graph_calibration_ancestry_verified"] is False + assert report["release_eligible"] is False + assert report["retained_entity_ids"]["person"]["rows"] == candidate.n("person") + if view != "full": + # A complete survey origin may legitimately have zero calibrated weight. + assert candidate.table("household").origin.tolist() == ["acs"] * candidate.n( + "household" + ) + assert ( + contract.verify_retained_frame_export( + parent, candidate, expected_binding=binding, **args + ) + == binding + ) + + +def test_actual_checkpoint_readback_compares_retained_values_and_masks(tmp_path): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + before = contract.verify_retained_frame_export(parent, candidate, **args) + path = tmp_path / "invented-export.h5" + frame_checkpoint.write_frame_checkpoint( + path, candidate, metadata={"comparison": before.decode()} + ) + loaded = frame_checkpoint.load_frame_checkpoint( + path, frame_metadata=parent.metadata + ) + assert loaded.metadata["comparison"] == before.decode() + assert ( + contract.verify_retained_frame_export( + parent, + loaded.frame, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + == before + ) + loaded.frame.person.loc[loaded.frame.person.index[0], "money"] += 1 + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.money" + ): + contract.verify_retained_frame_export( + parent, + loaded.frame, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + + +@pytest.mark.parametrize( + "change", + [ + "value", + "mask", + "membership", + "id", + "geography", + "lineage", + "weight", + "strata", + "column", + ], +) +def test_retained_input_changes_refuse(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + if change == "value": + candidate.person.loc[2, "money"] += 1 + elif change == "mask": + candidate.person.loc[4, "hours"] = 0 + elif change == "membership": + candidate.person.loc[2, "person_household_id"] = 30 + elif change == "id": + candidate.person.loc[2, "person_id"] = 99 + elif change == "geography": + candidate.table("household").loc[0, "assigned_block"] = "004" + elif change == "lineage": + candidate.person.loc[2, "source_person_id"] = 999 + elif change == "weight": + candidate = _weighted(parent, [False, False, True, True, True, True], [8, 10]) + elif change == "strata": + candidate.strata.iloc[0] = "changed" + else: + candidate.person.drop(columns="money", inplace=True) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_|COLUMN_ROSTER" + ): + contract.verify_retained_frame_export(parent, candidate, **_arguments(parent)) + + +def test_partial_household_selection_refuses_even_with_valid_frame_links(): + parent = _parent() + candidate = _weighted(parent, [False, False, True, False, True, True], [9, 10]) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_ENTITY_IDS:person" + ): + contract.verify_retained_frame_export(parent, candidate, **_arguments(parent)) + + +@pytest.mark.parametrize( + "change", + ["order", "dtype", "nan", "negative", "shape", "scope_order", "scope_unknown"], +) +def test_weight_and_scope_axes_are_explicit(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + if change == "order": + args["ordered_household_ids"] = args["ordered_household_ids"][::-1] + elif change == "dtype": + args["ordered_household_ids"] = args["ordered_household_ids"].astype(float) + elif change == "nan": + args["calibrated_weights"][0] = np.nan + elif change == "negative": + args["calibrated_weights"][0] = -1 + elif change == "shape": + args["calibrated_weights"] = args["calibrated_weights"][:2] + elif change == "scope_order": + args["scope_household_ids"] = np.array([30, 20], dtype=np.int64) + else: + args["scope_household_ids"] = np.array([99], dtype=np.int64) + with pytest.raises(contract.RetainedFrameExportError): + contract.verify_retained_frame_export(parent, candidate, **args) + + +@pytest.mark.parametrize( + "change", ["specification", "parent_reference", "excluded_weight", "scope", "prune"] +) +def test_readback_binding_covers_even_inputs_outside_retained_rows(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + args["scope_household_ids"] = np.array([20, 30], dtype=np.int64) + before = contract.verify_retained_frame_export(parent, candidate, **args) + if change == "specification": + args["calibration_specification"] = b'{"targets":"different"}' + elif change == "parent_reference": + args["parent_reference"] = "another-unverified-reference" + elif change == "excluded_weight": + args["calibrated_weights"][0] = 100 + elif change == "scope": + args["scope_household_ids"] = None + else: + args["prune_zero_weight"] = False + with pytest.raises(contract.RetainedFrameExportError, match="EXPORT_BINDING"): + contract.verify_retained_frame_export( + parent, candidate, expected_binding=before, **args + ) + + +@pytest.mark.parametrize("scope", ["empty", "all_zero"]) +def test_empty_analysis_is_an_explicit_unsupported_frame_contract(scope): + parent = _parent() + candidate = _weighted(parent, [True] * 6, [0, 9, 10]) + args = _arguments(parent) + if scope == "empty": + args["scope_household_ids"] = np.array([], dtype=np.int64) + else: + args["calibrated_weights"] = np.zeros(3, dtype=np.float64) + with pytest.raises(contract.RetainedFrameExportError, match="EMPTY_EXPORT_SUPPORT"): + contract.verify_retained_frame_export(parent, candidate, **args) + + +def test_uint64_stable_ids_above_int64_are_not_coerced(): + ids = np.array([2**63 + 1, 2**63 + 7], dtype=np.uint64) + parent = Frame( + { + "person": pd.DataFrame( + { + "person_id": np.array([1, 2], dtype=np.int32), + "person_household_id": ids, + } + ), + "household": pd.DataFrame({"household_id": ids}), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.ones(2), WeightKind.IMPORTANCE)}, + ) + candidate = _weighted(parent, [True, True], [2, 3]) + args = { + "parent_reference": "large-ids", + "ordered_household_ids": ids, + "calibrated_weights": np.array([2.0, 3.0]), + "calibration_specification": b"{}", + } + report = json.loads( + contract.verify_retained_frame_export(parent, candidate, **args) + ) + assert report["complete_ordered_household_ids"]["dtype"] == ids.dtype.str + with pytest.raises(contract.RetainedFrameExportError, match="HOUSEHOLD_ID_VECTOR"): + contract.verify_retained_frame_export( + parent, candidate, **{**args, "ordered_household_ids": ids.astype(float)} + ) + + +def test_readback_normalization_is_closed_and_never_changes_knownness(): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + before = contract.verify_retained_frame_export(parent, candidate, **args) + values = candidate.person["hours"].array + assert values._mask[2] + values._data[2] ^= np.int32(1) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.hours" + ): + contract.verify_retained_frame_export( + parent, candidate, expected_binding=before, **args + ) + assert ( + contract.verify_retained_frame_export( + parent, + candidate, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + == before + ) + with pytest.raises(contract.RetainedFrameExportError, match="COMPARISON_MODE"): + contract.verify_retained_frame_export( + parent, candidate, comparison="approximate", **args + ) + values._mask[2] = False + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.hours" + ): + contract.verify_retained_frame_export( + parent, candidate, comparison="frame-checkpoint-readback", **args + ) + + +@pytest.mark.parametrize("prune_zero_weight", [True, False]) +@pytest.mark.parametrize("scope", ["all", "zero_weight_household"]) +def test_zero_weight_export_support_refuses(prune_zero_weight, scope): + parent = _parent() + # Keep a valid candidate so the export contract itself checks the supplied + # weight/scope inputs before comparing retained rows or weight storage. + candidate = _weighted(parent, [True] * 6, [1, 1, 1]) + args = _arguments(parent) + args["calibrated_weights"] = np.zeros(3, dtype=np.float64) + if scope == "zero_weight_household": + args["calibrated_weights"][1:] = [9, 10] + args["scope_household_ids"] = args["ordered_household_ids"][:1].copy() + with pytest.raises(contract.RetainedFrameExportError, match="EMPTY_EXPORT_SUPPORT"): + contract.verify_retained_frame_export( + parent, candidate, prune_zero_weight=prune_zero_weight, **args + ) + + +@pytest.mark.parametrize("prune_zero_weight", [True, False]) +def test_positive_calibrated_scope_can_have_zero_parent_weight(prune_zero_weight): + baseline = _parent() + parent = Frame( + {e: baseline.table(e).copy(deep=True) for e in baseline.entities}, + baseline.schema, + { + "household": Weights(np.array([0.0, 2.0, 3.0]), WeightKind.IMPORTANCE), + "tax_unit": Weights(np.array([7.0, 8.0, 9.0, 10.0]), WeightKind.DESIGN), + }, + baseline.strata.copy(), + metadata=baseline.metadata, + ) + before_tables = {e: parent.table(e).copy(deep=True) for e in parent.entities} + before_strata = parent.strata.copy(deep=True) + before_metadata, before_mass_log = parent.metadata, parent.mass_log + before_weights = { + e: (parent.weights_for(e).values.copy(), parent.weights_for(e).kind) + for e in parent.weighted_entities + } + retained = baseline.select(np.array([True, True, False, False, False, False])) + + def candidate(tax_weight): + return Frame( + {e: retained.table(e).copy(deep=True) for e in retained.entities}, + retained.schema, + { + "household": Weights(np.array([4.0]), WeightKind.CALIBRATED), + "tax_unit": Weights(np.array([tax_weight]), WeightKind.DESIGN), + }, + retained.strata.copy(), + metadata=retained.metadata, + ) + + args = _arguments(parent) + args["calibrated_weights"] = np.array([4.0, 0.0, 0.0]) + args["scope_household_ids"] = np.array([10], dtype=np.int64) + report = json.loads( + contract.verify_retained_frame_export( + parent, candidate(7.0), prune_zero_weight=prune_zero_weight, **args + ) + ) + assert report["actual_graph_calibration_ancestry_verified"] is False + assert report["release_eligible"] is False + assert report["complete_ordered_household_ids"]["rows"] == 3 + assert report["retained_entity_ids"]["household"]["rows"] == 1 + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_WEIGHTS:tax_unit" + ): + contract.verify_retained_frame_export( + parent, candidate(8.0), prune_zero_weight=prune_zero_weight, **args + ) + for entity, table in before_tables.items(): + pd.testing.assert_frame_equal(parent.table(entity), table, check_exact=True) + pd.testing.assert_series_equal(parent.strata, before_strata, check_exact=True) + assert parent.metadata == before_metadata and parent.mass_log == before_mass_log + for entity, (values, kind) in before_weights.items(): + assert parent.weights_for(entity).kind is kind + assert parent.weights_for(entity).values.tobytes() == values.tobytes() diff --git a/packages/microcosm-build/tests/test_us_completion_support_custody.py b/packages/microcosm-build/tests/test_us_completion_support_custody.py new file mode 100644 index 000000000..b56b37e09 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_completion_support_custody.py @@ -0,0 +1,657 @@ +"""Small graph/Frame custody controls, without source or run issuance. + +The private boundary shells below contain real declarations and compiled +versions solely to test the custody helpers. They cannot pass pure(), acquire +preparation authority, or issue a financial handle. Actual-owner cold/required +and permanent-revocation controls remain in test_us_graph_atomic_completion_host. +""" + +from dataclasses import replace +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_child_property_income_graph import ( + ORDER_TYPE, + _options, + _ordering, + _qualified, + _receiving_frame, +) + +from microcosm.build.us_runtime import graph_atomic_survey_financial as host +from microcosm.build.us_runtime import graph_child_property_income as child +from microcosm.build.us_runtime import graph_survey_completion_host as extension +from microcosm.frame import ( + US_GROUP_ENTITIES, + US_SCHEMA, + EntitySchema, + Frame, + MassChangeRecord, + WeightKind, + Weights, +) +from microcosm.graph import ( + ArtifactOutput, + Graph, + Node, + Owned, + SourceRef, + StructuralDelta, + WeightTransition, + compile_graph, +) +from microcosm.graph.manifest import PopulationView, RunManifest +from microcosm.graph.population import MassRecord, Population, token_for_dtype + + +def _full_us_parent_frame(original): + """Extend only this invented parent; private child support stays unchanged.""" + tables_before = { + entity: original.table(entity).copy(deep=True) for entity in original.entities + } + weights = { + entity: original.weights_for(entity) for entity in original.weighted_entities + } + weight_bytes = { + entity: weight.values.tobytes() for entity, weight in weights.items() + } + strata_before = original.strata.copy(deep=True) + metadata_before, mass_log_before = original.metadata, original.mass_log + nullable_before = { + (entity, column): (series.array._data.tobytes(), series.array._mask.tobytes()) + for entity in original.entities + for column, series in original.table(entity).items() + if isinstance(series.dtype, pd.BooleanDtype) + } + assert set(original.schema.group_entities) <= set(US_GROUP_ENTITIES) + assert original.schema.person_entity == US_SCHEMA.person_entity + assert not original.links and not original._link_tables + missing = tuple( + group + for group in US_GROUP_ENTITIES + if group not in original.schema.group_entities + ) + tables = {entity: table.copy(deep=True) for entity, table in tables_before.items()} + people = tables[US_SCHEMA.person_entity] + household_membership = US_SCHEMA.membership_column("household") + household_ids = original.table("household")[US_SCHEMA.entity_id_column("household")] + sentinels = {} + for group in missing: + membership = US_SCHEMA.membership_column(group) + assert membership not in people.columns + # One invented group per existing cloned household, without assigning + # actual SPM/family/marital roles or changing any original identifier. + people[membership] = original.person[household_membership].copy(deep=True) + # The actual child graph requires a nonempty Slice for each entity. + # These invented cells carry no survey or policy meaning. + sentinel_column = f"invented_{group}_custody_sentinel" + assert all( + sentinel_column not in table.columns for table in tables_before.values() + ) + sentinels[group] = pd.Series( + 1, index=household_ids.index, dtype="int64", name=sentinel_column + ) + tables[group] = pd.DataFrame( + { + US_SCHEMA.entity_id_column(group): household_ids.copy(deep=True), + sentinel_column: sentinels[group].copy(deep=True), + } + ) + full = Frame( + tables, + US_SCHEMA, + weights, + strata=original.strata, + mass_log=original.mass_log, + # Generic fixture metadata is not a country-authority receipt. Admit an + # empty context only on this ordinary US parent; retain the input below. + metadata={}, + ) + + assert full.schema is US_SCHEMA and full.entities == US_SCHEMA.entities + assert set(full._tables) == set(full.entities) + assert not full.links and not full._link_tables + assert full.weighted_entities == original.weighted_entities + assert set(full._weights) == set(weights) + for entity, before in tables_before.items(): + pd.testing.assert_frame_equal(original.table(entity), before, check_exact=True) + pd.testing.assert_frame_equal( + full.table(entity).loc[:, before.columns], before, check_exact=True + ) + for (entity, column), backing in nullable_before.items(): + for frame in (original, full): + array = frame.table(entity)[column].array + assert (array._data.tobytes(), array._mask.tobytes()) == backing + assert list(full.person.columns) == [ + *original.person.columns, + *(US_SCHEMA.membership_column(group) for group in missing), + ] + for group in missing: + id_column = US_SCHEMA.entity_id_column(group) + sentinel = sentinels[group] + assert sentinel.dtype == "int64" and sentinel.eq(1).all() + assert list(full.table(group).columns) == [id_column, sentinel.name] + for table in (tables[group], full.table(group)): + pd.testing.assert_series_equal( + table[sentinel.name], sentinel, check_exact=True + ) + pd.testing.assert_series_equal( + full.table(group)[id_column], + household_ids.rename(id_column), + check_exact=True, + ) + pd.testing.assert_series_equal( + full.person[US_SCHEMA.membership_column(group)], + original.person[household_membership].rename( + US_SCHEMA.membership_column(group) + ), + check_exact=True, + ) + for entity, weight in weights.items(): + assert full.weights_for(entity) is original.weights_for(entity) is weight + assert weight.values.tobytes() == weight_bytes[entity] + assert full.weights_for(entity).kind is weight.kind + for frame in (original, full): + pd.testing.assert_series_equal(frame.strata, strata_before, check_exact=True) + assert frame.mass_log == mass_log_before + assert len(full.metadata) == 0 + assert original.metadata == metadata_before + assert original.metadata is metadata_before and original.mass_log is mass_log_before + return full + + +def _case(): + qualified, origins = _qualified() + parent = Population.from_frame( + _full_us_parent_frame(_receiving_frame(qualified, origins)), "test.allocated" + ) + edge, _, pins = _ordering() + nodes = child.child_property_nodes( + qualified, + origins, + parent, + options=_options(), + host_edges=(edge,), + host_pins=pins, + ) + frame = parent.frame + create = Node( + "test.parent", + "test.child.receiving@1", + sources=(child.SOURCE_NAME,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned(entity, column, token_for_dtype(frame.table(entity)[column].dtype)) + for entity in frame.entities + for column in frame.table(entity) + if column != frame.schema.entity_id_column(entity) + and not ( + entity == frame.schema.person_entity + and column + in { + frame.schema.membership_column(group) + for group in frame.schema.group_entities + } + ) + ), + artifact_outputs=(ArtifactOutput("ordering", ORDER_TYPE),), + ) + allocated = Node( + parent.version, + "test.child.allocate@1", + base=create.id, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "importance", "conserve"), + mass="conserve", + ) + compiled = compile_graph( + Graph( + "us", + (SourceRef(child.SOURCE_NAME, "frame-store"),), + (create, allocated, *nodes), + ) + ) + donor, recipient, *_ = child._projections(qualified, origins) + supports = { + node_id: Population.from_frame(child._support_frame(table), node_id) + for node_id, table in ((child.DONOR, donor), (child.RECIPIENT, recipient)) + } + # Descriptive shells exercise dispatch only; no monkeypatched owner methods. + boundary = object.__new__(extension._CompletionHost) + boundary.child = object.__new__(child.ChildPropertyBoundary) + boundary.child.nodes = nodes + boundary.child.declaration = tuple(node.normative() for node in nodes) + boundary.child.revoked = boundary.revoked = False + boundary.compiled = compiled + populations = { + node_id: supports[version] + if version in supports + else replace(parent, version=version) + for node_id, version in ((n, compiled.versions[n]) for n in compiled.order) + } + manifest = RunManifest( + "us", + {}, + populations={v: populations[v].frame for v in set(compiled.versions.values())}, + mass_ledgers={ + v: populations[v].mass_ledger for v in set(compiled.versions.values()) + }, + ) + return SimpleNamespace( + parent=parent, + compiled=compiled, + boundary=boundary, + populations=populations, + manifest=manifest, + ) + + +def _stamp(case, node_id): + return extension._population_stamp( + case.boundary, case.compiled, node_id, case.populations[node_id] + ) + + +@pytest.mark.parametrize( + "node_id", [child.DONOR, child.RECIPIENT, child.FIT, child.DRAW] +) +def test_four_declared_support_nodes_preserve_actual_population_context(node_id): + case = _case() + actual = case.populations[node_id] + frame = actual.frame + before = child.child.physical._population_stamp(actual) + seal = _stamp(case, node_id) + assert seal == (child._support_frame_stamp(frame), before) + assert actual.frame is frame and _stamp(case, node_id) == seal + assert actual.version == ( + child.DONOR if node_id in (child.DONOR, child.FIT) else child.RECIPIENT + ) + assert actual.design_weights and actual.owners + assert frame.schema == EntitySchema(group_entities=("household",)) + with pytest.raises(ValueError, match="^FRAME_TYPE$"): + host.reconstruction._population_stamp(actual) + + +@pytest.mark.parametrize( + "node_id", ["test.parent", "test.allocated", child.ATTACH, child.VERIFY] +) +def test_regular_nodes_keep_identical_us_seal_and_reject_private_frames(node_id): + case = _case() + actual = case.populations[node_id] + assert _stamp(case, node_id) == host.reconstruction._population_stamp(actual) + assert host._node_population_stamp(case.compiled, node_id, actual) == _stamp( + case, node_id + ) + actual.frame._metadata = { + **actual.frame.metadata, + "private_model_support": child.PROTOCOL, + } + metadata_reason = ( + r"^US graph context has undeclared metadata: \['private_model_support'\]\.$" + ) + with pytest.raises(ValueError, match=metadata_reason): + _stamp(case, node_id) + with pytest.raises(ValueError, match=metadata_reason): + host._node_population_stamp(case.compiled, node_id, actual) + wrong = replace(case.populations[child.DONOR], version=actual.version) + with pytest.raises(ValueError, match="^FRAME_TYPE$"): + extension._population_stamp(case.boundary, case.compiled, node_id, wrong) + + +@pytest.mark.parametrize( + "fault,reason", + [ + ("duck_boundary", "COMPLETION_STAMP_BOUNDARY"), + ("duck_child", "COMPLETION_STAMP_BOUNDARY"), + ("copied_compiled", "COMPLETION_STAMP_BOUNDARY"), + ("unknown_node", "COMPLETION_STAMP_VERSION"), + ("wrong_version", "COMPLETION_STAMP_VERSION"), + ("changed_declaration", "COMPLETION_SUPPORT_DECLARATION"), + ("changed_bound_nodes", "COMPLETION_SUPPORT_DECLARATION"), + ("forged_version_map", "COMPLETION_SUPPORT_VERSION"), + ("empty_nodes", "COMPLETION_FRAGMENT_ROSTER"), + ], +) +def test_private_route_requires_exact_boundary_declarations_and_versions(fault, reason): + case = _case() + boundary, compiled, node_id, population = ( + case.boundary, + case.compiled, + child.FIT, + case.populations[child.FIT], + ) + if fault == "duck_boundary": + boundary = SimpleNamespace(**vars(boundary)) + elif fault == "duck_child": + boundary.child = SimpleNamespace( + nodes=boundary.child.nodes, declaration=boundary.child.declaration + ) + elif fault == "copied_compiled": + compiled = replace(compiled) + elif fault == "unknown_node": + node_id = "test.unknown" + elif fault == "wrong_version": + population = replace(population, version=child.RECIPIENT) + elif fault == "changed_declaration": + boundary.child.declaration = () + elif fault == "changed_bound_nodes": + nodes = tuple( + replace(n, description="changed retained declaration") + if n.id == child.FIT + else n + for n in boundary.child.nodes + ) + # Description is deliberately non-normative; compare actual declaration + # objects too, so equality cannot be replaced with an opaque JSON label. + assert tuple(n.normative() for n in nodes) == boundary.child.declaration + boundary.child.nodes = nodes + elif fault == "forged_version_map": + compiled = replace( + compiled, versions={**compiled.versions, child.FIT: child.RECIPIENT} + ) + boundary.compiled = compiled + population = replace(population, version=child.RECIPIENT) + else: + boundary.child.nodes = () + with pytest.raises(ValueError, match="^CURRENT_SURVEY_PREDICTOR_" + reason + "$"): + extension._population_stamp(boundary, compiled, node_id, population) + + +@pytest.mark.parametrize( + "surface", + [ + "person_cell", + "household_cell", + "nullable_backing", + "frame_weight", + "strata", + "metadata", + "mass_log", + "owners", + "weight_kind", + "design_weights", + "mass_ledger", + ], +) +def test_full_support_custody_covers_frame_and_actual_population_metadata(surface): + case = _case() + population = case.populations[child.DONOR] + frame = population.frame + if surface == "nullable_backing": + frame.person["invented_missing"] = pd.array( + [None] * frame.n("person"), dtype="Int64" + ) + population = Population.from_frame(frame, population.version) + case.populations[child.DONOR] = population + before = _stamp(case, child.DONOR) + if surface == "person_cell": + frame.person.loc[0, "source_year"] += 1 + elif surface == "household_cell": + frame.table("household").loc[0, "household_id"] += 1 + elif surface == "nullable_backing": + array = frame.person.invented_missing.array + assert array._mask[0] + array._data[0] += 1 + assert pd.isna(frame.person.invented_missing.iloc[0]) + elif surface == "frame_weight": + frame._weights["household"] = Weights( + frame.weights_for("household").values + 1, WeightKind.DESIGN + ) + elif surface == "strata": + frame.strata.iloc[0] = "changed" + elif surface == "metadata": + frame._metadata = {**frame.metadata, "changed": True} + elif surface == "mass_log": + frame._mass_log = (MassChangeRecord("household", 1.0, 2.0, 2.0, "test"),) + elif surface == "owners": + owners = dict(population.owners) + owners["person", "source_year"] = "invented.other.owner" + object.__setattr__(population, "owners", owners) + elif surface == "weight_kind": + object.__setattr__( + population, "weight_kind", {"household": WeightKind.IMPORTANCE} + ) + elif surface == "design_weights": + object.__setattr__( + population, + "design_weights", + {"household": population.design_weights["household"] + 1}, + ) + else: + record = MassRecord( + "invented.filter", + "filter", + "conserve", + 4.0, + 4.0, + (), + (), + entity="household", + ) + object.__setattr__(population, "mass_ledger", (record,)) + after = _stamp(case, child.DONOR) + assert after != before + if surface in ("owners", "weight_kind", "design_weights", "mass_ledger"): + assert after[0] == before[0] # Frame-only stamp would miss the change. + assert after[1] != before[1] + + +@pytest.mark.parametrize( + "surface", ["extra_table", "extra_weight", "hidden_link", "wrong_schema"] +) +def test_hidden_support_content_is_refused_before_stamping(surface): + case = _case() + frame = case.populations[child.DONOR].frame + if surface == "extra_table": + frame._tables["unexpected"] = pd.DataFrame({"unexpected_id": [1]}) + elif surface == "extra_weight": + frame._weights["unexpected"] = Weights([1.0], WeightKind.DESIGN) + elif surface == "hidden_link": + frame._link_tables["unexpected"] = pd.DataFrame({"person_id": [1]}) + else: + frame._schema = EntitySchema(group_entities=("household", "tax_unit")) + with pytest.raises(ValueError, match="SUPPORT_FRAME_TYPE$"): + _stamp(case, child.DONOR) + + +@pytest.mark.parametrize("fault", ["empty", "missing", "extra", "reordered"]) +def test_completion_retained_roster_must_match_compiled_order(fault): + case = _case() + observations = dict(case.populations) + expected = host._node_population_seals(case.compiled, observations, case.boundary) + assert len(expected) == len(case.compiled.order) + assert all( + p is observations[n] + for n, (p, _) in zip(case.compiled.order, expected, strict=True) + ) + if fault == "empty": + observations.clear() + elif fault == "missing": + observations.pop(child.FIT) + elif fault == "extra": + observations["unexpected"] = observations[child.DONOR] + else: + observations = dict(reversed(tuple(observations.items()))) + with pytest.raises( + ValueError, match="^CURRENT_SURVEY_PREDICTOR_FINANCIAL_NODE_POPULATION_ROSTER$" + ): + host._node_population_seals(case.compiled, observations, case.boundary) + assert host._node_population_seals(case.compiled, None) == () + + +def test_refused_observation_atomically_revokes_and_cannot_be_reused(): + case = _case() + observed, stamps = {}, {} + valid = case.populations[child.DONOR] + wrong = replace(valid, version=child.RECIPIENT) + with pytest.raises( + ValueError, match="^CURRENT_SURVEY_PREDICTOR_COMPLETION_STAMP_VERSION$" + ): + extension._observe_population( + case.boundary, case.compiled, observed, stamps, child.DONOR, wrong + ) + assert observed == stamps == {} + assert case.boundary.revoked and case.boundary.child.revoked + with pytest.raises( + ValueError, match="^CURRENT_SURVEY_PREDICTOR_COMPLETION_STAMP_BOUNDARY$" + ): + extension._observe_population( + case.boundary, case.compiled, observed, stamps, child.DONOR, valid + ) + + +def test_duplicate_observation_keeps_original_custody_and_revokes(): + case = _case() + observed, stamps = {}, {} + actual = case.populations[child.DONOR] + extension._observe_population( + case.boundary, case.compiled, observed, stamps, child.DONOR, actual + ) + saved = dict(stamps) + with pytest.raises( + ValueError, match="^CURRENT_SURVEY_PREDICTOR_COMPLETION_OBSERVER_DUPLICATE$" + ): + extension._observe_population( + case.boundary, case.compiled, observed, stamps, child.DONOR, replace(actual) + ) + assert observed[child.DONOR] is actual and stamps == saved + assert case.boundary.revoked and case.boundary.child.revoked + + +def test_new_dispatch_and_binding_methods_are_in_existing_live_fence(monkeypatch): + baseline = extension._live(False) + assert (extension.__name__, "_population_stamp") in baseline + assert (extension.__name__, "_observe_population") in baseline + assert (extension.__name__, "_CompletionHost", "bind_compiled") in baseline + with monkeypatch.context() as patch: + patch.setattr(extension, "_population_stamp", lambda *args: "forged") + assert extension._live(False) != baseline + assert extension._live(False) == baseline + + +def test_manifest_support_admission_preserves_every_slot_and_actual_metadata(): + case = _case() + frame = case.manifest.population(child.DONOR) + assert type(frame) is PopulationView + admitted = extension._manifest_support_frame(frame) + assert type(admitted) is Frame + assert all( + getattr(admitted, name) is getattr(frame, name) for name in Frame.__slots__ + ) + actual = Population.from_frame( + frame, child.DONOR, mass_ledger=case.manifest.mass_ledger(child.DONOR) + ) + seal = extension._population_stamp( + case.boundary, case.compiled, child.DONOR, actual, manifest=True + ) + assert seal == ( + child._support_frame_stamp(admitted), + child.child.physical._population_stamp(actual), + ) + assert actual.frame is frame + with pytest.raises(ValueError, match="SUPPORT_FRAME_TYPE$"): + extension._population_stamp(case.boundary, case.compiled, child.DONOR, actual) + # No schema/class widening leaks into the default survey-only manifest path. + with pytest.raises(ValueError, match="^FRAME_TYPE$"): + host._manifest_population_seals(case.manifest, case.compiled) + seals = host._manifest_population_seals( + case.manifest, case.compiled, completion_boundary=case.boundary + ) + assert dict(seals)[child.DONOR] == seal + assert tuple(n for n, _ in seals) == tuple( + sorted(set(case.compiled.versions.values())) + ) + assert set(n for n, seal in seals if isinstance(seal, tuple)) == { + child.DONOR, + child.RECIPIENT, + } + + +@pytest.mark.parametrize("kind", ["plain_frame", "frame_subclass", "view_subclass"]) +def test_manifest_inverse_view_refuses_every_other_frame_type(kind): + case = _case() + frame = case.populations[child.DONOR].frame + if kind != "plain_frame": + cls = type( + "InventedFrameSubclass", + (Frame if kind == "frame_subclass" else PopulationView,), + {"__slots__": ()}, + ) + candidate = object.__new__(cls) + for slot in Frame.__slots__: + object.__setattr__(candidate, slot, getattr(frame, slot)) + frame = candidate + with pytest.raises( + ValueError, match="^CURRENT_SURVEY_PREDICTOR_COMPLETION_MANIFEST_SUPPORT_VIEW$" + ): + extension._manifest_support_frame(frame) + if kind == "frame_subclass": + with pytest.raises(ValueError, match="SUPPORT_FRAME_TYPE$"): + extension._population_stamp( + case.boundary, + case.compiled, + child.DONOR, + replace(case.populations[child.DONOR], frame=frame), + ) + + +@pytest.mark.parametrize( + "surface", + ["person_cell", "metadata", "ledger", "extra_table", "extra_weight", "hidden_link"], +) +def test_attached_manifest_custody_covers_content_ledger_and_hidden_rosters(surface): + case = _case() + before = host._manifest_population_seals( + case.manifest, case.compiled, completion_boundary=case.boundary + ) + frame = case.manifest.population(child.RECIPIENT) + if surface == "person_cell": + frame.person.loc[0, "source_year"] += 1 + elif surface == "metadata": + frame._metadata = {**frame.metadata, "changed": True} + elif surface == "ledger": + record = MassRecord( + "invented.filter", + "filter", + "conserve", + 3.0, + 3.0, + (), + (), + entity="household", + ) + case.manifest = replace( + case.manifest, + mass_ledgers={**case.manifest.mass_ledgers, child.RECIPIENT: (record,)}, + ) + elif surface == "extra_table": + frame._tables["hidden"] = pd.DataFrame({"hidden_id": [1]}) + elif surface == "extra_weight": + frame._weights["hidden"] = Weights([1.0], WeightKind.DESIGN) + else: + frame._link_tables["hidden"] = pd.DataFrame({"person_id": [1]}) + if surface in ("extra_table", "extra_weight", "hidden_link"): + with pytest.raises(ValueError, match="SUPPORT_FRAME_TYPE$"): + host._manifest_population_seals( + case.manifest, case.compiled, completion_boundary=case.boundary + ) + else: + after = host._manifest_population_seals( + case.manifest, case.compiled, completion_boundary=case.boundary + ) + assert dict(after)[child.RECIPIENT] != dict(before)[child.RECIPIENT] + assert all( + seal == dict(before)[version] + for version, seal in after + if version != child.RECIPIENT + ) + + +def test_inverse_view_slot_contract_is_live_sealed(monkeypatch): + before = extension._live(False) + assert (extension.__name__, "_manifest_support_frame") in before + with monkeypatch.context() as patch: + patch.setattr(Frame, "__slots__", Frame.__slots__[:-1]) + assert extension._live(False) != before + assert extension._live(False) == before diff --git a/packages/microcosm-build/tests/test_us_csv_record_scanner.py b/packages/microcosm-build/tests/test_us_csv_record_scanner.py new file mode 100644 index 000000000..d2c3ee44c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_csv_record_scanner.py @@ -0,0 +1,218 @@ +"""Differential raw-byte framing tests; no data/source authority is minted.""" + +import itertools +import random + +import pytest + +from microcosm.build.us_runtime import asec_coverage_authentication as coverage + + +class _ReferenceBounds(coverage._CsvBounds): + # Original bcd6f0d97 byte loop, with module globals explicitly qualified. + def feed(self, chunk): + for byte in chunk: + if self.after_cr and byte == 10: + self.after_cr = False + continue + self.after_cr = False + self.row += 1 + self.field += 1 + coverage._require( + self.row + <= (coverage._CSV_HEADER_MAX if self.header else coverage._ROW_MAX), + "COVERAGE_CSV_RECORD_BYTES", + ) + coverage._require( + self.field <= coverage._TOKEN_MAX, "COVERAGE_CSV_TOKEN_BYTES" + ) + if self.header: + self.header_bytes.append(byte) + else: + if self.row == 1: + # Row length + six integers + three lengths + fixed native + # key + a conservative 32 bytes for any closed status. + self.budget[0] -= 4 + coverage._NUMBERS.size + 12 + 22 + 32 + if self.column == self.token_column: + self.budget[0] -= 1 + coverage._require(self.budget[0] >= 0, "COVERAGE_BODY_BYTES") + if self.quoted: + if byte == 34: + self.quoted, self.after_quote = False, True + elif self.after_quote and byte == 34: + self.quoted, self.after_quote = True, False + elif byte in (10, 13): + if self.header: + self._end_header() + self.row = self.field = 0 + self.column = 0 + self.header = self.after_quote = False + self.start = True + self.after_cr = byte == 13 + elif byte == 44: + self.field = 0 + self.column += 1 + self.start, self.after_quote = True, False + else: + self.quoted = self.start and byte == 34 + self.start = self.after_quote = False + + +def _snapshot(guard): + return tuple( + getattr(guard, name) + for name in ( + "row", + "field", + "header", + "start", + "quoted", + "after_quote", + "after_cr", + "column", + "token_column", + ) + ) + (bytes(guard.header_bytes), guard.budget[0]) + + +def _feed(guard, chunk): + try: + guard.feed(chunk) + except Exception as error: + return type(error), error.args + return None + + +def _compare(chunks, *, token_column=None, budget=1000000): + reference, actual = _ReferenceBounds([budget]), coverage._CsvBounds([budget]) + if token_column is not None: + # A fresh body record state reachable after a valid header. + for guard in (reference, actual): + guard.header = False + guard.token_column = token_column + for index, chunk in enumerate(chunks): + expected = _feed(reference, chunk) + observed = _feed(actual, chunk) + assert (observed, _snapshot(actual)) == (expected, _snapshot(reference)), ( + index, + chunk, + observed, + expected, + _snapshot(actual), + _snapshot(reference), + ) + if expected is not None: + break + return actual + + +@pytest.mark.parametrize("token_column", [0, 1, 3]) +def test_exhaustive_short_body_streams(monkeypatch, token_column): + monkeypatch.setattr(coverage, "_ROW_MAX", 4) + monkeypatch.setattr(coverage, "_TOKEN_MAX", 3) + alphabet = (b"a", b'"', b",", b"\r", b"\n", b"\xc3", b"\0") + for length in range(7): + for parts in itertools.product(alphabet, repeat=length): + raw = b"".join(parts) + for chunks in ((raw,), parts, (raw[:2], raw[2:])): + _compare(chunks, token_column=token_column, budget=130) + + +@pytest.mark.parametrize("row_limit", [0, 1, 4, 20]) +@pytest.mark.parametrize("field_limit", [0, 1, 3, 20]) +def test_exact_failure_state_and_budget_order(monkeypatch, row_limit, field_limit): + monkeypatch.setattr(coverage, "_ROW_MAX", row_limit) + monkeypatch.setattr(coverage, "_TOKEN_MAX", field_limit) + for budget in (-1, 0, 117, 118, 119, 250): + for raw in (b"", b"\n", b"\r\n\r\n", b"a,bc\n", b'"\na",b\n'): + for token_column in (0, 1, 3): + _compare((raw,), token_column=token_column, budget=budget) + _compare( + tuple(bytes([v]) for v in raw), + token_column=token_column, + budget=budget, + ) + + +@pytest.mark.parametrize( + "header", + [ + b"PRPERTYP,x", + b"x,PRPERTYP", + b"x,PRPERTYP,y", + b'"PRPERTYP",x', + b'"x,\ny",PRPERTYP', + b"PRPERTYP,PRPERTYP", + b"x,y", + b"\xff,PRPERTYP", + ], +) +def test_real_header_and_all_byte_values(header): + raw = header + b"\r\n" + bytes(range(256)) + b"\r\n" + for width in (1, 2, 17, 65536): + _compare(tuple(raw[i : i + width] for i in range(0, len(raw), width))) + + +def test_seeded_long_records_and_chunk_boundaries(): + rng = random.Random(881726) + records = [] + for _ in range(50): + fields = [] + for _ in range(80): + token = bytes(rng.choice(b"0123456789xyz") for _ in range(rng.randrange(8))) + if rng.randrange(12) == 0: + token = b'"' + token + b',\r\n""more"' + fields.append(token) + records.append(b",".join(fields) + rng.choice((b"\n", b"\r", b"\r\n"))) + raw = b"x,PRPERTYP,y\r\n" + b"".join(records) + for width in (1, 7, 63, 1024, 65536): + _compare(tuple(raw[i : i + width] for i in range(0, len(raw), width))) + _compare((raw,), budget=500) + + +def test_complete_unquoted_records_use_fast_path(monkeypatch): + guard = coverage._CsvBounds([10000]) + guard.feed(b"x,PRPERTYP,y\n") + monkeypatch.setattr(guard, "_slow_record", lambda *args: pytest.fail("slow path")) + guard.feed(b"a,123,b\n\r\n0,456,2\r") + assert guard.budget[0] == 10000 - 3 * 118 - 2 * 4 + assert guard.after_cr is True + + +@pytest.mark.parametrize("terminator", [b"\n", b"\r", b"\r\n"]) +def test_every_nonstructural_byte_uses_fast_admission(monkeypatch, terminator): + reference = _ReferenceBounds([1000000]) + actual = coverage._CsvBounds([1000000]) + for guard in (reference, actual): + guard.feed(b"x,PRPERTYP,y\n") + monkeypatch.setattr(actual, "_slow_record", lambda *args: pytest.fail("slow path")) + for byte in range(256): + if byte in (10, 13, 34, 44): + continue + token = bytes([byte]) + record = token + b"," + token + b"," + token + terminator + reference.feed(record) + actual.feed(record) + assert _snapshot(actual) == _snapshot(reference), byte + + +def test_empty_crlf_records_are_each_charged(): + guard = _compare((b"\r\n\r\n",), token_column=0, budget=1000) + assert guard.budget[0] == 1000 - 2 * (118 + 1) + + +def test_record_crossing_chunk_uses_unchanged_fallback(): + chunks = (b"PRPERTYP,x\r", b"\na,", b'"b\r', b'\n""c"\nnext,x', b"\n") + _compare(chunks) + + +def test_long_unquoted_records_and_absent_newline_kind(): + fields = [str(i % 17).encode() for i in range(800)] + for token_column in (0, 400, 799, 800): + for terminator in (b"\n", b"\r", b"\r\n"): + raw = (b",".join(fields) + terminator) * 25 + for width in (63, 65536, len(raw)): + _compare( + tuple(raw[i : i + width] for i in range(0, len(raw), width)), + token_column=token_column, + ) diff --git a/packages/microcosm-build/tests/test_us_current_acs_immigration_source_projection.py b/packages/microcosm-build/tests/test_us_current_acs_immigration_source_projection.py new file mode 100644 index 000000000..307719561 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_acs_immigration_source_projection.py @@ -0,0 +1,446 @@ +"""ACS literal qualification and genuine invented-source owner controls.""" + +import copy +import json +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import ( + current_acs_immigration_source_projection as owner, +) + + +def raw_rows(rows): + return pd.DataFrame( + [ + dict(SERIALNO="2024HU0000001", SPORDER=str(i + 1), **r) + for i, r in enumerate(rows) + ], + columns=owner.literals.ACS_COLUMNS, + dtype=owner.literals.STRING, + ) + + +@pytest.mark.parametrize( + "cit,year,precision,lower,upper", + [ + ("1", "", "not_in_universe", None, None), + ("2", "1938", "bottom_coded", None, 1938), + ("3", "1939", "interval", 1939, 1944), + ("4", "1945", "calendar_year", 1945, 1945), + ("5", "2024", "calendar_year", 2024, 2024), + ], +) +def test_published_entry_universe_and_precision(cit, year, precision, lower, upper): + raw = raw_rows([dict(CIT=cit, POBP="464", YOEP=year, AGEP="99")]) + typed = owner._numeric_literals(raw) + assert typed.at[0, "CIT"] == int(cit) + assert typed.at[0, "YOEP_precision"] == precision + assert typed.at[0, "YOEP_is_niu"] == (year == "") + assert typed.at[0, "POBP_precision"] == "Tunisia_Western_Sahara_South_Sudan_group" + for name, expected in (("YOEP_lower", lower), ("YOEP_upper", upper)): + assert ( + pd.isna(typed.at[0, name]) + if expected is None + else typed.at[0, name] == expected + ) + assert raw.at[0, "YOEP"] == year + + +@pytest.mark.parametrize("cit", ["2", "3", "4", "5"]) +def test_only_cit_one_allows_entry_niu(cit): + with pytest.raises(ValueError, match="YOEP_REQUIRED"): + owner._numeric_literals( + raw_rows([dict(CIT=cit, POBP="451", YOEP="", AGEP="45")]) + ) + + +@pytest.mark.parametrize("year", ["1938", "1939", "2024"]) +def test_cit_one_refuses_entry_observation_outside_question_universe(year): + with pytest.raises(ValueError, match="YOEP_OUTSIDE_UNIVERSE"): + owner._numeric_literals( + raw_rows([dict(CIT="1", POBP="001", YOEP=year, AGEP="45")]) + ) + + +@pytest.mark.parametrize( + "field,token", + [ + ("CIT", "0"), + ("CIT", "6"), + ("CIT", "True"), + ("CIT", "1.0"), + ("POBP", "057"), + ("POBP", "073"), + ("POBP", "463"), + ("POBP", "999"), + ("POBP", ""), + ("YOEP", "0"), + ("YOEP", "1900"), + ("YOEP", "1937"), + ("YOEP", "1940"), + ("YOEP", "1944"), + ("YOEP", "2025"), + ("YOEP", "2000.5"), + ("YOEP", "True"), + ("AGEP", "100"), + ("AGEP", "-1"), + ], +) +def test_literal_domains_refuse_unresolved_codes(field, token): + values = dict(CIT="5", POBP="451", YOEP="2001", AGEP="45") + values[field] = token + with pytest.raises(ValueError): + owner._numeric_literals(raw_rows([values])) + + +def source_arguments(path, patch, *, missing_entry=False, outside_universe=False): + import test_us_current_survey_immigration_source as fixture + import test_us_survey_population_preparation as preparation_fixture + + old = preparation_fixture._person + from test_us_acs_person_coverage_authentication import _csv as csv + + def person(*args, **kwargs): + row = old(*args, **kwargs) + row["SEX"] = "2" if int(row["SPORDER"]) == 2 else "1" + return row + + def source_csv(rows): + if rows and "SPORDER" in rows[0]: + rows = [dict(r) for r in rows] + for row, cit, birthplace, entry in zip( + rows, + ("1", "2", "3", "4", "5"), + ("001", "072", "200", "451", "464"), + ("", "2020", "2000", "2010", "2024"), + strict=True, + ): + row.update( + CIT=cit, POBP=birthplace, YOEP="" if missing_entry else entry + ) + if outside_universe and cit == "1": + row["YOEP"] = "2020" + return csv(rows) + + patch.setattr(preparation_fixture, "_person", person) + patch.setattr(preparation_fixture, "_csv", source_csv) + return fixture.source_arguments(path, patch) + + +@pytest.fixture(scope="module") +def actual(tmp_path_factory): + with pytest.MonkeyPatch.context() as patch: + full_args, partial_args, _ = source_arguments( + tmp_path_factory.mktemp("acs-immigration"), patch + ) + full = owner.source.prepare_authenticated_survey_population(**full_args) + partial = owner.source.prepare_authenticated_survey_population(**partial_args) + yield SimpleNamespace( + full=full, + partial=partial, + view=owner.borrow_current_acs_immigration_projection(full), + smaller=owner.borrow_current_acs_immigration_projection(partial), + ) + + +def test_real_selected_source_projection_has_no_stock_or_status_authority(actual): + view = actual.view + view.validate() + assert view.raw.index.equals(view.person.index) + assert view.person.source.eq("acs").all() + assert view.person.observation_year.eq(2024).all() + assert view.person.source_year.eq(2024).all() + assert sorted(view.person.CIT) == [1, 2, 3, 4, 5] + assert view.raw.YOEP.eq("").sum() == 1 + assert view.person.YOEP.isna().sum() == 1 + assert view.person.YOEP_is_niu.eq(view.person.CIT.eq(1)).all() + assert view.person.sort_values("CIT").POBP.tolist() == [1, 72, 200, 451, 464] + assert ( + view.person.loc[view.person.POBP.eq(464), "POBP_precision"].iloc[0] + == "Tunisia_Western_Sahara_South_Sudan_group" + ) + assert view.person.is_female.eq(view.person.SEX.eq(2)).all() + assert view.person.state_fips.eq(6).all() + assert not { + "person_weight", + "household_weight", + "WSAL_VAL", + "immigration_status", + } & set(view.person) + receipt = json.loads(view.receipt) + assert receipt["projection_scope"] == "selected_original_ACS_people" + for key in ( + "status_assignment_performed", + "national_stock_alignment_qualified", + "source_admission_issued", + ): + assert receipt[key] is False + assert receipt["person_weight_authority"] == "none" + assert receipt["unallocated_observation_claim"] is False + assert "2024GQ" in " ".join(view.raw.SERIALNO) + # Re-selection cannot become a full-country denominator claim. + assert set(actual.smaller.raw.index) <= set(view.raw.index) + + +@pytest.mark.parametrize("field", ["person", "raw", "households"]) +def test_copy_and_equal_detached_tables_refuse(actual, field): + with pytest.raises(ValueError, match="RETAINED_VIEW"): + copy.copy(actual.view).validate() + with pytest.raises(ValueError, match="RETAINED_VIEW"): + replace(actual.view).validate() + original = getattr(actual.view, field) + try: + object.__setattr__(actual.view, field, original.copy(deep=True)) + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + actual.view.validate() + finally: + object.__setattr__(actual.view, field, original) + + +@pytest.mark.parametrize( + "field,column,value", + [ + ("person", "AGEP", 99), + ("person", "SEX", 2), + ("person", "observation_year", 2025), + ("person", "native_person_id", 999), + ("person", "person_household_id", 999), + ("raw", "CIT", "5"), + ("households", "state_fips", 1), + ], +) +def test_mutated_projection_refuses(actual, field, column, value): + table = getattr(actual.view, field) + row, old = table.index[0], table.iloc[0][column] + try: + table.at[row, column] = value + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + actual.view.validate() + finally: + table.at[row, column] = old + actual.view.validate() + + +def test_unissued_preparation_and_wrong_owner_refuse(actual): + with pytest.raises(ValueError): + owner.borrow_current_acs_immigration_projection(copy.copy(actual.full)) + with pytest.raises(ValueError): + owner.borrow_current_acs_immigration_projection(actual.view) + old = actual.view.receipt + try: + object.__setattr__(actual.view, "receipt", actual.smaller.receipt) + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + actual.view.validate() + finally: + object.__setattr__(actual.view, "receipt", old) + + +def test_final_file_io_cannot_mutate_output(actual, monkeypatch): + table = actual.view.person + row, old = table.index[0], table.iloc[0].AGEP + read, triggered = Path.read_bytes, [] + + def mutate(path): + result = read(path) + if not triggered: + triggered.append(True) + table.at[row, "AGEP"] = 99 + return result + + try: + with monkeypatch.context() as patch: + patch.setattr(Path, "read_bytes", mutate) + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + actual.view.validate() + assert triggered + finally: + table.at[row, "AGEP"] = old + + +def test_literal_closure_and_value_mutation_cannot_move_retained_seal(actual): + retained = owner._RETAINED[id(actual.view)][2] + qualified = retained.qualified + raw, callback = qualified.acs_raw, qualified._revalidate + row, old = raw.index[0], raw.iloc[0].YOEP + cell = dict(zip(callback.__code__.co_freevars, callback.__closure__, strict=True))[ + "seals" + ] + previous = cell.cell_contents + try: + raw.at[row, "YOEP"] = "2001" + tables = ( + qualified.origins, + qualified.asec_full_raw, + qualified.asec_selected_raw, + qualified.acs_raw, + qualified.person_evidence, + ) + cell.cell_contents = tuple(owner.literals._table_seal(t) for t in tables) + qualified.validate() # Its editable closure is deliberately insufficient. + with pytest.raises(ValueError, match="LITERAL_CHANGED"): + actual.view.validate() + finally: + raw.at[row, "YOEP"] = old + cell.cell_contents = previous + actual.view.validate() + + +@pytest.mark.parametrize( + "defect,reason", + [("missing_entry", "YOEP_REQUIRED"), ("outside_universe", "YOEP_OUTSIDE_UNIVERSE")], +) +def test_genuine_source_entry_universe_refuses_before_projection( + tmp_path, monkeypatch, defect, reason +): + full, _, _ = source_arguments(tmp_path, monkeypatch, **{defect: True}) + preparation = owner.source.prepare_authenticated_survey_population(**full) + with pytest.raises(ValueError, match=reason): + owner.borrow_current_acs_immigration_projection(preparation) + + +def projection_inputs(actual): + retained = owner._RETAINED[id(actual.view)][2] + native = retained.preparation_entry[2].source_frames[0] + return [ + actual.view.raw.copy(deep=True), + retained.qualified.origins.copy(deep=True), + copy.deepcopy( + json.loads(retained.preparation_entry[1])["origins"]["households"] + ), + native.person.copy(deep=True), + native.table("household").copy(deep=True), + retained.preparation_entry[2].frame.person.copy(deep=True), + ] + + +@pytest.mark.parametrize( + "defect", + [ + "age", + "sex", + "sex_boolean", + "state", + "membership", + "line", + "literal_household", + "year", + "household_year", + "household_source", + "household_key", + ], +) +def test_pure_source_coordinate_checks(actual, defect): + raw, origins, h_origins, people, houses, receiving = args = projection_inputs( + actual + ) + if defect == "age": + people.loc[0, "A_AGE"] += 1 + elif defect in ("sex", "sex_boolean"): + people["SEX"] = people.SEX.astype(object) + people.loc[0, "SEX"] = True if defect == "sex_boolean" else 2 + elif defect == "state": + houses.loc[0, "ST"] = "72" + elif defect == "membership": + people.loc[0, "person_household_id"] = houses.household_id.iloc[-1] + elif defect == "line": + people.loc[0, "SPORDER"] = 2 + elif defect == "literal_household": + raw.loc[raw.index[0], "SERIALNO"] = "2024HU9999999" + elif defect == "year": + origins.loc[raw.index[0], "survey_year"] = 2025 + else: + target = next(r for r in h_origins if r["source"] == "acs") + target[ + { + "household_year": "survey_year", + "household_source": "source", + "household_key": "raw_native_id", + }[defect] + ] = { + "household_year": 2025, + "household_source": "asec", + "household_key": "2024HU9999999", + }[defect] + with pytest.raises(ValueError): + owner._project(*args) + + +def test_pure_projection_reorders_by_exact_keys_and_preserves_large_ids(actual): + raw, origins, h_origins, people, houses, receiving = args = projection_inputs( + actual + ) + offset = 2**90 + ids = {old: offset + int(old) for old in receiving.person_id} + raw.index = pd.Index([ids[v] for v in raw.index], name=raw.index.name) + origins.index = pd.Index([ids[v] for v in origins.index], name=origins.index.name) + receiving["person_id"] = receiving.person_id.map(ids) + hids = {old: 2 * offset + int(old) for old in receiving.person_household_id} + receiving["person_household_id"] = receiving.person_household_id.map(hids) + for row in h_origins: + row["household_id"] = hids[row["household_id"]] + args[3] = people.iloc[::-1].copy() + args[4] = houses.iloc[::-1].copy() + args[5] = receiving.iloc[::-1].copy() + person, household = owner._project(*args) + assert person.index.equals(raw.index) + assert person.person_household_id.tolist() == [ + hids[v] for v in actual.view.person.person_household_id + ] + assert set(household.household_id) == set(person.person_household_id) + pd.testing.assert_series_equal( + person.AGEP.reset_index(drop=True), + actual.view.person.AGEP.reset_index(drop=True), + ) + + +def test_literal_line_zero_padding_preserves_the_same_coordinate(actual): + args = projection_inputs(actual) + args[0]["SPORDER"] = args[0].SPORDER.str.zfill(2) + person, households = owner._project(*args) + pd.testing.assert_frame_equal(person, actual.view.person) + pd.testing.assert_frame_equal(households, actual.view.households) + + +@pytest.mark.parametrize("which", ["SEX", "AGEP", "source_year"]) +def test_authentic_native_table_mutation_is_refused(actual, which): + retained = owner._RETAINED[id(actual.view)][2] + table = retained.preparation_entry[2].source_frames[0].person + row, old = table.index[0], table.iloc[0][which] + try: + table.at[row, which] = old + 1 + with pytest.raises(ValueError): + actual.view.validate() + finally: + table.at[row, which] = old + actual.view.validate() + + +def test_late_io_cannot_mutate_literal_even_if_callback_was_already_checked( + actual, monkeypatch +): + retained = owner._RETAINED[id(actual.view)][2] + raw = retained.qualified.acs_raw + row, old = raw.index[0], raw.iloc[0].CIT + read, triggered = Path.read_bytes, [] + + def mutate(path): + result = read(path) + if path == Path(owner.__file__) and not triggered: + triggered.append(True) + raw.at[row, "CIT"] = "5" + return result + + try: + with monkeypatch.context() as patch: + patch.setattr(Path, "read_bytes", mutate) + with pytest.raises(ValueError, match="LITERAL_CHANGED"): + actual.view.validate() + assert triggered + finally: + raw.at[row, "CIT"] = old + actual.view.validate() diff --git a/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py b/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py new file mode 100644 index 000000000..3e53ca756 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py @@ -0,0 +1,523 @@ +"""Actual retained ACS source owners over privately pinned invented bytes.""" + +import copy +import io + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_acs_income_anchor_source as owner + + +@pytest.mark.parametrize( + "field,token,age,status,known", + [ + ("INTP", "-10000", "30", "observed", True), + ("INTP", "-4", "30", "observed", True), + ("INTP", "0", "30", "observed", True), + ("RETP", "0", "30", "observed", True), + ("RETP", "999999", "30", "observed", True), + ("INTP", "-3", "30", "outside_published_domain", False), + ("RETP", "-4", "30", "outside_published_domain", False), + ("INTP", "1000000", "30", "outside_published_domain", False), + ("INTP", "", "14", "outside_universe_blank", False), + ("INTP", "", "15", "missing_source_amount", False), + ("RETP", "4", "14", "outside_universe_observation", False), + ("INTP", "4.0", "30", "malformed_source_amount", False), + ("INTP", " 4", "30", "malformed_source_amount", False), + ("INTP", "NA", "30", "malformed_source_amount", False), + ], +) +def test_published_anchor_domain_is_distinct_from_storage_and_universe( + field, token, age, status, known +): + value = owner.parse_anchor( + token, age=age, adjustment="1000133", allocation="1", field=field + ) + assert value["status"] == status and value["known"] is known + assert value["allocation_status"] == "allocated" + if known: + expected = np.float64(int(token)) * (np.float64(1000133) / 1_000_000.0) + assert np.float64(value["amount"]).view("uint64") == expected.view("uint64") + else: + assert value["amount"] is None + + +@pytest.mark.parametrize( + "flag,status", + [ + ("0", "not_allocated"), + ("1", "allocated"), + ("", "allocation_missing"), + ("x", "allocation_unrecognized"), + ], +) +def test_allocation_does_not_decide_amount_knownness(flag, status): + value = owner.parse_anchor( + "4", age="15", adjustment="1000133", allocation=flag, field="RETP" + ) + assert value["known"] and value["allocation_status"] == status + + +@pytest.mark.parametrize("adjustment", ["", "0", "-1", "1.1", "inf"]) +def test_unknown_or_invalid_adjustment_never_supplies_an_amount(adjustment): + value = owner.parse_anchor( + "4", age="15", adjustment=adjustment, allocation="0", field="INTP" + ) + assert not value["known"] and value["amount"] is None + assert value["status"] == "invalid_adjustment" + + +def test_column_numeric_identity_keeps_scalar_mapper_bits_and_unknowns(): + tokens = ["-800", "0", "bad", "", "4.0", "-0.0", "NA", "999999", "1000133"] + expected = np.array( + [ + pd.to_numeric(pd.Series([v], dtype=object), errors="coerce").iloc[0] + for v in tokens + ], + dtype=np.float64, + ) + actual = owner._native_numbers(tokens) + assert np.array_equal(np.isnan(actual), np.isnan(expected)) + assert np.array_equal( + actual[~np.isnan(actual)].view("uint64"), + expected[~np.isnan(expected)].view("uint64"), + ) + + +def _literal_rows(): + return [ + { + "SERIALNO": "2024HU0000001", + "SPORDER": "1", + "INTP": "-800", + "RETP": "444", + "ADJINC": "1000133", + "AGEP": "30", + "FINTP": "0", + "FRETP": "1", + }, + { + "SERIALNO": "2024HU0000001", + "SPORDER": "2", + "INTP": "0", + "RETP": "", + "ADJINC": "1000133", + "AGEP": "15", + "FINTP": "1", + "FRETP": "", + }, + { + "SERIALNO": "2024HU0000002", + "SPORDER": "1", + "INTP": "bad", + "RETP": "4", + "ADJINC": "1000133", + "AGEP": "80", + "FINTP": "", + "FRETP": "x", + }, + *( + { + "SERIALNO": serial, + "SPORDER": "1", + "INTP": "0", + "RETP": "0", + "ADJINC": "1000000", + "AGEP": age, + "FINTP": "0", + "FRETP": "0", + } + for serial, age in (("2024GQ0000001", "40"), ("2024GQ0000002", "50")) + ), + ] + + +def _scan(rows, *, wanted=None): + import csv + + text = io.StringIO(newline="") + writer = csv.DictWriter(text, fieldnames=owner.COLUMNS) + writer.writeheader() + writer.writerows(rows) + selected = {} + wanted = ( + wanted + if wanted is not None + else {(r["SERIALNO"], int(r["SPORDER"])) for r in rows} + ) + count = owner._scan( + io.BytesIO(text.getvalue().encode()), wanted, selected, maximum=100 + ) + return count, selected + + +def _records_raw_table(origins, selected): + """Pre-iteration implementation, retained only as a parity oracle.""" + owner.require( + set(selected) == set(origins.anchor_source_key), "SELECTED_SOURCE_ROSTER" + ) + raw = pd.DataFrame( + [selected[k] for k in origins.anchor_source_key], + index=origins.index.copy(), + columns=owner.COLUMNS, + dtype=object, + ) + owner.require( + all( + owner._key(row) == key + for row, key in zip( + raw.to_dict("records"), origins.anchor_source_key, strict=True + ) + ), + "SOURCE_COORDINATE_CHANGED", + ) + return raw + + +def _records_parsed_table(raw, origins): + """Old anchor order, pandas dtypes and record boxing for exact comparison.""" + result = raw.copy(deep=True) + result["native_person_id"] = origins.selected_receiving_person_id.to_numpy( + copy=True + ) + result["source_year"] = 2024 + result["dollar_year"] = 2024 + for column, flag, prefix, _output in owner.ANCHORS: + values = [ + owner.parse_anchor( + row[column], + age=row["AGEP"], + adjustment=row["ADJINC"], + allocation=row[flag], + field=column, + ) + for row in raw.to_dict("records") + ] + parsed = pd.DataFrame(values, index=raw.index) + for name in parsed: + dtype = ( + "Float64" + if name == "amount" + else ( + bool + if name in ("known", "adjustment_known", "in_income_universe") + else owner.STRING_DTYPE + ) + ) + result[prefix + "_" + name] = pd.array(parsed[name], dtype=dtype) + return result + + +def _iteration_fixture(): + rows = _literal_rows() + for i, overrides in enumerate( + ( + {"INTP": "", "RETP": "4", "AGEP": "14"}, + {"INTP": "１２", "RETP": "−4", "FINTP": "é", "FRETP": "未"}, + {"INTP": "-10000", "RETP": "999999", "ADJINC": "9999999"}, + {"INTP": "-4", "RETP": "4", "ADJINC": "1"}, + {"INTP": "0", "RETP": "0", "ADJINC": ""}, + {"INTP": "9" * 64, "RETP": "-4", "ADJINC": "0"}, + ), + start=3, + ): + rows.append({**rows[0], "SERIALNO": f"2024HU{i:07d}", **overrides}) + _, selected = _scan(rows) + origins = pd.DataFrame( + { + "anchor_source_key": list(reversed(selected)), + "selected_receiving_person_id": np.arange(len(rows), dtype="int64") + 2**60, + }, + index=pd.Index( + np.iinfo("int64").max - np.arange(len(rows), dtype="int64") * 3, + name="person_id", + ), + ) + return origins, selected + + +def _assert_exact_anchor_parity(actual, expected): + pd.testing.assert_frame_equal(actual, expected, check_exact=True) + for column in ("property_income_amount", "retirement_income_amount"): + assert ( + actual[column].array._data.tobytes() + == expected[column].array._data.tobytes() + ) + assert ( + actual[column].array._mask.tobytes() + == expected[column].array._mask.tobytes() + ) + projections = [ + table.reset_index().to_json(orient="table", index=False).encode() + for table in (actual, expected) + ] + assert projections[0] == projections[1] + qualified = [ + owner.QualifiedAcsIncomeAnchors( + table, + projection, + {"projection_sha256": owner.hashlib.sha256(projection).hexdigest()}, + ) + for table, projection in zip((actual, expected), projections, strict=True) + ] + assert owner.income_anchor_seal(qualified[0]) == owner.income_anchor_seal( + qualified[1] + ) + + +def test_column_iteration_matches_records_projection_bits_and_digest(): + origins, selected = _iteration_fixture() + raw = owner._raw_table(origins, selected) + expected_raw = _records_raw_table(origins, selected) + pd.testing.assert_frame_equal(raw, expected_raw, check_exact=True) + _assert_exact_anchor_parity( + owner._parsed_table(raw, origins), _records_parsed_table(expected_raw, origins) + ) + + +def test_source_tables_do_not_expand_a_full_record_dictionary_list(monkeypatch): + origins, selected = _iteration_fixture() + original = pd.DataFrame.to_dict + + def refuse_records(self, orient="dict", *args, **kwargs): + assert orient != "records", "whole-roster records allocation" + return original(self, orient, *args, **kwargs) + + monkeypatch.setattr(pd.DataFrame, "to_dict", refuse_records) + parsed = owner._parsed_table(owner._raw_table(origins, selected), origins) + assert len(parsed) == len(origins) + + +@pytest.mark.parametrize( + "column,value", + [ + ("INTP", None), + ("INTP", pd.NA), + ("INTP", np.nan), + ("INTP", np.int64(4)), + ("RETP", np.float64(4)), + ("AGEP", "１５"), + ("ADJINC", 1000133), + ("FRETP", pd.NA), + ], +) +def test_iteration_preserves_invalid_literal_refusals(column, value): + origins, selected = _iteration_fixture() + raw = owner._raw_table(origins, selected) + raw.loc[raw.index[1], column] = value + with pytest.raises(ValueError) as previous: + _records_parsed_table(raw, origins) + with pytest.raises(ValueError) as current: + owner._parsed_table(raw, origins) + assert str(current.value) == str(previous.value) + + +@pytest.mark.parametrize("defect", ["missing", "coordinate", "household", "person"]) +def test_iteration_preserves_roster_and_coordinate_refusals(defect): + origins, selected = _iteration_fixture() + key = origins.anchor_source_key.iloc[0] + if defect == "missing": + del selected[key] + elif defect == "coordinate": + selected[key]["SPORDER"] = "20" + elif defect == "household": + selected[key]["SERIALNO"] = "bad" + else: + selected[key]["SPORDER"] = "21" + with pytest.raises(ValueError) as previous: + _records_raw_table(origins, selected) + with pytest.raises(ValueError) as current: + owner._raw_table(origins, selected) + assert str(current.value) == str(previous.value) + + +def test_iteration_preserves_anchor_then_row_validation_order(monkeypatch): + origins, selected = _iteration_fixture() + raw = owner._raw_table(origins, selected) + # A RETP failure in the first row must not precede a later INTP failure. + raw.loc[raw.index[0], "RETP"] = None + raw.loc[raw.index[2], "INTP"] = None + parse = owner.parse_anchor + calls = [] + + def record_parse(token, **kwargs): + calls.append((token, kwargs)) + return parse(token, **kwargs) + + monkeypatch.setattr(owner, "parse_anchor", record_parse) + with pytest.raises(ValueError): + _records_parsed_table(raw, origins) + previous = calls.copy() + calls.clear() + with pytest.raises(ValueError): + owner._parsed_table(raw, origins) + assert calls == previous + assert [kwargs["field"] for _, kwargs in calls] == ["INTP"] * 3 + + +def test_literal_scan_is_keyed_and_keeps_every_original_token(): + count, values = _scan(_literal_rows()[::-1]) + assert count == 5 + assert values["2024HU0000001", 2]["RETP"] == "" + assert values["2024HU0000002", 1]["INTP"] == "bad" + + +@pytest.mark.parametrize("defect", ["duplicate", "foreign_bad_key", "short_row"]) +def test_literal_reader_refuses_duplicate_selected_and_invalid_tail(defect): + rows = _literal_rows() + if defect == "duplicate": + rows.append(rows[0]) + elif defect == "foreign_bad_key": + rows.append({**rows[0], "SERIALNO": "bad"}) + else: + stream = io.BytesIO((",".join(owner.COLUMNS) + "\n1,2\n").encode()) + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + owner._scan(stream, set(), {}, maximum=100) + return + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + _scan(rows) + + +def _arguments(tmp_path, monkeypatch): + import test_us_survey_population_preparation as fixture + + original = fixture._person + lookup = {(r["SERIALNO"], int(r["SPORDER"])): r for r in _literal_rows()} + + def person(*args, **kwargs): + row = original(*args, **kwargs) + key = (row["SERIALNO"], int(row["SPORDER"])) + row.update( + { + k: v + for k, v in lookup.get(key, {}).items() + if k not in ("SERIALNO", "SPORDER") + } + ) + row.setdefault("FINTP", "0") + row.setdefault("FRETP", "0") + return row + + monkeypatch.setattr(fixture, "_person", person) + return fixture.fixture(tmp_path, monkeypatch, zero=False) + + +def test_actual_qualifier_binds_originals_and_adjusted_bits(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original = prepared.checked_view().frame + before = owner.preparation._frame_identity(original) + qualified = owner.qualify_current_acs_income_anchors(prepared) + assert qualified.evidence["source_admission_issued"] is False + assert qualified.evidence["decomposition_performed"] is False + assert len(qualified.anchors) == 5 + first = qualified.anchors.set_index(["SERIALNO", "SPORDER"]) + assert first.loc[("2024HU0000001", "1"), "property_income_amount"] < 0 + assert first.loc[("2024HU0000001", "2"), "property_income_amount"] == 0 + assert ( + first.loc[("2024HU0000001", "2"), "retirement_income_status"] + == "missing_source_amount" + ) + assert ( + first.loc[("2024HU0000002", "1"), "property_income_status"] + == "malformed_source_amount" + ) + assert pd.isna(first.loc[("2024HU0000002", "1"), "property_income_amount"]) + entry = prepared._checked() + origins = owner._origins(entry[2], owner.json.loads(entry[1])) + _, selected = _scan(_literal_rows()) + expected = _records_parsed_table(_records_raw_table(origins, selected), origins) + _assert_exact_anchor_parity(qualified.anchors, expected) + assert ( + qualified.projection + == expected.reset_index().to_json(orient="table", index=False).encode() + ) + assert owner.preparation._frame_identity(original) == before + fresh = owner.qualify_current_acs_income_anchors(prepared) + assert owner.income_anchor_seal(fresh) == owner.income_anchor_seal(qualified) + with pytest.raises(ValueError): + owner.qualify_current_acs_income_anchors(copy.copy(prepared)) + + +@pytest.mark.parametrize( + "defect", ["row_order", "person_id", "raw_amount", "adjusted_bit"] +) +def test_exact_retained_comparison_refuses_source_or_amount_mutation( + tmp_path, monkeypatch, defect +): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + entry = prepared._checked() + document = owner.json.loads(entry[1]) + origins = owner._origins(entry[2], document) + _, selected = _scan(_literal_rows()) + raw = owner._raw_table(origins, selected) + native = owner.preparation._copy_source(entry[2].source_frames[0]) + if defect == "row_order": + origins = origins.iloc[::-1].copy() + elif defect == "person_id": + native.person.loc[native.person.index[0], "person_id"] += 100 + elif defect == "raw_amount": + native.person.loc[native.person.index[0], "RETP"] = 888 + else: + column = "acs_interest_dividend_rental_income" + native.person.loc[native.person.index[0], column] = np.nextafter( + native.person[column].iloc[0], np.inf + ) + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + owner._compare_retained(raw, origins, native) + + +def test_final_source_io_mutation_cannot_return_qualified_value(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original = prepared.checked_view().frame + capture = owner._capture_person + mutated = [] + + def mutate_after_io(*args): + value = capture(*args) + original.person.loc[original.person.index[0], "age"] += 1 + mutated.append(True) + return value + + monkeypatch.setattr(owner, "_capture_person", mutate_after_io) + with pytest.raises(ValueError): + owner.qualify_current_acs_income_anchors(prepared) + assert mutated == [True] + + +def test_final_source_fence_seals_exact_returned_amount_bits(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original_seal = owner.income_anchor_seal + changed = [] + + def change_after_seal(qualified): + seal = original_seal(qualified) + if not changed: + changed.append(True) + table = qualified.anchors + index = table.index[ + table.SERIALNO.eq("2024HU0000001") & table.SPORDER.eq("1") + ][0] + table.loc[index, "property_income_amount"] = np.nextafter( + table.loc[index, "property_income_amount"], np.inf + ) + # Portable JSON's precision is not an exact physical mutation seal. + assert ( + qualified.projection + == table.reset_index().to_json(orient="table", index=False).encode() + ) + return seal + + monkeypatch.setattr(owner, "income_anchor_seal", change_after_seal) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_acs_income_anchors(prepared) + assert changed == [True] diff --git a/packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py b/packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py new file mode 100644 index 000000000..f250f65c2 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py @@ -0,0 +1,44 @@ +"""Published allocation meanings over source-pure invented projections.""" + +import copy + +import pytest +from test_us_current_asec_income_routing import _pure_rows, _set_amount + +from microcosm.build.us_runtime import current_asec_income_routing_source as owner + + +@pytest.mark.parametrize( + "earn_code,farm_code,farm_status,expected", + [ + (0, 4, "in_printed_range", "allocation_code_meaning_unpublished"), + (4, 4, "in_printed_range", "publisher_allocated"), + (4, 0, "in_printed_range", "publisher_allocated"), + (4, None, "missing", "publisher_allocated"), + (0, 0, "in_printed_range", "published_flags_all_zero_with_unflagged_fields"), + (0, None, "missing", "allocation_flag_not_populated"), + (0, None, "outside_printed_range", "unresolved_allocation_provenance"), + ], +) +def test_farm_allocation_requires_a_published_code_meaning( + earn_code, farm_code, farm_status, expected +): + raw, ages = _pure_rows(1, [40.0], ERN_YN=["1"], FRMOTR=["1"], FRSE_YN=["1"]) + _set_amount(raw, "FRSE_VAL", [100.0]) + raw["allocations"]["I_ERNYN"] = ([earn_code], ["in_printed_range"]) + raw["allocations"]["I_FRMYN"] = ([farm_code], [farm_status]) + before = copy.deepcopy(raw["allocations"]) + output = owner.project_income_routing(raw, ages) + assert output.farm_allocation_origin.iloc[0] == expected + assert output.farm_known_amount.iloc[0] == 100.0 + assert raw["allocations"] == before + assert "meaning of its codes" in owner.AMBIGUOUS_FLAG_COVERAGE["I_FRMYN"] + + +@pytest.mark.parametrize( + "name,code", [("I_RNTVAL", 4), ("I_DSTSC", 9), ("I_DSTYNCOMP", 11)] +) +def test_documented_nonzero_flags_still_establish_allocation(name, code): + assert code in owner.ALLOCATION_ENTRIES[name][5] + flags = {name: ([code], ["in_printed_range"])} + assert owner._allocation_origin(flags, unflagged=False)[0] == "publisher_allocated" diff --git a/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py b/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py new file mode 100644 index 000000000..d3c820736 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py @@ -0,0 +1,339 @@ +"""Child-support source semantics and invented actual-owner boundaries.""" + +import copy +import hashlib +import shutil + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_child_support_source as owner + + +def row(**changes): + return { + **{name: "0" for name in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "CSP_VAL": "1200", + "CSP_YN": "1", + "CHSP_VAL": "2000", + "CHSP_YN": "1", + "CHELSEW_YN": "1", + **changes, + } + + +def project(**changes): + return owner.project_child_support_literals(pd.DataFrame([row(**changes)])).iloc[0] + + +@pytest.mark.parametrize( + "receipt,amount,status,known", + [ + ("1", "1200", "known_receipt", True), + ("1", "0", "ambiguous_recipient_zero", False), + ("2", "0", "known_nonreceipt", True), + ("2", "1200", "contradictory_no_nonzero", False), + ("0", "0", "niu", False), + ("", "1200", "missing_receipt_literal", False), + ], +) +def test_received_support_uses_receipt_but_does_not_complete_recipient_zeros( + receipt, amount, status, known +): + value = project(CSP_YN=receipt, CSP_VAL=amount) + assert value.CSP_VAL_reporting_status == status + assert bool(value.CSP_VAL_amount_known) is known + + +@pytest.mark.parametrize("obligation", ["0", "1", "2", ""]) +def test_paid_zero_is_niu_under_every_obligation_answer(obligation): + value = project(CHSP_VAL="0", CHSP_YN=obligation) + assert value.CHSP_VAL_published_amount == 0 + assert value.CHSP_VAL_reporting_status == "declared_niu_amount" + assert not value.CHSP_VAL_amount_known and pd.isna(value.CHSP_VAL_amount) + assert not value.voluntary_payment_absence_known + assert not value.obligation_implies_payment + + +@pytest.mark.parametrize( + "elsewhere,obligation,status", + [ + ("1", "1", "observed_positive_payment"), + ("1", "2", "payment_outside_published_obligation_universe"), + ("1", "0", "payment_outside_published_obligation_universe"), + ("2", "1", "unresolved_child_elsewhere_route"), + ("0", "1", "unresolved_child_elsewhere_route"), + ("", "1", "unresolved_child_elsewhere_literal"), + ("1", "", "missing_obligation_literal"), + ("1", "x", "invalid_obligation_literal"), + ], +) +def test_positive_paid_cells_remain_visible_when_routing_is_unresolved( + elsewhere, obligation, status +): + value = project(CHELSEW_YN=elsewhere, CHSP_YN=obligation) + assert value.CHSP_VAL_reporting_status == status + assert value.CHSP_VAL_published_amount == 2000 + assert bool(value.CHSP_VAL_amount_known) is (status == "observed_positive_payment") + + +@pytest.mark.parametrize("field", ["CSP_VAL", "CHSP_VAL"]) +@pytest.mark.parametrize( + "token,status", + [ + ("", "missing"), + ("-1", "malformed"), + ("1.5", "malformed"), + ("NA", "malformed"), + ], +) +def test_amount_missing_malformed_and_negative_literals_remain_distinct( + field, token, status +): + value = project(**{field: token}) + assert value[field + "_literal"] == token + assert value[field + "_literal_status"] == status + assert not value[field + "_amount_known"] + + +def test_under15_observations_do_not_become_analytical_zeros(): + value = project( + A_AGE="14", CSP_VAL="0", CSP_YN="0", CHSP_VAL="0", CHSP_YN="0", CHELSEW_YN="0" + ) + for name in owner.AMOUNT_FIELDS: + assert value[name + "_reporting_status"] == "outside_reporting_universe" + assert not value[name + "_amount_known"] + + +def test_allocated_values_remain_known_and_missing_flags_remain_unknown(): + value = project(I_CHSPVAL="4", I_CSPVAL="", TCHSP_VAL="1") + assert value.CHSP_VAL_amount == 2000 and value.CHSP_VAL_amount_known + assert value.paid_allocation_origin == "publisher_allocated" + assert value.received_allocation_origin == "allocation_flag_not_populated" + assert value.I_CSPVAL_literal_status == "missing" + assert value.TCHSP_VAL_code == 1 + + +def test_pinned_domains_and_universe_discrepancy_are_preserved(): + entries = owner.amount_entries() + assert entries["CHSP_VAL"]["domain"]["zero_semantics"] == "niu" + assert entries["CHSP_VAL"]["domain"]["valid_dollar_range_inclusive"]["minimum"] == 1 + assert owner.RESPONSE_ENTRIES["CHSP_YN"][4] == "CHELSEW_YN" + assert owner.ALLOCATION_ENTRIES["I_CHSPYN"][4] == "CHELSEW_YN = 1" + assert "bare" in owner.OBLIGATION_UNIVERSE_NOTE + + +def test_nullable_seal_covers_values_masks_and_hidden_backing(): + raw = pd.DataFrame([row(CHSP_VAL="0")]) + values = owner.CurrentAsecChildSupportValues( + owner.project_child_support_literals(raw), raw, {} + ) + original = owner.child_support_values_seal(values) + assert owner.child_support_values_seal(copy.deepcopy(values)) == original + for change in ("bit", "mask", "hidden"): + other = copy.deepcopy(values) + array = ( + other.person.CSP_VAL_amount.array + if change == "bit" + else other.person.CHSP_VAL_amount.array + ) + if change == "bit": + array._data[0] = np.nextafter(array._data[0], np.inf) + elif change == "mask": + array._mask[0] = False + else: + array._data[0] = 17.0 + assert owner.child_support_values_seal(other) != original + + +def child_support_arguments(tmp_path, monkeypatch): + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + literals = { + 105: row(I_CHSPVAL="4"), + 106: row( + A_AGE="14", + CSP_VAL="0", + CSP_YN="0", + CHSP_VAL="0", + CHSP_YN="0", + CHELSEW_YN="0", + ), + 107: row(A_AGE="70", CSP_VAL="0", CSP_YN="2", CHSP_VAL="0", CHSP_YN="2"), + 108: row(CSP_VAL="0", CHSP_VAL="0"), + } + changes = {} + for name in (*owner.AMOUNT_FIELDS, "A_AGE"): + values = people[name].to_numpy(copy=True) + for person_id, literal in literals.items(): + values[ids == person_id] = int(literal[name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in (*owner.AMOUNT_FIELDS, "A_AGE"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + for name in ( + *owner.RESPONSE_ENTRIES, + *owner.ALLOCATION_ENTRIES, + *owner.TOPCODE_ENTRIES, + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row())[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "child-support-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(tmp_path, monkeypatch): + return owner.routing.source.prepare_authenticated_survey_population( + **child_support_arguments(tmp_path, monkeypatch) + ) + + +def test_actual_owner_keeps_obligation_distinct_and_refuses_copied_owner( + tmp_path, monkeypatch +): + parent = prepared(tmp_path, monkeypatch) + result = owner.qualify_current_asec_child_support(parent) + values = result.person.set_index("native_person_id") + assert values.loc[105, "CHSP_VAL_amount"] == 2000 + assert values.loc[105, "CSP_VAL_amount"] == 1200 + assert values.loc[107, "CSP_VAL_amount"] == 0 + assert not values.loc[107, "CHSP_VAL_amount_known"] + assert not values.loc[108, "CHSP_VAL_amount_known"] + assert not values.loc[108, "CSP_VAL_amount_known"] + assert not result.evidence["paid_niu_completed_with_zero"] + assert owner.child_support_values_seal( + owner.qualify_current_asec_child_support(parent) + ) == owner.child_support_values_seal(result) + with pytest.raises(ValueError): + owner.qualify_current_asec_child_support(copy.copy(parent)) + + # The actual qualifier checks the retained MoneyDomain before source capture. + check, checked = owner._domain_agreement, [] + + def rejected_domain(ready): + check(ready) + checked.append(True) + raise ValueError("INVENTED_DOMAIN_REFUSAL") + + def forbidden_capture(*args): + pytest.fail("source capture preceded live domain agreement") + + with monkeypatch.context() as patch: + patch.setattr(owner, "_domain_agreement", rejected_domain) + patch.setattr(owner, "_capture_member", forbidden_capture) + with pytest.raises(ValueError, match="INVENTED_DOMAIN_REFUSAL"): + owner.qualify_current_asec_child_support(parent) + assert checked == [True] + + +@pytest.mark.parametrize("field", ["PERIDNUM", "A_AGE", "CSP_VAL", "CHSP_VAL"]) +def test_actual_owner_requires_original_source_identity_and_amounts( + tmp_path, monkeypatch, field +): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if field == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(field)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_child_support(parent) + + +def test_actual_owner_requalifies_after_capture_io(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + calls = [] + + def changed(*args): + raw = capture(*args) + table = parent._checked()[2].frame.person + table.iloc[0, table.columns.get_loc("age")] += 1 + calls.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_child_support(parent) + assert calls + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "literal", "evidence"] +) +def test_actual_owner_final_seal_covers_complete_returned_values( + tmp_path, monkeypatch, mutation +): + parent = prepared(tmp_path, monkeypatch) + seal, calls = owner.child_support_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + if mutation == "amount_bit": + array = value.person.CSP_VAL_amount.array + i = np.flatnonzero(~array._mask)[0] + array._data[i] = np.nextafter(array._data[i], np.inf) + elif mutation == "masked_backing": + array = value.person.CHSP_VAL_amount.array + array._data[np.flatnonzero(array._mask)[0]] = 17.0 + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("CHSP_YN") + ] = "2" + else: + value.evidence["paid_niu_completed_with_zero"] = True + return before + + monkeypatch.setattr(owner, "child_support_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_child_support(parent) diff --git a/packages/microcosm-build/tests/test_us_current_asec_demographics.py b/packages/microcosm-build/tests/test_us_current_asec_demographics.py new file mode 100644 index 000000000..c61c62b15 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_demographics.py @@ -0,0 +1,191 @@ +"""Actual source readers over constructed, privately pinned invented source bytes.""" + +import csv +import hashlib +import json +import shutil +from dataclasses import replace +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_survey_population_preparation import fixture + +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import current_asec_demographics as owner +from microcosm.build.us_runtime import native_household_origin as origin + + +def _csv(path, headers, rows): + with path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(headers) + writer.writerows(rows) + data = path.read_bytes() + return SimpleNamespace( + size_bytes=len(data), + rows=len(rows), + member_sha256=hashlib.sha256(data).hexdigest(), + ) + + +@pytest.mark.parametrize( + "literal,code,status", + ( + ("06", 6, "in_printed_range"), + ("36", 36, "in_printed_range"), + ("", None, "missing"), + ("NA", None, "malformed"), + (" 6", None, "malformed"), + ("6.0", None, "malformed"), + ("57", None, "outside_printed_range"), + ("00", None, "outside_printed_range"), + ), +) +def test_household_state_reader_preserves_literal_and_knownness( + tmp_path, literal, code, status +): + path = tmp_path / "invented.csv" + pin = _csv(path, ("H_SEQ", "GESTFIPS", "unread"), [("00007", literal, "anything")]) + value = owner._read_state_capture(path, pin)[7] + assert value == { + "H_SEQ": "00007", + "GESTFIPS": literal, + "state_code": code, + "status": status, + "member_row_1based": 1, + } + + +@pytest.mark.parametrize("defect", ("duplicate", "digest", "size", "rows", "header")) +def test_state_capture_structure_and_exact_bytes_refuse(tmp_path, defect): + path = tmp_path / "invented.csv" + headers, rows = ("H_SEQ", "GESTFIPS"), [("00007", "06")] + if defect == "duplicate": + rows.append(("7", "36")) + if defect == "header": + headers = ("H_SEQ", "other") + pin = _csv(path, headers, rows) + if defect == "digest": + pin.member_sha256 = "f" * 64 + elif defect == "size": + pin.size_bytes += 1 + elif defect == "rows": + pin.rows += 1 + with pytest.raises(ValueError, match="CURRENT_ASEC_DEMOGRAPHICS_"): + owner._read_state_capture(path, pin) + + +def _demographic_arguments(tmp_path, monkeypatch, *, unknown=False, zero=True): + """Extend invented source members before actual preparation/issuance. + + Rebuild the real person-income attachment after member bytes change. Only + fixture registry pins are changed; no source issuer or ready() is replaced. + """ + arguments = fixture(tmp_path, monkeypatch, zero=zero) + source = arguments["source_dir"] / "asec" + members, pins = {}, [] + for year, member, archive, *_ in coverage._MEMBER_PINS: + path = source / f"pppub{year - 1999}.csv" + table = pd.read_csv(path, dtype=str, keep_default_na=False) + lines = table.A_LINENO.map(int) + table["A_SEX"] = lines.map({1: "1", 2: "2"}) + table["AXSEX"] = lines.map({1: "0", 2: "4"}) + table["A_EXPRRP"] = lines.map({1: "1", 2: "5"}) + table["P_SEQ"] = table.A_LINENO + if unknown and year == 2024: + table.loc[table.index[0], "AXSEX"] = "1" + assert table.notna().all().all() + # Reverse the actual source order, so receiving-row alignment cannot pass. + table.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(table), + len(data), + ) + ) + members[year] = path + for module in (coverage, restoration, owner.demographic): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "demographic-restored-money" + restoration.restore_asec_person_income_source( + source / "parent.h5", + source / "household-attachment.h5", + member_paths=members, + output_dir=output, + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, source / "person-income-attachment.h5" + ) + path = source / "hhpub25.csv" + table = pd.read_csv(path, dtype=str, keep_default_na=False) + table["GESTFIPS"] = table.H_SEQ.map({"00007": "06", "00008": "36"}) + if unknown: + table.loc[table.H_SEQ.eq("00008"), "GESTFIPS"] = "" + assert table.notna().all().all() + table.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + monkeypatch.setattr( + origin, + "_ASEC_MEMBER_PINS", + tuple( + replace( + pin, + member_sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + for pin in origin._ASEC_MEMBER_PINS + ), + ) + return arguments + + +@pytest.mark.parametrize("unknown", (False, True)) +def test_actual_current_native_parent_binds_demographics_by_origin( + tmp_path, monkeypatch, unknown +): + arguments = _demographic_arguments(tmp_path, monkeypatch, unknown=unknown) + prepared = owner.preparation_owner.prepare_authenticated_survey_population( + **arguments + ) + original = prepared.checked_view().frame + retained = {e: original.table(e).copy(deep=True) for e in original.entities} + result = owner.qualify_current_asec_demographics(prepared) + receipt = json.loads(result.receipt) + assert receipt["sex_observation_year"] == receipt["state_observation_year"] == 2025 + assert receipt["income_year"] == 2024 + assert not receipt["state_is_income_year_residence_claim"] + assert not receipt["source_admission_issued"] and not receipt["release_eligible"] + h = result.household.set_index("H_SEQ_integer") + assert h.loc[7, "GESTFIPS"] == "06" and h.loc[7, "state_fips"] == 6 + if unknown: + assert h.loc[8, "GESTFIPS"] == "" and pd.isna(h.loc[8, "state_fips"]) + assert receipt["state_unknown_households"] == 1 + assert receipt["sex_unknown_persons"] == 1 + unbound = result.person.asec_AXSEX.eq(1) + assert result.person.loc[unbound, "is_female"].isna().all() + assert not result.person.loc[unbound, "sex_known"].any() + else: + assert h.loc[8, "state_fips"] == 36 + assert ( + receipt["state_unknown_households"] == receipt["sex_unknown_persons"] == 0 + ) + allocated = result.person.asec_AXSEX.eq(4) + assert allocated.any() and result.person.loc[allocated, "is_female"].all() + assert result.person.loc[allocated, "sex_origin"].eq("census_allocated").all() + for entity, expected in retained.items(): + pd.testing.assert_frame_equal( + expected, original.table(entity), check_exact=True + ) + assert owner.qualify_current_asec_demographics(prepared).receipt == result.receipt + # Value transport is deliberately not an issued authority. Mutating it must + # not affect a fresh live projection or its source-bound receipt. + result.household.loc[result.household.index[0], "state_fips"] = 99 + fresh = owner.qualify_current_asec_demographics(prepared) + assert not fresh.household.state_fips.eq(99).any() diff --git a/packages/microcosm-build/tests/test_us_current_asec_dividend_source.py b/packages/microcosm-build/tests/test_us_current_asec_dividend_source.py new file mode 100644 index 000000000..70483f62f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_dividend_source.py @@ -0,0 +1,497 @@ +"""Invented dividend literals, survivor routes and actual retained source owners.""" + +import copy +import hashlib +import json +import shutil +from fractions import Fraction +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_dividend_source as owner + + +def row(**changes): + return { + **{name: "0" for name in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "DIV_YN": "1", + "DIV_VAL": "140", + "SUR_YN": "2", + **changes, + } + + +def project(**changes): + return owner.project_dividend_literals(pd.DataFrame([row(**changes)])).iloc[0] + + +@pytest.mark.parametrize( + "changes,status,amount", + [ + ({}, "known_receipt", 140), + ({"DIV_VAL": "0", "DIV_YN": "2"}, "known_nonreceipt", 0), + ({"DIV_VAL": "0"}, "ambiguous_recipient_zero", None), + ({"DIV_VAL": "0", "DIV_YN": "0"}, "niu", None), + ({"DIV_YN": "0"}, "contradictory_niu_nonzero", None), + ({"DIV_YN": "2"}, "contradictory_no_nonzero", None), + ({"DIV_YN": ""}, "missing_receipt_literal", None), + ({"DIV_YN": "x"}, "unrecognized_receipt_literal", None), + ({"DIV_VAL": ""}, "missing_amount", None), + ({"DIV_VAL": "-1"}, "invalid_amount_literal", None), + ({"DIV_VAL": "1.0"}, "invalid_amount_literal", None), + ( + {"A_AGE": "14", "DIV_VAL": "0", "DIV_YN": "0"}, + "outside_reporting_universe", + None, + ), + ({"A_AGE": "14"}, "contradictory_outside_reporting_universe", None), + ], +) +def test_dividend_receipt_unknownness_never_infers_age_only_or_recipient_zeros( + changes, status, amount +): + value = project(**changes) + assert value.DIV_VAL_reporting_status == status + assert bool(value.DIV_VAL_amount_known) is (amount is not None) + assert ( + pd.isna(value.DIV_VAL_amount) + if amount is None + else value.DIV_VAL_amount == amount + ) + assert value.DIV_VAL_literal == changes.get("DIV_VAL", "140") + + +@pytest.mark.parametrize( + "token,header,reference,agreement", + [ + ("0", "in_printed_range", "in_printed_range", "header_reference_agree"), + ("1", "in_printed_range", "in_printed_range", "header_reference_agree"), + ("5", "outside_printed_range", "in_printed_range", "header_reference_conflict"), + ("9", "outside_printed_range", "in_printed_range", "header_reference_conflict"), + ("", "missing", "missing", "literal_unresolved"), + ("x", "malformed", "malformed", "literal_unresolved"), + ("01", "malformed", "malformed", "literal_unresolved"), + ], +) +def test_dividend_receipt_allocation_preserves_both_conflicting_codebook_axes( + token, header, reference, agreement +): + value = project(I_DIVYN=token) + assert value.I_DIVYN_literal == token + assert value.I_DIVYN_header_range_status == header + assert value.I_DIVYN_referenced_values_status == reference + assert value.I_DIVYN_codebook_status == agreement + if reference == "in_printed_range": + assert value.I_DIVYN_code == int(token) + assert value.I_DIVYN_literal_status == "well_formed" + assert value.DIV_VAL_amount == 140 and value.DIV_VAL_amount_known + + +@pytest.mark.parametrize( + "changes,status,clear", + [ + ({}, "known_nonreceipt", True), + ({"SUR_YN": "1", "SUR_SC1": "1"}, "known_other_survivor_sources", True), + ({"SUR_YN": "1", "SUR_SC2": "9"}, "known_other_survivor_sources", True), + ({"SUR_YN": "1", "SUR_SC2": "8"}, "possible_estate_or_trust_route", False), + ( + {"SUR_YN": "1", "SUR_SC1": "8", "SUR_SC2": ""}, + "possible_estate_or_trust_route", + False, + ), + ({"SUR_YN": "1", "SUR_SC1": "10"}, "source_type_unspecified", None), + ({"SUR_YN": "1"}, "unreported_source_slots", None), + ( + {"SUR_YN": "1", "SUR_SC1": "1", "SUR_SC2": ""}, + "unresolved_source_literal", + None, + ), + ({"SUR_YN": "1", "SUR_SC1": "11"}, "unresolved_source_literal", None), + ({"SUR_YN": "1", "SUR_SC1": "x"}, "unresolved_source_literal", None), + ({"SUR_YN": "", "SUR_SC1": "8"}, "missing_receipt_literal", None), + ({"SUR_YN": "x"}, "unrecognized_receipt_literal", None), + ({"SUR_SC1": "8"}, "unresolved_nonreceipt_source_route", None), + ({"SUR_SC1": ""}, "unresolved_nonreceipt_source_route", None), + ({"SUR_YN": "0"}, "niu", None), + ({"SUR_YN": "0", "SUR_SC1": "1"}, "unresolved_niu_route", None), + ({"A_AGE": "14", "SUR_YN": "0"}, "outside_reporting_universe", None), + ({"A_AGE": "14", "SUR_YN": ""}, "unresolved_outside_reporting_universe", None), + ( + {"A_AGE": "14", "SUR_YN": "0", "SUR_SC1": ""}, + "unresolved_outside_reporting_universe", + None, + ), + ], +) +def test_survivor_clearance_requires_resolved_receipt_and_sources( + changes, status, clear +): + value = project(**changes) + assert value.survivor_property_route_status == status + assert ( + pd.isna(value.survivor_property_route_clear) + if clear is None + else bool(value.survivor_property_route_clear) is clear + ) + assert value.DIV_VAL_published_amount == 140 # Routes never change donor values. + if "8" in (changes.get("SUR_SC1"), changes.get("SUR_SC2")): + assert value.survivor_estate_or_trust_code_present + if changes.get("SUR_SC1") == "10": + assert value.SUR_SC1_literal_status == "in_printed_range" + assert value.survivor_unspecified_source_code_present + + +@pytest.mark.parametrize( + "field,token", + [ + ("I_DIVVAL", "9"), + ("I_DIVVAL", ""), + ("I_DIVVAL", "x"), + ("I_DIVYN", "5"), + ("TDIV_VAL", "1"), + ("TDIV_VAL", ""), + ("TRNT_VAL", "1"), + ("TRNT_VAL", ""), + ], +) +def test_source_allocation_topcode_flags_are_never_donor_filters(field, token): + value = project(**{field: token}) + assert value.DIV_VAL_amount == 140 and value.DIV_VAL_amount_known + assert bool(value.survivor_property_route_clear) + assert value[field + "_literal"] == token + + +def test_source_roster_and_primary_pin_have_no_survivor_amount_or_tax_outputs(): + assert owner.amount_entries()["DIV_VAL"][:4] == (6, 478, 45, "6C-24") + assert owner.ALLOCATION_ENTRIES["I_DIVYN"][5:] == ("0:1", "See I_ANNVAL") + assert owner.SURVIVOR_CODES[8] == "regular payments from estates or trusts" + assert owner.SURVIVOR_CODES[10] == "other or don't know" + assert not { + "SUR_VAL1", + "SUR_VAL2", + "SRVS_VAL", + "OI_VAL", + "RNT_VAL", + "CAP_VAL", + } & set(owner.READ_COLUMNS) + + +def test_cached_domain_entry_does_not_expose_a_shared_mutable_mapping(): + original = owner.amount_entries() + changed = owner.amount_entries() + changed["DIV_VAL"] = (99, *changed["DIV_VAL"][1:]) + changed["invented"] = () + assert owner.amount_entries() == original + assert type(owner._amount_entry()) is tuple + assert project().DIV_VAL_published_amount == 140 + + +@pytest.mark.parametrize( + "change", + [ + "duplicate_field", + "absent_vintage", + "duplicate_vintage", + "negative_code", + "negative_dollars", + "zero_semantics", + "missing_code", + "minimum", + "dictionary", + ], +) +def test_supported_domain_and_unique_current_vintage_are_required(change): + data = json.loads( + owner.resources.files(owner.__package__) + .joinpath(owner.routing.DOMAINS_RESOURCE) + .read_bytes() + ) + field = next(f for f in data["fields"] if f["name"] == "DIV_VAL") + current = next(v for v in field["vintages"] if v["income_year"] == 2024) + assert owner._decode_amount_entry(data) == owner.amount_entries()["DIV_VAL"] + if change == "duplicate_field": + data["fields"].append(copy.deepcopy(field)) + elif change == "absent_vintage": + field["vintages"].remove(current) + elif change == "duplicate_vintage": + field["vintages"].append(copy.deepcopy(current)) + elif change == "negative_code": + field["domain"]["declared_negative_nonmoney_codes"] = [-1] + elif change == "negative_dollars": + field["domain"]["negative_dollars_permitted"] = True + elif change == "zero_semantics": + field["domain"]["zero_semantics"] = "valid_zero_dollars" + elif change == "missing_code": + field["domain"]["declared_other_missing_codes"] = [999999] + elif change == "minimum": + field["domain"]["encoded_range_inclusive"]["minimum"] = -1 + else: + current["dictionary_spelling"] = "WRONG" + with pytest.raises(ValueError, match="ASEC_DIVIDEND_SOURCE_(DOMAIN_|DICTIONARY_)"): + owner._decode_amount_entry(data) + + +@pytest.mark.parametrize( + "change", ["missing", "duplicate", "entity", "minimum", "maximum", "zero_semantics"] +) +def test_live_ready_money_domain_must_match_the_pinned_source_contract(change): + domain = SimpleNamespace( + name="DIV_VAL", + entity="person", + minimum=0, + maximum=999999, + zero_semantics="none_or_niu_not_distinguishable_from_amount_alone", + ) + fields = [domain] + ready = SimpleNamespace( + bindings=SimpleNamespace(spec=SimpleNamespace(fields=fields)) + ) + owner._domain_agreement(ready) + if change == "missing": + fields.clear() + elif change == "duplicate": + fields.append(copy.copy(domain)) + else: + setattr( + domain, change, "wrong" if change in ("entity", "zero_semantics") else 1 + ) + with pytest.raises(ValueError, match="MONEY_DOMAIN_"): + owner._domain_agreement(ready) + + +@pytest.mark.parametrize("change", ["amount_bit", "validity", "literal", "missing"]) +def test_current_money_total_requires_exact_bits_and_literal_validity(change): + field = SimpleNamespace(amounts=np.array([140.0, 0.0]), validity=np.array([1, 0])) + ready = SimpleNamespace(field=lambda name: field) + positions, literals = np.array([0, 1]), ["140", ""] + assert owner._compare_total(ready, positions, literals) is field + if change == "amount_bit": + field.amounts[0] = np.nextafter(140.0, np.inf) + elif change == "validity": + field.validity[1] = 1 + elif change == "literal": + literals[0] = "-1" + else: + literals[1] = "0" + with pytest.raises(ValueError, match="TOTAL_"): + owner._compare_total(ready, positions, literals) + + +def test_physical_seal_covers_nullable_boolean_and_money_hidden_backing(): + raw = pd.DataFrame([row(DIV_VAL="0", SUR_YN="1")]) + values = owner.CurrentAsecDividendValues( + owner.project_dividend_literals(raw), raw, {"descriptive": True} + ) + original = owner.dividend_values_seal(values) + assert owner.dividend_values_seal(copy.deepcopy(values)) == original + for name in ("DIV_VAL_amount", "survivor_property_route_clear"): + changed = copy.deepcopy(values) + array = changed.person[name].array + assert array._mask[0] + array._data[0] = not array._data[0] if name.endswith("clear") else 99.0 + assert owner.dividend_values_seal(changed) != original + + +def dividend_arguments(tmp_path, monkeypatch, *, fraction=Fraction(1)): + """Amend complete invented literals and retained money before any issuance.""" + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch, fraction=fraction, zero=False) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + literals = { + 105: row(I_DIVYN="5"), + 106: row(A_AGE="14", DIV_VAL="0", DIV_YN="0", SUR_YN="0"), + 107: row(A_AGE="70", DIV_VAL="500", SUR_YN="1", SUR_SC2="8", TDIV_VAL="1"), + 108: row(DIV_VAL="0", SUR_YN="1", SUR_SC1="10"), + } + changes = {} + for name in ("DIV_VAL", "A_AGE"): + values = people[name].to_numpy(copy=True) + for person_id, literal in literals.items(): + values[ids == person_id] = int(literal[name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in ("DIV_VAL", "A_AGE"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + for name in ( + *owner.RECEIPT_ENTRIES, + *owner.SURVIVOR_ENTRIES, + *owner.ALLOCATION_ENTRIES, + *owner.TOPCODE_ENTRIES, + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row())[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "dividend-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(tmp_path, monkeypatch, *, fraction=Fraction(1)): + return owner.routing.source.prepare_authenticated_survey_population( + **dividend_arguments(tmp_path, monkeypatch, fraction=fraction) + ) + + +def test_actual_owner_qualified_source_values_and_requalification( + tmp_path, monkeypatch +): + parent = prepared(tmp_path, monkeypatch) + result = owner.qualify_current_asec_dividend(parent) + values = result.person.set_index("native_person_id") + assert values.loc[105, "DIV_VAL_amount"] == 140 + assert values.loc[105, "I_DIVYN_codebook_status"] == "header_reference_conflict" + assert values.loc[106, "DIV_VAL_reporting_status"] == "outside_reporting_universe" + assert values.loc[107, "DIV_VAL_amount"] == 500 + assert not bool(values.loc[107, "survivor_property_route_clear"]) + assert pd.isna(values.loc[108, "survivor_property_route_clear"]) + assert not values.loc[108, "DIV_VAL_amount_known"] + assert { + "DIV_VAL_parent_statuses", + "DIV_VAL_parent_validity", + "DIV_VAL_parent_zero_origin", + } <= set(values) + assert owner.dividend_values_seal( + owner.qualify_current_asec_dividend(parent) + ) == owner.dividend_values_seal(result) + assert ( + not result.evidence["source_admission_issued"] + and not result.evidence["survivor_amounts_read"] + ) + assert result.evidence == json.loads(json.dumps(result.evidence)) + with pytest.raises(ValueError): + owner.qualify_current_asec_dividend(copy.copy(parent)) + + +@pytest.mark.parametrize( + "coordinate", ["PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "DIV_VAL"] +) +def test_actual_owner_refuses_original_coordinate_or_money_change( + tmp_path, monkeypatch, coordinate +): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if coordinate == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(coordinate)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_dividend(parent) + + +def test_actual_owner_compares_unselected_current_year_amounts(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch, fraction=Fraction(1, 2)) + values = owner.qualify_current_asec_dividend(parent) + omitted = set(values.asec_literals.index) - set(values.person.native_person_id) + assert omitted + key = values.asec_literals.loc[next(iter(omitted)), "PERIDNUM"] + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + raw.loc[key, "DIV_VAL"] = "997" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError, match="TOTAL_BITS"): + owner.qualify_current_asec_dividend(parent) + + +def test_actual_owner_requalifies_after_capture_io(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + calls = [] + + def changed(*args): + raw = capture(*args) + table = parent._checked()[2].frame.person + table.iloc[0, table.columns.get_loc("age")] += 1 + calls.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_dividend(parent) + assert calls + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "route_mask", "literal", "evidence"] +) +def test_final_physical_seal_covers_complete_returned_values( + tmp_path, monkeypatch, mutation +): + parent = prepared(tmp_path, monkeypatch) + seal, calls = owner.dividend_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + if mutation == "amount_bit": + array = value.person.DIV_VAL_amount.array + i = np.flatnonzero(~array._mask)[0] + array._data[i] = np.nextafter(array._data[i], np.inf) + elif mutation == "masked_backing": + array = value.person.DIV_VAL_amount.array + array._data[np.flatnonzero(array._mask)[0]] = 17.0 + elif mutation == "route_mask": + array = value.person.survivor_property_route_clear.array + array._mask[np.flatnonzero(array._mask)[0]] = False + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("SUR_SC1") + ] = "8" + else: + value.evidence["survivor_amounts_read"] = True + return before + + monkeypatch.setattr(owner, "dividend_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_dividend(parent) diff --git a/packages/microcosm-build/tests/test_us_current_asec_immigration_assignment.py b/packages/microcosm-build/tests/test_us_current_asec_immigration_assignment.py new file mode 100644 index 000000000..52641f673 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_immigration_assignment.py @@ -0,0 +1,559 @@ +"""A real retained source draw on tiny invented Census/parent fixture bytes.""" + +import hashlib +import json +import shutil +import sys +from dataclasses import asdict, replace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_immigration_assignment as owner +from microcosm.frame import WeightKind + + +def source_arguments(tmp_path, monkeypatch): + import test_us_current_asec_immigration_donor as fixture + + from microcosm.build.us_runtime import asec_person_income_source as restoration + + full, partial, unselected = fixture.source_arguments(tmp_path, monkeypatch) + donor = owner.donor_owner + root = full["source_dir"] / "asec" + pins, paths = [], {} + for year, member, archive, *_ in donor.literals.original.asec._MEMBER_PINS: + path = root / member + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + if year == 2024: + adult, teen = raw.A_AGE.eq("55"), raw.A_AGE.eq("15") + assert adult.sum() == teen.sum() == 1 + raw.loc[adult | teen, "PRCITSHP"] = "5" + raw.loc[adult, ["PENATVTY", "PEINUSYR", "A_LFSR"]] = ["303", "25", "1"] + raw.loc[teen, ["PENATVTY", "PEINUSYR"]] = ["164", "28"] + raw.to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + shutil.copyfile(path, partial["source_dir"] / "asec" / member) + for module in ( + donor.literals.original.asec, + restoration, + donor.demographics.demographic, + ): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "immigration-assignment-source" + restoration.restore_asec_person_income_source( + root / "parent.h5", + root / "household-attachment.h5", + member_paths=paths, + output_dir=output, + ) + for args in (full, partial): + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, + args["source_dir"] / "asec" / "person-income-attachment.h5", + ) + return full, partial, unselected + + +@pytest.fixture(scope="module") +def actual(tmp_path_factory): + with pytest.MonkeyPatch.context() as patch: + full_args, partial_args, unselected = source_arguments( + tmp_path_factory.mktemp("asec-immigration-assignment"), patch + ) + full = owner.source.prepare_authenticated_survey_population(**full_args) + partial = owner.source.prepare_authenticated_survey_population(**partial_args) + left = owner.donor_owner.borrow_full_asec_immigration_donor(full) + right = owner.donor_owner.borrow_full_asec_immigration_donor(partial) + originals = [ + (donor, owner.source._frame_identity(donor.frame)) + for donor in (left, right) + ] + calls, draws = [], [] + previous = sys.getprofile() + + def observe(frame, event, arg): + # Read only invented fixture call inputs, without replacing the live + # rule function or bypassing the owner's implementation fences. + if event != "call": + return + if frame.f_code is owner.source_runtime.run_source_stage.__code__: + calls.append(dict(frame.f_locals)) + elif frame.f_code is owner.rules._stable_person_draws.__code__: + draws.append( + frame.f_locals["person"].loc[:, list(owner._LINEAGE)].copy() + ) + + try: + sys.setprofile(observe) + assigned = owner.assign_full_asec_immigration(left, seed=42) + finally: + sys.setprofile(previous) + smaller = owner.assign_full_asec_immigration(right, seed=42) + yield SimpleNamespace( + full=full, + partial=partial, + left=left, + right=right, + assigned=assigned, + smaller=smaller, + unselected=unselected, + calls=calls, + draws=draws, + ) + for donor, seal in originals: + donor.validate() + assert owner.source._frame_identity(donor.frame) == seal + + +def test_real_full_assignment_survives_selection_with_original_pairs(actual): + left, right = actual.assigned, actual.smaller + left.validate() + right.validate() + assert ( + actual.unselected + not in actual.partial._checked()[2].native[1].frame.person.person_id.tolist() + ) + assert actual.unselected in right.pairs.index + assert ( + right.pairs.loc[actual.unselected, "immigration_status_str"] + == "PAROLED_ONE_YEAR" + ) + assert ( + right.pairs.loc[actual.unselected, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" + ) + assert sorted(right.frame.person.immigration_status_str.tolist()) == [ + "CITIZEN", + "CITIZEN", + "PAROLED_ONE_YEAR", + "UNDOCUMENTED", + ] + assert owner.source._frame_identity(left.frame) == owner.source._frame_identity( + right.frame + ) + pd.testing.assert_frame_equal(left.pairs, right.pairs) + pd.testing.assert_frame_equal( + right.pairs.loc[:, owner.donor_owner.literals.original.ASEC_KEYS], + actual.right.raw.loc[:, owner.donor_owner.literals.original.ASEC_KEYS], + ) + assert right.pairs.income_year.eq(2024).all() + assert right.pairs.index.equals(actual.right.raw.index) + + +def test_exactly_one_call_uses_original_design_weights_and_observation_year(actual): + assert len(actual.calls) == 1 + call = actual.calls[0] + assert call["config"].seed == 42 + assert call["config"].target_year == 2025 + handler = call["operation_handlers"]["derive_immigration_status"] + assert handler.func is owner.rules.derive_us_immigration_status_from_manifest + assert handler.keywords["observation_year_column"] == owner.OBSERVATION_YEAR_COLUMN + person = call["tables"]["person"] + assert person[owner.OBSERVATION_YEAR_COLUMN].eq(2025).all() + assert actual.left.frame.weights_for("household").kind is WeightKind.DESIGN + assert actual.left.frame.weights_for("household").values.tolist() == [ + 2552.12, + 100.0, + ] + np.testing.assert_array_equal( + person.person_weight, + actual.left.frame.resolve_weights("person").values, + ) + source_stage = asdict(call["stage"]) + assert ( + json.loads(owner.source._encode(source_stage)) + == json.loads(actual.assigned.receipt)["control_stage"] + ) + assert actual.draws + expected = pd.DataFrame( + { + "source_year": 2024, + "source_household_id": actual.left.frame.person.PH_SEQ.map(int), + "source_person_id": actual.left.frame.person.PERIDNUM, + } + ) + for keys in actual.draws: + pd.testing.assert_frame_equal(keys, expected) + assert not {"WSAL_VAL", "SEMP_VAL"} & set(person) + assert "person_weight" not in actual.assigned.frame.person + + +def test_every_nonowned_cell_and_typed_frame_property_survives(actual): + source, assigned = actual.right.frame, actual.smaller.frame + for entity in source.entities: + table = assigned.table(entity) + if entity == "person": + table = table.drop(columns=list(owner.rules.US_IMMIGRATION_OUTPUT_COLUMNS)) + pd.testing.assert_frame_equal(table, source.table(entity)) + assert assigned.weighted_entities == source.weighted_entities + for entity in source.weighted_entities: + assert assigned.weights_for(entity) is source.weights_for(entity) + pd.testing.assert_series_equal(assigned.strata, source.strata) + assert assigned.metadata == source.metadata + assert assigned.mass_log == source.mass_log + assert ( + tuple(owner.rules.US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS) + == owner.donor_owner.literals.ASEC_VALUE_COLUMNS + ) + + +def test_captured_stage_matches_maintained_frame_operator(actual): + person = actual.calls[0]["tables"]["person"].drop(columns=["person_weight"]) + expected = owner.rules.with_us_immigration_inputs( + owner._with_person(actual.left.frame, person), + seed=42, + time_period=2025, + person_weight_scale=1.0, + observation_year_column=owner.OBSERVATION_YEAR_COLUMN, + ) + columns = list(owner.rules.US_IMMIGRATION_OUTPUT_COLUMNS) + pd.testing.assert_frame_equal( + actual.assigned.frame.person.loc[:, columns], expected.person.loc[:, columns] + ) + + +def test_receipt_binds_source_rules_and_unqualified_mixed_date_assumptions(actual): + assigned = actual.smaller + receipt = json.loads(assigned.receipt) + assert receipt["purpose"] == "development_imputation" + assert receipt["status_assignment_performed"] + for flag in ( + "observed_legal_status", + "national_stock_alignment_qualified", + "composition_gate_performed", + "prior_income_columns_consumed", + "source_admission_issued", + ): + assert receipt[flag] is False + assert receipt["income_year"] == 2024 and receipt["observation_year"] == 2025 + assert receipt["person_weight_scale"] == 1.0 + assert receipt["weight_source"] == "original_HSUP_WGT/100" + assert receipt["person_weight_authority"] == "none" + assert receipt["readset"] == list(owner.donor_owner.literals.ASEC_VALUE_COLUMNS) + assert receipt["donor_receipt_sha256"] == owner.source._sha(actual.right.receipt) + assert receipt["control_stage"] == json.loads(owner._stage()) + assert receipt["control_stage_sha256"] == owner.source._sha(owner._stage()) + assert receipt["implementation"] == json.loads(owner._implementation()) + assert ( + receipt["control_reference_bases"]["humanitarian_status_stocks.tps"] + == "as of 2025-03-31" + ) + assert ( + "not qualified" in receipt["control_reference_bases"]["undocumented_students"] + ) + + +@pytest.mark.parametrize("seed", [True, -1, 2**64, 4.5, "42", None]) +def test_invalid_seed_refuses(actual, seed): + with pytest.raises(ValueError, match="ASSIGNMENT_SEED"): + owner.assign_full_asec_immigration(actual.right, seed=seed) + + +def test_detached_inputs_and_copies_cannot_gain_authority(actual): + for value in (actual.right.frame, actual.right.receipt, actual.partial): + with pytest.raises(ValueError, match="DONOR_TYPE"): + owner.assign_full_asec_immigration(value, seed=42) + with pytest.raises(ValueError, match="ISSUED_OWNER_REQUIRED"): + owner.assign_full_asec_immigration(replace(actual.right), seed=42) + with pytest.raises(ValueError, match="ISSUED_OWNER_REQUIRED"): + replace(actual.smaller).validate() + + +@pytest.mark.parametrize( + "columns", + [ + ("ssn_card_type",), + ("immigration_status_str",), + owner.rules.US_IMMIGRATION_OUTPUT_COLUMNS, + ], +) +def test_preexisting_partial_or_complete_outputs_never_passthrough(actual, columns): + people = actual.right.frame.person + try: + for name in columns: + people[name] = "unowned" + with pytest.raises(ValueError, match="PREEXISTING_OUTPUT"): + owner.assign_full_asec_immigration(actual.right, seed=42) + finally: + people.drop(columns=list(columns), inplace=True) + actual.right.validate() + + +@pytest.mark.parametrize( + "surface,column", + [ + ("pairs", "ssn_card_type"), + ("pairs", "PERIDNUM"), + ("frame", "immigration_status_str"), + ("frame", "A_AGE"), + ("donor_raw", "A_LFSR"), + ("donor_frame", "A_AGE"), + ], +) +def test_retained_input_output_and_keys_are_not_mutable_authority( + actual, surface, column +): + assigned = actual.smaller + table = { + "pairs": assigned.pairs, + "frame": assigned.frame.person, + "donor_raw": actual.right.raw, + "donor_frame": actual.right.frame.person, + }[surface] + index = table.index[0] + previous = table.at[index, column] + try: + table.at[index, column] = 99 if column == "A_AGE" else "changed" + with pytest.raises(ValueError): + assigned.validate() + finally: + table.at[index, column] = previous + assigned.validate() + + +@pytest.mark.parametrize("field", ["frame", "pairs", "receipt"]) +def test_owner_identity_and_projection_identity_are_retained(actual, field): + assigned = actual.smaller + previous = getattr(assigned, field) + replacement = ( + owner._with_person(previous, previous.person.copy(deep=True)) + if field == "frame" + else previous.copy(deep=True) + if field == "pairs" + else b"{}" + ) + try: + object.__setattr__(assigned, field, replacement) + with pytest.raises(ValueError, match="ASSIGNMENT_PROJECTION_CHANGED"): + assigned.validate() + finally: + object.__setattr__(assigned, field, previous) + assigned.validate() + + +@pytest.mark.parametrize( + "surface", ["table_attributes", "weight_kind", "weight_storage"] +) +def test_decorated_frame_and_typed_weights_remain_sealed(actual, surface): + assigned = actual.smaller + weights = assigned.frame.weights_for("household") + before_kind = weights.kind + before_writeable = weights.values.flags.writeable + before_attrs = dict(assigned.frame.person.attrs) + try: + if surface == "table_attributes": + assigned.frame.person.attrs["changed"] = True + elif surface == "weight_kind": + object.__setattr__(weights, "kind", WeightKind.IMPORTANCE) + else: + weights.values.setflags(write=not before_writeable) + with pytest.raises(ValueError): + assigned.validate() + finally: + assigned.frame.person.attrs.clear() + assigned.frame.person.attrs.update(before_attrs) + object.__setattr__(weights, "kind", before_kind) + weights.values.setflags(write=before_writeable) + assigned.validate() + + +def test_issuance_refuses_donor_mutation_after_rule_draw(actual, monkeypatch): + reader = Path.read_text + raw = actual.right.raw + index, before = raw.index[0], raw.A_LFSR.iloc[0] + reads = [] + + def late(path, *args, **kwargs): + result = reader(path, *args, **kwargs) + if path.name == "source_stages.json": + reads.append(True) + # The second read is the issuance validator, after the actual draw. + if len(reads) == 2: + raw.at[index, "A_LFSR"] = "changed" + return result + + try: + with monkeypatch.context() as patch: + patch.setattr(Path, "read_text", late) + with pytest.raises(ValueError): + owner.assign_full_asec_immigration(actual.right, seed=42) + assert len(reads) == 2 + finally: + raw.at[index, "A_LFSR"] = before + actual.right.validate() + + +def test_new_runtime_keeps_provenance_guards(): + import test_us_spine_blindness as guard + + path = Path(owner.__file__) + assert path.name in guard._US_LAUNCH_GRAPH_RUNTIME_MODULES + assert path.name not in guard._SOURCE_SPINE_PROVENANCE_OWNERS + assert not guard._non_owner_source_spine_accesses(path.name, path.read_text()) + + +@pytest.mark.parametrize("changed_read", [1, 2]) +def test_transient_manifest_change_cannot_misrepresent_consumed_controls( + actual, monkeypatch, changed_read +): + reader = Path.read_text + reads = [] + + def transient(path, *args, **kwargs): + text = reader(path, *args, **kwargs) + if path.name == "source_stages.json": + reads.append(True) + if len(reads) == changed_read: + # This control changes the invented recipient's parole draw. + return text.replace('"target": 158000', '"target": 0') + return text + + with monkeypatch.context() as patch: + patch.setattr(Path, "read_text", transient) + with pytest.raises(ValueError, match="CONTROL_STAGE_CHANGED"): + owner.assign_full_asec_immigration(actual.right, seed=42) + assert len(reads) == 2 + actual.right.validate() + + +def test_final_manifest_io_cannot_invalidate_retained_source_files( + tmp_path, monkeypatch +): + full_args, _, _ = source_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**full_args) + donor = owner.donor_owner.borrow_full_asec_immigration_donor(prepared) + assigned = owner.assign_full_asec_immigration(donor, seed=42) + source_path = full_args["source_dir"] / "asec" / "pppub25.csv" + original = source_path.read_bytes() + reader = Path.read_text + changed = [] + + def late(path, *args, **kwargs): + text = reader(path, *args, **kwargs) + if path.name == "source_stages.json" and not changed: + changed.append(True) + source_path.write_bytes(original + b"\n") + return text + + with monkeypatch.context() as patch: + patch.setattr(Path, "read_text", late) + with pytest.raises(ValueError, match="SOURCE_FILES_CHANGED"): + assigned.validate() + assert changed + + +def test_final_manifest_io_cannot_invalidate_retained_donor_code(tmp_path, monkeypatch): + # Point implementation-byte validation at an identical scratch copy before + # issuance, so the mutation never edits a maintained implementation file. + copied_code = tmp_path / "retained-donor-implementation.py" + copied_code.write_bytes(Path(owner.donor_owner.__file__).read_bytes()) + monkeypatch.setattr(owner.donor_owner, "__file__", str(copied_code)) + full_args, _, _ = source_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**full_args) + donor = owner.donor_owner.borrow_full_asec_immigration_donor(prepared) + assigned = owner.assign_full_asec_immigration(donor, seed=42) + original = copied_code.read_bytes() + reader = Path.read_text + changed = [] + + def late(path, *args, **kwargs): + text = reader(path, *args, **kwargs) + if path.name == "source_stages.json" and not changed: + changed.append(True) + copied_code.write_bytes(original + b"\n# changed retained code\n") + return text + + with monkeypatch.context() as patch: + patch.setattr(Path, "read_text", late) + with pytest.raises(ValueError, match="IMPLEMENTATION_FILES_CHANGED"): + assigned.validate() + assert changed + + +@pytest.mark.parametrize("surface", ["function", "defaults", "constant"]) +def test_changed_live_rule_implementation_refuses(actual, monkeypatch, surface): + with monkeypatch.context() as patch: + if surface == "function": + patch.setattr( + owner.rules, + "derive_us_immigration_status_from_manifest", + lambda *a, **kw: None, + ) + elif surface == "defaults": + patch.setattr( + owner.rules.derive_us_immigration_status_from_manifest, + "__kwdefaults__", + {}, + ) + else: + patch.setitem(owner.rules._PAROLE_ORIGIN_CODES, "ukraine", (303,)) + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + actual.smaller.validate() + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + owner.assign_full_asec_immigration(actual.right, seed=42) + actual.smaller.validate() + + +@pytest.mark.parametrize( + "surface", ["pair", "donor", "function", "code_bytes", "stage"] +) +def test_final_foreign_io_and_dependency_changes_refuse(actual, monkeypatch, surface): + assigned = actual.smaller + index = assigned.pairs.index[0] + pair_before = assigned.pairs.at[index, "ssn_card_type"] + raw_before = actual.right.raw.at[index, "A_LFSR"] + reader = Path.read_bytes + changed = [] + + def late(path): + result = reader(path) + if not changed and str(path) == owner.__file__: + changed.append(True) + if surface == "pair": + assigned.pairs.at[index, "ssn_card_type"] = "changed" + elif surface == "donor": + actual.right.raw.at[index, "A_LFSR"] = "changed" + elif surface == "function": + monkeypatch.setattr( + owner.rules, "_stable_person_draws", lambda *a, **k: None + ) + elif surface == "code_bytes": + return result + b"\n# changed implementation\n" + return result + + try: + with monkeypatch.context() as patch: + patch.setattr(Path, "read_bytes", late) + if surface == "stage": + # Stage loader reads text, after final implementation-byte I/O. + text_reader = Path.read_text + + def changed_stage(path, *args, **kwargs): + text = text_reader(path, *args, **kwargs) + if path.name == "source_stages.json": + return text.replace('"target": 9700000', '"target": 9700001') + return text + + patch.setattr(Path, "read_text", changed_stage) + with pytest.raises(ValueError): + assigned.validate() + assert changed + finally: + assigned.pairs.at[index, "ssn_card_type"] = pair_before + actual.right.raw.at[index, "A_LFSR"] = raw_before + if surface != "function": + assigned.validate() diff --git a/packages/microcosm-build/tests/test_us_current_asec_immigration_donor.py b/packages/microcosm-build/tests/test_us_current_asec_immigration_donor.py new file mode 100644 index 000000000..84d63c5d6 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_immigration_donor.py @@ -0,0 +1,461 @@ +"""Full-source donor controls on tiny genuine issuers over invented bytes.""" + +import hashlib +import json +import shutil +from dataclasses import replace +from fractions import Fraction +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_immigration_donor as owner +from microcosm.frame import US_SCHEMA, WeightKind + + +def source_arguments(tmp_path, monkeypatch, *, defect=None): + import test_us_current_survey_immigration_source as fixture + + from microcosm.build.us_runtime import asec_person_income_source as restoration + from microcosm.build.us_runtime import native_household_origin as origin + + full, partial, donor = fixture.source_arguments(tmp_path, monkeypatch) + root = full["source_dir"] / "asec" + pins, paths = [], {} + for year, member, archive, *_ in owner.literals.original.asec._MEMBER_PINS: + path = root / member + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + raw["PENATVTY"] = "057" + raw["A_MARITL"] = "7" + raw["PEAFEVER"] = raw.A_AGE.map(lambda age: "2" if int(age) >= 17 else "-1") + for name in ("MCARE", "CAID", "IHSFLG", "CHAMPVA", "MIL"): + raw[name] = "2" + raw["A_SEX"] = raw.A_LINENO.map({"1": "1", "2": "2"}) + raw["AXSEX"] = raw.A_LINENO.map({"1": "0", "2": "4"}) + raw["A_EXPRRP"] = raw.A_LINENO.map({"1": "1", "2": "5"}) + raw["P_SEQ"] = raw.A_LINENO + if year == 2024: + selected = raw.PERIDNUM.eq(str(donor - 100).zfill(22)) + assert selected.sum() == 1 + if defect == "blank-owning-token": + raw.loc[selected, "MCARE"] = "" + elif defect == "unresolved-owning-code": + raw.loc[selected, "PENATVTY"] = "-4" + elif defect == "unresolved-sex": + raw.loc[selected, "AXSEX"] = "1" + elif defect == "missing-person": + raw = raw.loc[~selected] + elif defect == "duplicate-person": + raw = pd.concat([raw, raw.loc[selected]], ignore_index=True) + raw.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(raw), + len(data), + ) + ) + paths[year] = path + shutil.copyfile(path, partial["source_dir"] / "asec" / member) + for module in ( + owner.literals.original.asec, + restoration, + owner.demographics.demographic, + ): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "full-donor-restoration" + restoration.restore_asec_person_income_source( + root / "parent.h5", + root / "household-attachment.h5", + member_paths=paths, + output_dir=output, + ) + for args in (full, partial): + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, + args["source_dir"] / "asec" / "person-income-attachment.h5", + ) + path = root / "hhpub25.csv" + households = pd.read_csv(path, dtype=str, keep_default_na=False) + households["GESTFIPS"] = households.H_SEQ.map({"00007": "06", "00008": "36"}) + state_defects = {"blank-state": "", "unnamed-state": "03", "territory-state": "72"} + if defect in state_defects: + source_household = "00007" if donor == 105 else "00008" + households.loc[households.H_SEQ.eq(source_household), "GESTFIPS"] = ( + state_defects[defect] + ) + if defect == "unresolved-household-weight": + source_household = "00007" if donor == 105 else "00008" + households.loc[households.H_SEQ.eq(source_household), "HSUP_WGT"] = "" + households.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + monkeypatch.setattr( + origin, + "_ASEC_MEMBER_PINS", + tuple( + replace( + pin, + member_sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + for pin in origin._ASEC_MEMBER_PINS + ), + ) + shutil.copyfile(path, partial["source_dir"] / "asec" / "hhpub25.csv") + return full, partial, donor + + +@pytest.fixture(scope="module") +def actual(tmp_path_factory): + with pytest.MonkeyPatch.context() as patch: + full_args, partial_args, donor = source_arguments( + tmp_path_factory.mktemp("full-asec-donor"), patch + ) + full = owner.source.prepare_authenticated_survey_population(**full_args) + partial = owner.source.prepare_authenticated_survey_population(**partial_args) + originals = [ + (prepared, owner.source._frame_identity(prepared._checked()[2].frame)) + for prepared in (full, partial) + ] + left = owner.borrow_full_asec_immigration_donor(full) + right = owner.borrow_full_asec_immigration_donor(partial) + yield SimpleNamespace( + full=full, partial=partial, left=left, right=right, unselected=donor + ) + for prepared, seal in originals: + assert owner.source._frame_identity(prepared._checked()[2].frame) == seal + + +def test_full_donor_is_selection_invariant_and_exactly_design_weighted(actual): + left, right = actual.left, actual.right + left.validate() + right.validate() + assert left.frame.n("person") == 4 and left.frame.n("household") == 2 + for entity in left.frame.entities: + pd.testing.assert_frame_equal( + left.frame.table(entity), right.frame.table(entity) + ) + pd.testing.assert_frame_equal(left.raw, right.raw) + pd.testing.assert_frame_equal(left.households, right.households) + assert actual.unselected in right.frame.person.person_id.tolist() + selected_asec = actual.partial._checked()[2].native[1].frame.person.person_id + assert actual.unselected not in selected_asec.tolist() + assert right.frame.weights_for("household").kind is WeightKind.DESIGN + assert right.frame.weights_for("household").values.tolist() == [2552.12, 100.0] + assert [ + Fraction(r.fraction_numerator, r.fraction_denominator) + for r in right.households.itertuples() + ] == [Fraction(63803, 25), Fraction(100)] + assert right.households.HSUP_WGT.tolist() == ["000255212", "000010000"] + assert right.frame.table("household").state_fips.tolist() == [6, 36] + assert right.frame.person.is_female.tolist() == [False, True, False, True] + assert np.array_equal(right.frame.person.age, right.frame.person.A_AGE) + assert right.raw.A_LFSR.tolist().count(" 7 ") == 1 + assert right.frame.person.A_LFSR.tolist().count(7) == 1 + assert right.frame.person.PEAFEVER.tolist().count(-1) == 3 + for frame in (left.frame, right.frame): + assert tuple(frame.person) == ( + US_SCHEMA.person_id_column, + *(US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities), + *owner.literals.ASEC_VALUE_COLUMNS, + *owner.literals.original.ASEC_KEYS, + "age", + "is_female", + ) + assert not { + "WSAL_VAL", + "SEMP_VAL", + "employment_income", + "person_weight", + "ssn_card_type", + "immigration_status_str", + } & set(frame.person) + receipt = json.loads(right.receipt) + assert receipt["observation_year"] == 2025 and receipt["income_year"] == 2024 + assert receipt["weight_source"] == "original_HSUP_WGT/100" + assert receipt["person_weight_authority"] == "none" + assert "person_weight_scale" not in receipt + assert not receipt["national_stock_alignment_qualified"] + assert not receipt["status_assignment_performed"] + + +def raw_row(): + values = {name: "0" for name in owner.literals.ASEC_COLUMNS} + values.update( + PH_SEQ="00007", + PERIDNUM="0000000000000000000001", + A_LINENO="1", + PRCITSHP="1", + PENATVTY="057", + A_MARITL="7", + PEAFEVER="-1", + SPM_CAPHOUSESUB="0.00", + ) + return pd.DataFrame([values], columns=owner.literals.ASEC_COLUMNS) + + +@pytest.mark.parametrize("field", owner.literals.ASEC_VALUE_COLUMNS) +@pytest.mark.parametrize("token", ["", " ", "not-a-number"]) +def test_every_owning_field_refuses_blank_or_malformed_before_coercion(field, token): + raw = raw_row() + raw.loc[0, field] = token + original = raw.copy(deep=True) + with pytest.raises(ValueError, match="MISSING_TOKEN|MALFORMED_TOKEN"): + owner._numeric_literals(raw) + pd.testing.assert_frame_equal(raw, original) + + +@pytest.mark.parametrize( + "field,token", + [ + ("PRCITSHP", "-4"), + ("PRCITSHP", "0"), + ("PENATVTY", "999"), + ("PENATVTY", "-1"), + ("PEINUSYR", "29"), + ("A_AGE", "81"), + ("A_MARITL", "0"), + ("A_SPOUSE", "17"), + ("A_HSCOL", "3"), + ("A_LFSR", "5"), + ("MCARE", "3"), + ("PEN_SC1", "9"), + ("RESNSS2", "9"), + ("PEIO1COW", "9"), + ("PEIO1COW", "-4"), + ("A_MJOCC", "-1"), + ("PEAFEVER", "0"), + ("SPM_CAPHOUSESUB", "100000"), + ("SPM_CAPHOUSESUB", "0.01"), + ], +) +def test_unexplained_or_out_of_domain_source_codes_refuse(field, token): + raw = raw_row() + raw.loc[0, field] = token + with pytest.raises(ValueError, match="UNRESOLVED_DOMAIN"): + owner._numeric_literals(raw) + + +@pytest.mark.parametrize( + "field,token,expected", + [ + ("PEINUSYR", "27", 27), + ("PEINUSYR", "28", 28), + ("PEAFEVER", "-1", -1), + ("A_LFSR", " 7 ", 7), + ("A_AGE", "80", 80), + ("A_AGE", "85", 85), + ("SPM_CAPHOUSESUB", "99999.00", 99999), + ], +) +def test_legitimate_niu_recent_arrival_and_topcoded_tokens_survive( + field, token, expected +): + raw = raw_row() + raw.loc[0, field] = token + assert owner._numeric_literals(raw).loc[0, field] == expected + assert raw.loc[0, field] == token + + +@pytest.mark.parametrize("field", ["raw", "households", "frame", "receipt"]) +def test_copies_or_changed_borrowed_output_cannot_gain_authority(actual, field): + value = actual.right + previous = getattr(value, field) + replacement = ( + previous.copy(deep=True) + if isinstance(previous, pd.DataFrame) + else (b"changed" if field == "receipt" else actual.left.frame) + ) + try: + object.__setattr__(value, field, replacement) + with pytest.raises(ValueError, match="DONOR_PROJECTION_CHANGED"): + value.validate() + finally: + object.__setattr__(value, field, previous) + value.validate() + with pytest.raises(ValueError, match="ISSUED_OWNER_REQUIRED"): + replace(value).validate() + + +@pytest.mark.parametrize("surface", ["raw", "households", "frame"]) +def test_in_place_output_mutation_refuses(actual, surface): + value = actual.right + table, column = ( + (value.raw, "MCARE") + if surface == "raw" + else (value.households, "numerator") + if surface == "households" + else (value.frame.person, "A_LFSR") + ) + index = table.index[0] + before = table.at[index, column] + try: + table.at[index, column] = "changed" if surface == "raw" else 999 + with pytest.raises(ValueError, match="DONOR_PROJECTION_CHANGED"): + value.validate() + finally: + table.at[index, column] = before + value.validate() + + +@pytest.mark.parametrize("surface", ["raw", "frame", "owner", "implementation"]) +def test_final_foreign_io_mutation_is_refused(actual, monkeypatch, surface): + value = actual.right + native = owner._ISSUED[id(value)][2].native + raw_index = value.raw.index[0] + raw_before = value.raw.at[raw_index, "MCARE"] + age_before = value.frame.person.A_AGE.copy(deep=True) + payload_before = native.payload + reader = Path.read_bytes + changed = [] + + def late(path): + result = reader(path) + if not changed and str(path) == owner.__file__: + changed.append(True) + if surface == "raw": + value.raw.at[raw_index, "MCARE"] = "changed" + elif surface == "frame": + value.frame.person.loc[:, "A_AGE"] += 1 + elif surface == "owner": + object.__setattr__(native, "payload", b"changed") + else: + monkeypatch.setattr(owner, "_numeric_literals", lambda raw: raw) + return result + + try: + with monkeypatch.context() as patch: + patch.setattr(Path, "read_bytes", late) + with pytest.raises(ValueError): + value.validate() + assert changed + finally: + value.raw.at[raw_index, "MCARE"] = raw_before + value.frame.person.loc[:, "A_AGE"] = age_before + object.__setattr__(native, "payload", payload_before) + # Outer monkeypatch restores implementation at teardown. + if surface != "implementation": + value.validate() + + +@pytest.mark.parametrize( + "defect,reason", + [ + ("blank-owning-token", "MISSING_TOKEN_MCARE"), + ("unresolved-owning-code", "UNRESOLVED_DOMAIN_PENATVTY"), + ], +) +def test_genuine_issuer_refuses_incomplete_or_unresolved_original_source( + tmp_path, monkeypatch, defect, reason +): + _, partial_args, _ = source_arguments(tmp_path, monkeypatch, defect=defect) + prepared = owner.source.prepare_authenticated_survey_population(**partial_args) + literal_owner = owner.literals.qualify_current_survey_immigration(prepared) + literal_owner.validate() + column, token = ( + ("MCARE", "") if defect == "blank-owning-token" else ("PENATVTY", "-4") + ) + assert token in literal_owner.asec_full_raw[column].tolist() + with pytest.raises(ValueError, match=reason): + owner.borrow_full_asec_immigration_donor(prepared) + + +@pytest.mark.parametrize( + "defect,reason", + [ + ("unresolved-sex", "SEX_UNRESOLVED"), + ("blank-state", "STATE_UNRESOLVED"), + ("unnamed-state", "STATE_UNRESOLVED"), + ("territory-state", "STATE_UNRESOLVED"), + ], +) +def test_genuine_preparation_does_not_admit_unresolved_full_donor_predictors( + tmp_path, monkeypatch, defect, reason +): + # The defect is on an original donor omitted by support selection. Actual + # preparation must succeed before this tests the full-donor boundary. + _, partial_args, _ = source_arguments(tmp_path, monkeypatch, defect=defect) + prepared = owner.source.prepare_authenticated_survey_population(**partial_args) + prepared.checked_view() + with pytest.raises(ValueError, match=reason): + owner.borrow_full_asec_immigration_donor(prepared) + + +@pytest.mark.parametrize("coordinate", ["A_AGE", "A_LINENO", "source_household_id"]) +def test_independent_demographic_coordinate_join_refuses_mismatch(coordinate): + # Pure join control only: this stand-in has no issuer or source authority. + arrays = { + "A_AGE": np.array([12, 40]), + "A_LINENO": np.array([2, 1]), + "source_household_id": np.array([7, 7]), + } + observed = SimpleNamespace(array=arrays.__getitem__) + positions = np.array([1, 0]) + numeric = pd.DataFrame({"A_AGE": [40, 12]}) + raw = pd.DataFrame({"A_LINENO": ["1", "2"], "PH_SEQ": ["00007", "00007"]}) + owner._check_demographic_coordinates(observed, positions, numeric, raw) + arrays[coordinate][0] += 1 + with pytest.raises(ValueError, match="SEX_SOURCE_COORDINATE_DISAGREE"): + owner._check_demographic_coordinates(observed, positions, numeric, raw) + + +@pytest.mark.parametrize("defect", ["order", "length"]) +def test_state_projection_requires_exact_household_axis(defect): + projection = pd.DataFrame({"household_id": [11, 12]}) + households = projection.copy(deep=True) + assert owner._state_column(projection, households, [6, 36]).tolist() == [6, 36] + if defect == "order": + households = households.iloc[::-1] + codes = [6, 36] + else: + codes = [6] + with pytest.raises(ValueError, match="STATE_HOUSEHOLD_ALIGNMENT"): + owner._state_column(projection, households, codes) + + +def test_changed_original_anchor_is_not_replaced_by_current_projection(actual): + value = actual.right + anchors = owner._ISSUED[id(value)][2].native_entry[2].anchors + previous = anchors.payload + try: + object.__setattr__(anchors, "payload", previous + b"changed") + with pytest.raises(ValueError): + value.validate() + finally: + object.__setattr__(anchors, "payload", previous) + value.validate() + + +def test_independent_seal_refuses_mutated_qualifier_closure_and_literal(actual): + value = actual.right + qualified = owner._ISSUED[id(value)][2].qualified + index = qualified.asec_full_raw.index[0] + before = qualified.asec_full_raw.at[index, "MCARE"] + # The retained donor seal is outside the older literal owner's callback. + cells = dict( + zip( + qualified._revalidate.__code__.co_freevars, + qualified._revalidate.__closure__, + strict=True, + ) + ) + previous = cells["seals"].cell_contents + try: + qualified.asec_full_raw.at[index, "MCARE"] = "1" + tables = cells["tables"].cell_contents + cells["seals"].cell_contents = tuple( + owner.literals._table_seal(table) for table in tables + ) + with pytest.raises(ValueError, match="SOURCE_PROJECTION_CHANGED"): + value.validate() + finally: + qualified.asec_full_raw.at[index, "MCARE"] = before + cells["seals"].cell_contents = previous + value.validate() diff --git a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py new file mode 100644 index 000000000..73b68e66e --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py @@ -0,0 +1,1169 @@ +"""Actual source readers over constructed, privately pinned invented bytes. + +No native microdata, no country engine and no full PUF donor fixture: the +current ASEC income routing qualifier only needs the bounded survey preparation +fixture plus invented member literals. +""" + +import ast +import hashlib +import json +import shutil +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from test_us_asec_coverage_authentication import _changed_parent +from test_us_survey_population_preparation import fixture + +from microcosm.build.frame_checkpoint import load_frame_checkpoint +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import current_asec_income_routing_source as owner + +# Invented current-year people. Ages 55/14 come from the shared fixture; the +# older adult exercises the printed age-58 distribution route. +CURRENT_PEOPLE = (105, 106, 107, 108) +AGES = {105: 55, 106: 14, 107: 70, 108: 14} +AMOUNTS = { + # PNSN ANN DSTV1 DSTV1Y DSTV2 DSTV2Y RNT FRSE OI + 105: (12000.0, -1.0, 0.0, 7000.0, 0.0, 0.0, -400.0, 0.0, 3000.0), + 106: (0.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0), + 107: (9000.0, 0.0, 5000.0, 0.0, 0.0, 0.0, 500.0, -9000.0, 250.0), + 108: (0.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 700.0), +} +AMOUNT_ORDER = ( + "PNSN_VAL", + "ANN_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "RNT_VAL", + "FRSE_VAL", + "OI_VAL", +) +LITERALS = { + 105: { + "PEN_YN": "1", + "ANN_YN": "0", + "DST_YN": "0", + "DST_YN_YNG": "1", + "DST_SC1_YNG": "4", + "RNT_YN": "1", + "FRSE_YN": "1", + "ERN_YN": "1", + "OI_YN": "1", + "OI_OFF": "20", + }, + 106: {}, + 107: { + "PEN_YN": "2", + "ANN_YN": "1", + "DST_YN": "1", + "DST_SC1": "9", + "RNT_YN": "0", + "FRSE_YN": "1", + "ERN_YN": "1", + "OI_YN": "1", + "OI_OFF": "0", + }, + 108: {"OI_YN": "0", "ERN_YN": "", "FRMOTR": ""}, +} +DEFAULT_LITERAL = { + **{name: "0" for name in owner.RECEIPT_ENTRIES}, + **{name: "0" for name in owner.ACCOUNT_ENTRIES}, + "OI_OFF": "0", + **{name: "0" for name in owner.ALLOCATION_ENTRIES}, +} + + +def _member_literals(person_id): + return {**DEFAULT_LITERAL, **LITERALS.get(person_id, {})} + + +def routing_arguments(tmp_path, monkeypatch, *, reverse=True, ages=None): + """Extend the invented member before any owner issues a preparation. + + Only fixture registry pins change. No source issuer, ready() or native + loader is replaced, and the person-income attachment is rebuilt from the + changed member bytes exactly as production would. + """ + arguments = fixture(tmp_path, monkeypatch) + asec = arguments["source_dir"] / "asec" + parent_path, attachment = asec / "parent.h5", asec / "household-attachment.h5" + person = load_frame_checkpoint(parent_path).frame.person + ids = person.person_id.to_numpy() + ages = AGES if ages is None else ages + changes = {} + for offset, field in enumerate(AMOUNT_ORDER): + data = person[field].to_numpy(dtype="float64", copy=True) + for person_id in CURRENT_PEOPLE: + positions = np.flatnonzero(ids == person_id) + assert len(positions) == 1 + data[positions[0]] = AMOUNTS[person_id][offset] + changes[field] = data + age_column = person.A_AGE.to_numpy(copy=True) + for person_id in CURRENT_PEOPLE: + age_column[np.flatnonzero(ids == person_id)[0]] = ages[person_id] + changes["A_AGE"] = age_column + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in coverage._MEMBER_PINS: + path = asec / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for field in (*AMOUNT_ORDER, "A_AGE"): + raw[field] = [str(int(updated.loc[key, field])) for key in raw.PERIDNUM] + keys = [int(key) for key in raw.PERIDNUM] + current = dict(zip(keys, updated.loc[raw.PERIDNUM].person_id, strict=True)) + for name in DEFAULT_LITERAL: + raw[name] = [_member_literals(int(current[key]))[name] for key in keys] + assert raw.notna().all().all() + ordered = raw.iloc[::-1] if reverse else raw + ordered.to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(raw), + len(data), + ) + ) + paths[year] = path + for module in (coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "routing-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, asec / "person-income-attachment.h5" + ) + return arguments + + +def _qualified(tmp_path, monkeypatch, **kwargs): + arguments = routing_arguments(tmp_path, monkeypatch, **kwargs) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + return prepared, owner.qualify_current_asec_income_routing(prepared) + + +def test_actual_invented_member_joins_the_real_preparation_by_native_keys( + tmp_path, monkeypatch +): + prepared, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert set(person.index) == set(CURRENT_PEOPLE) + # Borrowed money, native identity and member literal agree row by row. + assert person.loc[105, "pension_annuity_pension_source_total"] == 12000.0 + assert person.loc[105, "pension_annuity_pension_reporting_status"] == ( + "known_receipt" + ) + assert person.loc[105, "pension_annuity_pension_known_amount"] == 12000.0 + assert person.source_age.to_dict() == {k: float(v) for k, v in AGES.items()} + literals = qualified.asec_literals.set_index("PERIDNUM") + for person_id in CURRENT_PEOPLE: + key = str(person_id - 100).zfill(22) + assert int(literals.loc[key, "PNSN_VAL"]) == AMOUNTS[person_id][0] + assert int(literals.loc[key, "A_AGE"]) == AGES[person_id] + assert person.amount_validity_PNSN_VAL.eq(1).all() + assert ( + hashlib.sha256(qualified.person.to_json(orient="table").encode()).hexdigest() + == qualified.evidence["projection_sha256"] + ) + assert not qualified.evidence["source_admission_issued"] + assert not qualified.evidence["release_eligible"] + # A host records this receipt, so it must round-trip as JSON unchanged. + receipt = json.dumps(qualified.evidence, sort_keys=True) + assert json.loads(receipt) == qualified.evidence + assert qualified.evidence["joined_person_years"] == [2024] + assert qualified.evidence["acs_channel_rows"] > 0 + assert not qualified.evidence["acs_components_modeled"] + assert set(qualified.evidence["families"]) == set(owner.FAMILIES) + assert ( + qualified.evidence["dictionary"]["amount_entries_source"]["sha256"] + == owner.money.RESOURCE_PINS[0] + ) + assert qualified.evidence["declared_niu_normalized_to_zero"] == ["ANN_VAL"] + printed = qualified.evidence["dictionary"] + assert set(printed["printed_universe_questions"]) == { + "DST_VAL1", + "DST_SC2_YNG", + "I_DSTVAL1COMP", + "DST_YN", + } + assert set(printed["printed_scope_and_code_questions"]) == { + "RNT_YN", + "RNT_VAL", + "FRSE_VAL", + "PNSN_VAL", + "OI_YN", + "DST_SC1", + "FRSE_YN", + } + assert set(printed["ambiguous_allocation_flag_coverage"]) == set( + owner.AMBIGUOUS_FLAG_COVERAGE + ) + prepared.checked_view() + + +def test_pension_and_annuity_totals_stay_separate_and_unsplit(tmp_path, monkeypatch): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + # PNSN_VAL is the printed combined total over all pension sources. Neither + # a private share nor a taxable share is derived from it. + assert person.loc[105, "pension_annuity_combined_total_scope"] == ( + owner.PENSION_TOTAL_SCOPE + ) + assert not person.pension_annuity_private_share_applied.any() + assert not person.pension_annuity_taxable_amount_known.any() + assert not person.pension_annuity_pension_total_has_published_flag.any() + # ANN_VAL's printed -1 is a NIU code, never a one dollar annuity loss. + assert np.isnan(person.loc[105, "pension_annuity_annuity_source_total"]) + assert person.loc[105, "pension_annuity_annuity_amount_kind"] == "declared_niu" + assert person.loc[105, "pension_annuity_annuity_reporting_status"] == "niu" + # ANN_VAL is the one entry here whose printed zero is valid dollars (its NIU + # is the separate -1 code), so a yes answer with a zero amount resolves. + assert person.loc[107, "pension_annuity_annuity_reporting_status"] == ( + "known_recipient_zero" + ) + assert person.loc[107, "pension_annuity_annuity_known_amount"] == 0.0 + # PNSN_VAL prints "0 = none or niu", so the same pattern would not resolve + # there; the module reads that distinction from the pinned domains artifact. + assert owner._zero_is_dollars("ANN_VAL") + assert not owner._zero_is_dollars("PNSN_VAL") + # A no answer against a positive total is a retained contradiction. + assert person.loc[107, "pension_annuity_pension_reporting_status"] == ( + "contradictory_no_nonzero" + ) + assert person.loc[107, "pension_annuity_pension_source_total"] == 9000.0 + assert np.isnan(person.loc[107, "pension_annuity_pension_known_amount"]) + + +def test_distribution_routing_preserves_slots_ages_and_unresolved_accounts( + tmp_path, monkeypatch +): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert person.loc[105, "retirement_distribution_route"] == "under_age58" + assert person.loc[107, "retirement_distribution_route"] == "age58_and_over" + # Regular IRA is account code 4; the slot amount is not a taxable amount. + assert person.loc[105, "retirement_distribution_slot1_young_account_code"] == 4 + assert person.loc[105, "retirement_distribution_slot1_young_account_label"] == ( + "Regular IRA" + ) + assert person.loc[105, "retirement_distribution_regular_ira_amount"] == 7000.0 + assert person.loc[105, "retirement_distribution_regular_ira_slots"] == 1 + assert person.loc[105, "retirement_distribution_source_total"] == 7000.0 + assert not person.retirement_distribution_taxable_amount_known.any() + assert bool(person.loc[107, "retirement_distribution_route_has_published_flag"]) + assert not bool(person.loc[105, "retirement_distribution_route_has_published_flag"]) + # An account literal outside the printed range leaves the composition and + # the regular IRA share unresolved rather than defaulting either way. + assert person.loc[107, "retirement_distribution_slot1_account_literal"] == "9" + assert person.loc[107, "retirement_distribution_slot1_account_literal_status"] == ( + "outside_printed_range" + ) + assert person.loc[107, "retirement_distribution_slot1_slot_status"] == ( + "unresolved_slot_account" + ) + assert pd.isna(person.loc[107, "retirement_distribution_slot1_account_code"]) + assert np.isnan(person.loc[107, "retirement_distribution_regular_ira_amount"]) + assert pd.isna(person.loc[107, "retirement_distribution_regular_ira_slots"]) + # The slot total is still the retained literal sum for the applicable route. + assert person.loc[107, "retirement_distribution_source_total"] == 5000.0 + # The printed distribution universes name only the age 58 split, so + # coverage below age 15 is a source question, not a resolved answer. + assert pd.isna(person.loc[106, "retirement_distribution_source_reporting_universe"]) + assert person.loc[106, "retirement_distribution_reporting_status"] == ( + "unresolved_reporting_universe" + ) + + +def test_signed_property_and_farm_totals_keep_losses_and_net_zero_receipt( + tmp_path, monkeypatch +): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert person.loc[105, "net_property_source_total"] == -400.0 + assert person.loc[105, "net_property_known_amount"] == -400.0 + assert person.loc[105, "net_property_reporting_status"] == "known_receipt" + assert bool(person.loc[105, "net_property_is_net_loss"]) + # The receipt question is wider than the amount question, so the total is + # not independently labelled rental and no component split is claimed. + assert person.loc[105, "net_property_receipt_scope"] == ( + owner.NET_PROPERTY_RECEIPT_SCOPE + ) + assert person.loc[105, "net_property_amount_scope"] == ( + owner.NET_PROPERTY_AMOUNT_SCOPE + ) + assert not person.net_property_component_split_known.any() + # A NIU receipt literal against a positive amount stays a contradiction. + assert person.loc[107, "net_property_reporting_status"] == ( + "contradictory_niu_nonzero" + ) + assert person.loc[107, "net_property_source_total"] == 500.0 + assert np.isnan(person.loc[107, "net_property_known_amount"]) + # Farm receipt with a net zero stays distinct from absence, NIU and missing, + # but FRSE_VAL prints "0 = none or niu" just as the gross entries do, so the + # amount is not completed to a known zero. + assert person.loc[105, "farm_reporting_status"] == "receipt_with_net_zero" + assert np.isnan(person.loc[105, "farm_known_amount"]) + assert person.loc[105, "farm_source_total"] == 0.0 + assert "receipt_with_net_zero" not in owner.KNOWN_AMOUNT_STATUSES + assert person.loc[107, "farm_source_total"] == -9000.0 + assert person.loc[107, "farm_reporting_status"] == "known_receipt" + assert bool(person.loc[107, "farm_is_net_loss"]) + assert not person.farm_is_nonfarm_self_employment.any() + # The farm universe comes from ERN_YN/FRMOTR evidence, never from age. + assert bool(person.loc[105, "farm_source_reporting_universe"]) + assert pd.isna(person.loc[108, "farm_source_reporting_universe"]) + assert person.loc[108, "farm_reporting_status"] == "unresolved_reporting_universe" + + +def test_other_income_keeps_reported_alimony_apart_from_residual_rules( + tmp_path, monkeypatch +): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert person.loc[105, "other_income_category_code"] == 20 + assert person.loc[105, "other_income_category_label"] == "alimony" + assert person.loc[105, "other_income_routing_status"] == "reported_category" + assert bool(person.loc[105, "other_income_is_reported_alimony"]) + assert person.loc[105, "other_income_source_total"] == 3000.0 + # A receipt without a category is retained as an unresolved pattern; no + # residual rule assigns it to alimony or to miscellaneous income. + assert person.loc[107, "other_income_routing_status"] == "receipt_without_category" + assert not bool(person.loc[107, "other_income_is_reported_alimony"]) + assert not person.other_income_residual_rule_applied.any() + # A NIU receipt with a positive amount is a contradiction, not a nonfiler. + assert person.loc[108, "other_income_reporting_status"] == ( + "contradictory_outside_reporting_universe" + ) + assert person.loc[108, "other_income_source_total"] == 700.0 + assert np.isnan(person.loc[108, "other_income_known_amount"]) + + +def test_row_order_of_the_source_member_cannot_change_the_projection( + tmp_path, monkeypatch +): + roots = {} + for name in ("forward", "reverse"): + roots[name] = tmp_path / name + roots[name].mkdir() + with pytest.MonkeyPatch.context() as forward: + _, ordered = _qualified(roots["forward"], forward, reverse=False) + with pytest.MonkeyPatch.context() as backward: + _, reversed_ = _qualified(roots["reverse"], backward, reverse=True) + pd.testing.assert_frame_equal(ordered.person, reversed_.person, check_exact=True) + assert ( + ordered.evidence["projection_sha256"] == reversed_.evidence["projection_sha256"] + ) + # The member digest legitimately differs; the projection must not. + assert ( + ordered.evidence["source_member_sha256"] + != reversed_.evidence["source_member_sha256"] + ) + + +def test_repeated_qualification_is_stable_and_transport_is_not_authority( + tmp_path, monkeypatch +): + prepared, qualified = _qualified(tmp_path, monkeypatch) + again = owner.qualify_current_asec_income_routing(prepared) + assert ( + again.evidence["projection_sha256"] == qualified.evidence["projection_sha256"] + ) + # Mutating the returned transport cannot reach a fresh live projection. + qualified.person.loc[qualified.person.index[0], "farm_source_total"] = 99.0 + qualified.asec_literals.loc[qualified.asec_literals.index[0], "OI_OFF"] = "99" + fresh = owner.qualify_current_asec_income_routing(prepared) + assert not fresh.person.farm_source_total.eq(99.0).any() + assert not fresh.asec_literals.OI_OFF.eq("99").any() + pd.testing.assert_frame_equal(fresh.person, again.person, check_exact=True) + + +@pytest.mark.parametrize("defect", ("digest", "member", "rows", "bytes")) +def test_changed_member_bytes_or_pin_refuse_before_any_projection( + tmp_path, monkeypatch, defect +): + arguments = routing_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + pins = list(coverage._MEMBER_PINS) + if defect == "bytes": + member = next(p[1] for p in pins if p[0] == 2024) + path = arguments["source_dir"] / "asec" / member + path.write_bytes(path.read_bytes().replace(b"12000", b"12001", 1)) + else: + index = next(i for i, p in enumerate(pins) if p[0] == 2024) + year, name, archive, digest, rows, size = pins[index] + pins[index] = { + "digest": (year, name, archive, "f" * 64, rows, size), + "member": (year, "pppub99.csv", archive, digest, rows, size), + "rows": (year, name, archive, digest, rows + 1, size), + }[defect] + monkeypatch.setattr(coverage, "_MEMBER_PINS", tuple(pins)) + with pytest.raises(ValueError): + owner.qualify_current_asec_income_routing(prepared) + + +@pytest.mark.parametrize("swap", ("coordinate", "amount")) +def test_borrowed_member_must_match_the_parent_row_for_row(tmp_path, monkeypatch, swap): + arguments = routing_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + real = owner._read_capture + + def tampered(path, *, rows): + frame = real(path, rows=rows) + column = "A_LINENO" if swap == "coordinate" else "PNSN_VAL" + values = frame[column].to_numpy(copy=True) + frame[column] = np.concatenate([values[1:], values[:1]]) + return frame + + monkeypatch.setattr(owner, "_read_capture", tampered) + expected = ( + "PARENT_COORDINATE_IDENTITY" + if swap == "coordinate" + else "CURRENT_AMOUNT_SOURCE_IDENTITY" + ) + with pytest.raises(ValueError, match=expected): + owner.qualify_current_asec_income_routing(prepared) + + +def test_a_tampered_preparation_refuses_before_any_projection(tmp_path, monkeypatch): + """A replaced checked method is refused by the owner, not worked around.""" + arguments = routing_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + real = type(prepared)._checked + calls = [] + + def late(self): + entry = real(self) + calls.append(entry) + return entry if len(calls) == 1 else (entry[0], b"{}", entry[2]) + + monkeypatch.setattr(type(prepared), "_checked", late) + with pytest.raises(ValueError, match="FINAL_AUTHORITY_CHANGED|SOURCE_CHANGED"): + owner.qualify_current_asec_income_routing(prepared) + + +def test_the_final_requalification_cannot_be_replaced_or_skipped(tmp_path, monkeypatch): + """The owner refuses a substituted final requalification, so this qualifier + cannot be made to return a projection that skipped it.""" + prepared, qualified = _qualified(tmp_path, monkeypatch) + real = owner.source._pure_final + seen = [] + + def spy(state): + seen.append(state) + return real(state) + + # Scoped so undoing the substitution leaves the fixture pins in place. + with pytest.MonkeyPatch.context() as replaced: + replaced.setattr(owner.source, "_pure_final", spy) + with pytest.raises(ValueError, match="FINAL_AUTHORITY_CHANGED"): + owner.qualify_current_asec_income_routing(prepared) + # Unreplaced, the same preparation still yields the identical projection. + again = owner.qualify_current_asec_income_routing(prepared) + assert ( + again.evidence["projection_sha256"] == qualified.evidence["projection_sha256"] + ) + assert ( + hashlib.sha256(again.person.to_json(orient="table").encode()).hexdigest() + == again.evidence["projection_sha256"] + ) + + +def test_the_qualifier_imports_only_the_accepted_source_neighbourhood(): + """Source-only portability: no graph, fit, engine or modelled stage import.""" + path = Path(owner.__file__) + tree = ast.parse(path.read_text()) + absolute, relative = set(), set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + absolute.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + if node.level: + relative.update( + alias.name for alias in node.names + ) if node.module is None else relative.add(node.module) + else: + absolute.add(node.module.split(".")[0]) + assert absolute <= { + "__future__", + "csv", + "hashlib", + "json", + "re", + "tempfile", + "dataclasses", + "functools", + "importlib", + "pathlib", + "typing", + "numpy", + "pandas", + } + assert relative == { + "asec_coverage_authentication", + "asec_current_money", + "source_csv_builtin", + "survey_population_preparation", + "support_provenance", + } + # The legacy modelled assumptions are deliberately not reachable from here. + text = path.read_text() + for forbidden in ("microcosm.graph", "policyengine", "0.590", "0.59"): + assert forbidden not in text + + +def _member_row(**overrides): + values = { + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + **{name: "0" for name in owner.AMOUNT_FIELDS}, + **DEFAULT_LITERAL, + **overrides, + } + return [values[name] for name in owner.READ_COLUMNS] + + +def _write_member(path, rows): + lines = [",".join(owner.READ_COLUMNS)] + lines.extend(",".join(row) for row in rows) + path.write_text("\n".join(lines) + "\n") + return path + + +@pytest.mark.parametrize( + "universe,token,kind,net,dollars,label", + [ + (True, "1", "nonzero", False, False, "known_receipt"), + (True, "1", "zero", False, False, "ambiguous_recipient_zero"), + # A signed net measure's recipient zero stays distinct, and stays + # unknown: RNT_VAL and FRSE_VAL print the same "0 = none or niu" label + # the gross entries print. + (True, "1", "zero", True, False, "receipt_with_net_zero"), + # Only an entry whose printed zero is valid dollars (ANN_VAL) resolves. + (True, "1", "zero", False, True, "known_recipient_zero"), + (True, "1", "nonzero", True, False, "known_receipt"), + (True, "2", "zero", False, False, "known_nonreceipt"), + (True, "2", "nonzero", False, False, "contradictory_no_nonzero"), + (True, "0", "zero", False, False, "niu"), + (True, "0", "nonzero", False, False, "contradictory_niu_nonzero"), + (True, "0", "declared_niu", False, True, "niu"), + (True, "1", "declared_niu", False, True, "contradictory_declared_niu_amount"), + (True, "1", "missing", False, False, "missing_amount"), + (True, "", "zero", False, False, "missing_receipt_literal"), + (True, "9", "zero", False, False, "unrecognized_receipt_literal"), + (True, "01", "zero", False, False, "unrecognized_receipt_literal"), + (False, "0", "zero", False, False, "outside_reporting_universe"), + (False, "0", "missing", False, False, "outside_reporting_universe"), + ( + False, + "1", + "nonzero", + False, + False, + "contradictory_outside_reporting_universe", + ), + ( + False, + "2", + "zero", + False, + False, + "contradictory_outside_reporting_universe", + ), + (None, "1", "nonzero", False, False, "unresolved_reporting_universe"), + (None, "0", "zero", True, False, "unresolved_reporting_universe"), + ], +) +def test_receipt_and_amount_knownness_stay_separate( + universe, token, kind, net, dollars, label +): + receipt = owner.literal_code(token, owner.receipt_codes("PEN_YN"), width=1) + assert ( + owner.receipt_status( + universe, receipt, kind, net_measure=net, zero_is_dollars=dollars + ) + == label + ) + # Only three statuses establish a dollar reading; nothing else completes. + assert (label in owner.KNOWN_AMOUNT_STATUSES) == ( + label in ("known_receipt", "known_recipient_zero", "known_nonreceipt") + ) + + +def test_only_ann_val_prints_a_zero_that_reads_as_valid_dollars(): + entries = owner.printed_amount_entries() + dollars = { + name + for name, entry in entries.items() + if entry.zero_semantics == owner.DOLLAR_ZERO_SEMANTICS + } + assert dollars == {"ANN_VAL"} + for name in set(owner.AMOUNT_FIELDS) - dollars: + assert ( + entries[name].zero_semantics + == "none_or_niu_not_distinguishable_from_amount_alone" + ) + assert owner._zero_is_dollars("ANN_VAL") + assert not owner._zero_is_dollars("RNT_VAL") + assert not owner._zero_is_dollars("FRSE_VAL") + + +@pytest.mark.parametrize( + "status,value,kind,dollars", + [ + (owner.money.CodebookStatus.AMOUNT_NONZERO, 120.0, "nonzero", 120.0), + (owner.money.CodebookStatus.AMOUNT_NONZERO, -900.0, "nonzero", -900.0), + (owner.money.CodebookStatus.ZERO_NONE_OR_NIU, 0.0, "zero", 0.0), + (owner.money.CodebookStatus.ZERO_DOLLARS_AS_CODED, 0.0, "zero", 0.0), + (owner.money.CodebookStatus.DECLARED_NIU, 0.0, "declared_niu", None), + (owner.money.CodebookStatus.MISSING_NULL, np.nan, "missing", None), + ], +) +def test_amount_meaning_comes_from_the_parent_status_axis(status, value, kind, dollars): + read_kind, read_dollars = owner.amount_state(value, status) + assert read_kind == kind + if dollars is None: + assert np.isnan(read_dollars) + else: + assert read_dollars == dollars + # A normalized NIU zero must never be read back as a zero dollar amount. + if status == owner.money.CodebookStatus.DECLARED_NIU: + assert read_kind != "zero" + + +@pytest.mark.parametrize( + "token,width,code,status", + [ + ("4", 1, 4, "in_printed_range"), + ("20", 2, 20, "in_printed_range"), + ("", 1, None, "missing"), + ("01", 1, None, "malformed"), + (" 4", 1, None, "malformed"), + ("4.0", 1, None, "malformed"), + ("NA", 1, None, "malformed"), + ("9", 1, None, "outside_printed_range"), + ("21", 2, None, "outside_printed_range"), + ], +) +def test_routing_literals_stay_unknown_rather_than_recoded(token, width, code, status): + allowed = owner.ACCOUNT_CODES if width == 1 else owner.OTHER_INCOME_CATEGORIES + assert owner.literal_code(token, allowed, width=width) == (code, status) + + +@pytest.mark.parametrize( + "defect,match", + [ + ("duplicate_key", "DUPLICATE_OR_INVALID_COORDINATE"), + ("duplicate_coordinate", "DUPLICATE_OR_INVALID_COORDINATE"), + ("short_key", "PERSON_KEY"), + ("bad_age", "COORDINATE:A_AGE"), + ("unsigned_negative", "AMOUNT_TOKEN:PNSN_VAL"), + ("wide_amount", "AMOUNT_TOKEN:OI_VAL"), + ("oversized_token", "TOKEN_BOUND:OI_OFF"), + ("missing_column", "HEADER"), + ("row_count", "ROW_COUNT"), + ], +) +def test_bounded_member_reader_refuses_structural_defects(tmp_path, defect, match): + path = tmp_path / "invented.csv" + rows = [_member_row()] + if defect == "duplicate_key": + rows.append(_member_row(PH_SEQ="2")) + elif defect == "duplicate_coordinate": + rows.append(_member_row(PERIDNUM="0000000000000000000002")) + elif defect == "short_key": + rows = [_member_row(PERIDNUM="1")] + elif defect == "bad_age": + rows = [_member_row(A_AGE="123")] + elif defect == "unsigned_negative": + rows = [_member_row(PNSN_VAL="-5")] + elif defect == "wide_amount": + rows = [_member_row(OI_VAL="1234567")] + elif defect == "oversized_token": + rows = [_member_row(OI_OFF="9" * 65)] + _write_member(path, rows) + if defect == "missing_column": + text = path.read_text().split("\n") + keep = [i for i, c in enumerate(owner.READ_COLUMNS) if c != "OI_OFF"] + path.write_text( + "\n".join(",".join(line.split(",")[i] for i in keep) for line in text[:-1]) + + "\n" + ) + expected = 2 if defect.startswith("duplicate") else 1 + if defect == "row_count": + expected = 2 + with pytest.raises(ValueError, match=match): + owner._read_capture(path, rows=expected) + + +def test_bounded_member_reader_accepts_signed_and_missing_amount_literals(tmp_path): + path = _write_member( + tmp_path / "invented.csv", + [_member_row(RNT_VAL="-9999", FRSE_VAL="-9999999", ANN_VAL="-1", OI_VAL="")], + ) + frame = owner._read_capture(path, rows=1) + assert frame.iloc[0].RNT_VAL == "-9999" + assert frame.iloc[0].FRSE_VAL == "-9999999" + assert frame.iloc[0].ANN_VAL == "-1" + assert frame.iloc[0].OI_VAL == "" + + +def _field(name, amounts, statuses, validity): + return owner.money.MoneyField( + name, + np.array(amounts, dtype=" 0 and a_age ≥ 58"} + assert owner.ACCOUNT_ENTRIES["DST_SC1_YNG"][4] == ("DST_YN_YNG = 1 and a_age < 58") + assert owner.ACCOUNT_ENTRIES["DST_SC2_YNG"][4] == ("DST_VAL_YNG > 0 and a_age < 58") + assert owner.RECEIPT_ENTRIES["OI_YN"][5] == "none or niu" + assert owner.receipt_codes("OI_YN")[0] == "none or niu" + for name in ("PEN_YN", "ANN_YN", "DST_YN", "DST_YN_YNG", "RNT_YN", "ERN_YN"): + assert owner.receipt_codes(name)[0] == "niu" + assert owner.receipt_codes("FRSE_YN")[0] == "Niu" + assert owner.RECEIPT_ENTRIES["FRSE_YN"].universe_as_printed == ( + "ERN_YN=1 or FRMOTR=1" + ) + for name in ("PEN_YN", "ANN_YN", "RNT_YN", "OI_YN"): + assert owner.RECEIPT_ENTRIES[name].universe_as_printed == ( + "All Persons aged 15+" + ) + + +def _pure_rows(n, ages, **literals): + codebook = owner.money.CodebookStatus + amounts = {name: np.zeros(n) for name in owner.AMOUNT_FIELDS} + statuses = { + name: np.full(n, int(codebook.ZERO_NONE_OR_NIU), "u1") + for name in owner.AMOUNT_FIELDS + } + statuses["ANN_VAL"] = np.full(n, int(codebook.ZERO_DOLLARS_AS_CODED), "u1") + raw = { + "amounts": amounts, + "statuses": statuses, + "allocations": { + name: ([0] * n, ["in_printed_range"] * n) + for name in owner.ALLOCATION_ENTRIES + }, + } + for name in (*owner.RECEIPT_ENTRIES, *owner.ACCOUNT_ENTRIES, "OI_OFF"): + raw[name] = ["0"] * n + raw.update(literals) + return raw, np.asarray(ages, dtype="float64") + + +def _set_amount(raw, field, values): + codebook = owner.money.CodebookStatus + raw["amounts"][field] = np.asarray(values, dtype="float64") + raw["statuses"][field] = np.array( + [ + int(codebook.AMOUNT_NONZERO) + if v + else int( + codebook.ZERO_DOLLARS_AS_CODED + if field == "ANN_VAL" + else codebook.ZERO_NONE_OR_NIU + ) + for v in values + ], + dtype="u1", + ) + + +def test_offroute_distribution_dollars_block_a_known_absence(): + raw, ages = _pure_rows(1, [70.0], DST_YN=["2"], DST_SC1_YNG=["4"]) + _set_amount(raw, "DST_VAL1_YNG", [8000.0]) + out = owner.project_income_routing(raw, ages) + assert out.retirement_distribution_route.iloc[0] == "age58_and_over" + assert bool(out.retirement_distribution_offroute_nonzero.iloc[0]) + # The applicable literals alone would read as a known zero; the retained + # off-route dollars contradict that, so nothing is resolved. + assert out.retirement_distribution_reporting_status.iloc[0] == ( + "contradictory_offroute_evidence" + ) + assert np.isnan(out.retirement_distribution_known_amount.iloc[0]) + assert out.retirement_distribution_slot1_young_amount.iloc[0] == 8000.0 + + +def test_an_ambiguous_slot_zero_leaves_the_total_and_the_ira_share_unknown(): + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["4"], DST_SC2=["1"]) + _set_amount(raw, "DST_VAL1", [5000.0]) + out = owner.project_income_routing(raw, ages) + # Slot 2 declares a 401k account whose amount is a "none or niu" zero. + assert out.retirement_distribution_slot2_slot_status.iloc[0] == ( + "ambiguous_slot_zero" + ) + assert bool(out.retirement_distribution_slot_zero_ambiguity.iloc[0]) + assert out.retirement_distribution_source_total.iloc[0] == 5000.0 + assert out.retirement_distribution_reporting_status.iloc[0] == ( + "unresolved_slot_composition" + ) + assert np.isnan(out.retirement_distribution_known_amount.iloc[0]) + assert np.isnan(out.retirement_distribution_regular_ira_amount.iloc[0]) + assert pd.isna(out.retirement_distribution_regular_ira_slots.iloc[0]) + # A regular IRA slot whose own amount is that ambiguous zero is not a known + # zero distribution from that account either. + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["4"], DST_SC2=["1"]) + _set_amount(raw, "DST_VAL2", [5000.0]) + ambiguous = owner.project_income_routing(raw, ages) + assert ambiguous.retirement_distribution_slot1_slot_status.iloc[0] == ( + "ambiguous_slot_zero" + ) + assert np.isnan(ambiguous.retirement_distribution_regular_ira_amount.iloc[0]) + # A contradictory NIU slot carrying dollars is likewise unresolved. + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["0"]) + _set_amount(raw, "DST_VAL1", [5000.0]) + contradictory = owner.project_income_routing(raw, ages) + assert contradictory.retirement_distribution_slot1_slot_status.iloc[0] == ( + "contradictory_niu_slot_amount" + ) + assert np.isnan(contradictory.retirement_distribution_regular_ira_amount.iloc[0]) + assert pd.isna(contradictory.retirement_distribution_regular_ira_slots.iloc[0]) + # Two fully resolved slots do give a known composition. + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["4"], DST_SC2=["0"]) + _set_amount(raw, "DST_VAL1", [5000.0]) + resolved = owner.project_income_routing(raw, ages) + assert resolved.retirement_distribution_reporting_status.iloc[0] == "known_receipt" + assert resolved.retirement_distribution_known_amount.iloc[0] == 5000.0 + assert resolved.retirement_distribution_regular_ira_amount.iloc[0] == 5000.0 + assert resolved.retirement_distribution_regular_ira_slots.iloc[0] == 1 + + +@pytest.mark.parametrize( + "age,ern,frmotr,routing", + [ + (14, "1", "0", "outside_reporting_universe_routing"), + (40, "", "", "reported_category"), + ], +) +def test_other_income_routing_follows_the_printed_receipt_universe( + age, ern, frmotr, routing +): + raw, ages = _pure_rows( + 1, + [float(age)], + OI_YN=["1"], + OI_OFF=["20"], + ERN_YN=[ern], + FRMOTR=[frmotr], + ) + _set_amount(raw, "OI_VAL", [3000.0]) + out = owner.project_income_routing(raw, ages) + assert out.other_income_routing_status.iloc[0] == routing + assert bool(out.other_income_is_reported_alimony.iloc[0]) == ( + routing == "reported_category" + ) + # The farm universe on the same rows comes from ERN_YN/FRMOTR, not from age. + assert pd.isna(out.farm_source_reporting_universe.iloc[0]) == (ern == "") + + +def test_allocation_origins_do_not_assert_publisher_confirmed_non_allocation(): + raw, ages = _pure_rows(1, [40.0]) + zeroed = owner.project_income_routing(raw, ages) + # Every flag here prints a conditional universe that is not evaluated, so an + # all-zero reading is reported as exactly that, never as "no allocation". + assert zeroed.net_property_allocation_origin.iloc[0] == "published_flags_all_zero" + assert zeroed.pension_annuity_allocation_origin.iloc[0] == ( + "published_flags_all_zero_with_unflagged_fields" + ) + for column in [c for c in zeroed.columns if c.endswith("_allocation_origin")]: + assert "no_allocation" not in zeroed[column].iloc[0] + raw, ages = _pure_rows(1, [40.0]) + raw["allocations"]["I_RNTVAL"] = ([4], ["in_printed_range"]) + assert ( + owner.project_income_routing(raw, ages).net_property_allocation_origin.iloc[0] + == "publisher_allocated" + ) + raw, ages = _pure_rows(1, [40.0]) + raw["allocations"]["I_RNTYN"] = ([None], ["missing"]) + assert ( + owner.project_income_routing(raw, ages).net_property_allocation_origin.iloc[0] + == "allocation_flag_not_populated" + ) + raw, ages = _pure_rows(1, [40.0]) + raw["allocations"]["I_RNTYN"] = ([None], ["outside_printed_range"]) + assert ( + owner.project_income_routing(raw, ages).net_property_allocation_origin.iloc[0] + == "unresolved_allocation_provenance" + ) + + +@pytest.mark.parametrize("field", ("OI_YN", "DST_SC1", "OI_OFF")) +def test_the_public_projection_validates_its_routing_token_arrays(field): + raw, ages = _pure_rows(2, [40.0, 40.0]) + raw[field] = ["0"] + with pytest.raises(ValueError, match="ROUTING_TOKEN_CONTRACT:" + field): + owner.project_income_routing(raw, ages) + raw[field] = ["0", 0] + with pytest.raises(ValueError, match="ROUTING_TOKEN_CONTRACT:" + field): + owner.project_income_routing(raw, ages) + + +def test_the_literal_transport_carries_its_own_digest(tmp_path, monkeypatch): + _, qualified = _qualified(tmp_path, monkeypatch) + assert ( + hashlib.sha256( + qualified.asec_literals.to_json(orient="table").encode() + ).hexdigest() + == qualified.evidence["literals_sha256"] + ) + assert ( + qualified.evidence["literals_sha256"] != qualified.evidence["projection_sha256"] + ) + # A host comparing the documented digests detects a corrupted literal frame. + index = qualified.asec_literals.index[0] + qualified.asec_literals.loc[index, "PNSN_VAL"] = "999999" + assert ( + hashlib.sha256( + qualified.asec_literals.to_json(orient="table").encode() + ).hexdigest() + != qualified.evidence["literals_sha256"] + ) diff --git a/packages/microcosm-build/tests/test_us_current_asec_interest_source.py b/packages/microcosm-build/tests/test_us_current_asec_interest_source.py new file mode 100644 index 000000000..bf8e34334 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_interest_source.py @@ -0,0 +1,409 @@ +"""Invented literal and real-source-owner controls for the interest extension.""" + +import copy +import hashlib +import json +import shutil +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_interest_source as owner + + +def row(**changes): + return { + **{c: "0" for c in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "INT_VAL": "140", + "INT_YN": "1", + "TRDINT_VAL": "100", + "RINT_YN": "1", + "RINT_SC1": "4", + "RINT_VAL1": "40", + **changes, + } + + +def project(**changes): + return owner.project_interest_literals(pd.DataFrame([row(**changes)])).iloc[0] + + +def test_component_totals_remain_observed_and_discrepancy_is_not_balanced(): + value = project( + INT_VAL="500", TRDINT_VAL="0", RINT_VAL1="400", RINT_SC2="7", RINT_VAL2="200" + ) + assert value.INT_VAL_amount == 500 + assert value.TRDINT_VAL_amount == 0 and value.TRDINT_VAL_amount_known + assert value.TRDINT_VAL_reporting_status == "observed_zero_component" + assert value.RINT_VAL1_amount == 400 and value.RINT_VAL2_amount == 200 + assert value.published_components_sum == 600 + assert value.combined_minus_published_components == -100 + assert value.component_discrepancy_known and value.all_component_amounts_observed + + +def test_an_unreported_account_slot_is_not_an_observed_zero(): + value = project() + assert value.RINT_VAL2_published_amount == 0 + assert value.RINT_VAL2_reporting_status == "unreported_account_slot" + assert not value.RINT_VAL2_amount_known and pd.isna(value.RINT_VAL2_amount) + assert not value.RINT_SC2_account_known + assert ( + value.component_discrepancy_known + and value.combined_minus_published_components == 0 + ) + assert not value.all_component_amounts_observed + + +@pytest.mark.parametrize( + "changes,field,status,known", + [ + ({"INT_VAL": "0"}, "INT_VAL", "ambiguous_recipient_zero", False), + ({"RINT_VAL1": "0"}, "RINT_VAL1", "ambiguous_recipient_zero", False), + ({"TRDINT_VAL": ""}, "TRDINT_VAL", "missing_amount", False), + ({"TRDINT_VAL": "-1"}, "TRDINT_VAL", "invalid_amount_literal", False), + ({"RINT_VAL1": "-1"}, "RINT_VAL1", "invalid_amount_literal", False), + ({"RINT_VAL1": "4.0"}, "RINT_VAL1", "invalid_amount_literal", False), + ({"RINT_VAL1": "NA"}, "RINT_VAL1", "invalid_amount_literal", False), + ({"RINT_SC1": ""}, "RINT_VAL1", "missing_account_literal", False), + ({"RINT_SC1": "9"}, "RINT_VAL1", "invalid_account_literal", False), + ({"RINT_SC1": "0"}, "RINT_VAL1", "amount_without_account", False), + ({"RINT_YN": ""}, "RINT_VAL1", "missing_receipt_literal", False), + ({"RINT_YN": "2"}, "RINT_VAL1", "account_without_receipt", False), + ({"INT_YN": "2"}, "TRDINT_VAL", "contradictory_no_nonzero", False), + ({"INT_YN": "0", "TRDINT_VAL": "0"}, "TRDINT_VAL", "niu", False), + ({"INT_YN": "2", "TRDINT_VAL": "0"}, "TRDINT_VAL", "known_nonreceipt", True), + ], +) +def test_receipt_account_and_amount_conflicts_remain_explicit( + changes, field, status, known +): + value = project(**changes) + assert value[field + "_reporting_status"] == status + assert bool(value[field + "_amount_known"]) is known + if not known: + assert pd.isna(value[field + "_amount"]) + + +def test_under15_universe_is_checked_without_analytical_zeros(): + value = project( + A_AGE="14", + INT_VAL="0", + INT_YN="0", + TRDINT_VAL="0", + RINT_YN="0", + RINT_SC1="0", + RINT_VAL1="0", + ) + for name in owner.AMOUNT_FIELDS: + assert value[name + "_reporting_status"] == "outside_reporting_universe" + assert not value[name + "_amount_known"] + assert value.published_components_sum == 0 # Published-cell diagnostic only. + + +@pytest.mark.parametrize( + "token,status", + [ + ("", "missing"), + ("1", "outside_printed_range"), + ("11", "in_printed_range"), + ("15", "in_printed_range"), + ("x", "malformed"), + ], +) +def test_composite_allocation_uses_printed_codes_not_header_interval(token, status): + value = project(I_INTVAL=token) + assert value.I_INTVAL_literal_status == status + assert ( + value.RINT_VAL1_amount_known + ) # Allocation never changes observation validity. + if token in ("11", "15"): + assert value.allocation_origin == "publisher_allocated" + + +def test_missing_component_or_flag_never_becomes_false_or_zero(): + value = project(RINT_VAL2="", TRINT_VAL2="", I_RINTVAL2="") + assert pd.isna(value.published_components_sum) + assert not value.component_discrepancy_known + assert value.TRINT_VAL2_literal_status == "missing" + assert value.allocation_origin == "allocation_flag_not_populated" + + +def test_shared_account_codes_and_pinned_total_domain_are_reused(): + assert owner.amount_entries()["INT_VAL"][4] == 999999 + assert owner.amount_entries()["TRDINT_VAL"][4] == 99999 + value = project(RINT_SC1="3") + assert value.RINT_SC1_label == owner.routing.ACCOUNT_CODES[3] == "Roth IRA" + assert value.RINT_SC1_account_known + value = project(INT_YN="0", RINT_YN="0") + assert value.INT_YN_label == value.RINT_YN_label == "niu" + + +def test_nullable_float_seal_preserves_masks_hidden_values_and_exact_bits(): + raw = pd.DataFrame([row()]) + values = owner.CurrentAsecInterestValues( + owner.project_interest_literals(raw), raw, {"descriptive": True} + ) + original = owner.interest_values_seal(values) + assert owner.interest_values_seal(copy.deepcopy(values)) == original + for change in ("bit", "hidden", "mask"): + altered = copy.deepcopy(values) + if change == "bit": + array = altered.person.INT_VAL_amount.array + array._data[0] = np.nextafter(array._data[0], np.inf) + else: + array = altered.person.RINT_VAL2_amount.array + assert array._mask[0] + if change == "hidden": + array._data[0] = 17.0 + else: + array._mask[0] = False + assert owner.interest_values_seal(altered) != original + + +@pytest.mark.parametrize("change", ["bit", "validity", "literal"]) +def test_retained_combined_total_requires_exact_amount_and_validity(change): + field = SimpleNamespace(amounts=np.array([140.0]), validity=np.array([1])) + ready = SimpleNamespace(field=lambda name: field) + positions, literals = np.array([0]), ["140"] + assert owner._compare_total(ready, positions, literals) is field + if change == "bit": + field.amounts[0] = np.nextafter(140.0, np.inf) + elif change == "validity": + field.validity[0] = 0 + else: + literals[0] = "-1" + with pytest.raises(ValueError, match="TOTAL_"): + owner._compare_total(ready, positions, literals) + + +def interest_arguments(tmp_path, monkeypatch): + """Amend original invented bytes before any preparation is issued.""" + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + amounts, ages = people.INT_VAL.to_numpy(copy=True), people.A_AGE.to_numpy(copy=True) + literals = { + 105: row(), + 106: row( + A_AGE="14", + INT_VAL="0", + INT_YN="0", + TRDINT_VAL="0", + RINT_YN="0", + RINT_SC1="0", + RINT_VAL1="0", + ), + 107: row( + A_AGE="70", + INT_VAL="500", + TRDINT_VAL="0", + RINT_VAL1="400", + RINT_SC2="7", + RINT_VAL2="200", + I_INTVAL="11", + TRINT_VAL1="1", + ), + 108: row( + INT_VAL="0", + INT_YN="2", + TRDINT_VAL="0", + RINT_YN="2", + RINT_SC1="0", + RINT_VAL1="0", + ), + } + for person_id, values in literals.items(): + amounts[ids == person_id], ages[ids == person_id] = ( + int(values["INT_VAL"]), + int(values["A_AGE"]), + ) + _changed_parent( + parent_path, attachment, monkeypatch, {"INT_VAL": amounts, "A_AGE": ages} + ) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + raw["INT_VAL"] = [str(int(updated.loc[key, "INT_VAL"])) for key in raw.PERIDNUM] + raw["A_AGE"] = [str(int(updated.loc[key, "A_AGE"])) for key in raw.PERIDNUM] + for name in ( + set(owner.READ_COLUMNS) + - set(owner.routing.COORDINATE_COLUMNS) + - {"INT_VAL"} + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row())[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "interest-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(tmp_path, monkeypatch): + return owner.routing.source.prepare_authenticated_survey_population( + **interest_arguments(tmp_path, monkeypatch) + ) + + +def test_actual_owner_join_keeps_source_total_and_requalifies(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + result = owner.qualify_current_asec_interest(parent) + values = result.person.set_index("native_person_id") + assert values.loc[107, "INT_VAL_amount"] == 500 + assert values.loc[107, "combined_minus_published_components"] == -100 + assert values.loc[107, "TRINT_VAL1_code"] == 1 + assert values.loc[105, "RINT_SC1_label"] == "Regular IRA" + assert values.loc[106, "INT_VAL_reporting_status"] == "outside_reporting_universe" + assert values.loc[108, "RINT_VAL2_reporting_status"] == "known_nonreceipt" + assert owner.interest_values_seal( + owner.qualify_current_asec_interest(parent) + ) == owner.interest_values_seal(result) + assert ( + not result.evidence["tax_treatment_assigned"] + and not result.evidence["source_admission_issued"] + ) + assert result.evidence == json.loads(json.dumps(result.evidence)) + with pytest.raises(ValueError): + owner.qualify_current_asec_interest(copy.copy(parent)) + + # The actual qualifier checks the retained MoneyDomain before source capture. + check, checked = owner._domain_agreement, [] + + def rejected_domain(ready): + check(ready) + checked.append(True) + raise ValueError("INVENTED_DOMAIN_REFUSAL") + + def forbidden_capture(*args): + pytest.fail("source capture preceded live domain agreement") + + with monkeypatch.context() as patch: + patch.setattr(owner, "_domain_agreement", rejected_domain) + patch.setattr(owner, "_capture_member", forbidden_capture) + with pytest.raises(ValueError, match="INVENTED_DOMAIN_REFUSAL"): + owner.qualify_current_asec_interest(parent) + assert checked == [True] + + +@pytest.mark.parametrize("coordinate", ["PH_SEQ", "A_LINENO", "A_AGE", "PERIDNUM"]) +def test_actual_owner_refuses_changed_source_coordinates( + tmp_path, monkeypatch, coordinate +): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if coordinate == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(coordinate)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_interest(parent) + + +def test_actual_owner_refuses_mutation_after_capture_io(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + called = [] + + def changed(*args): + raw = capture(*args) + parent._checked()[2].frame.person.iloc[ + 0, parent._checked()[2].frame.person.columns.get_loc("age") + ] += 1 + called.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_interest(parent) + assert called + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "literal", "evidence"] +) +def test_final_physical_seal_covers_complete_returned_values( + tmp_path, monkeypatch, mutation +): + parent = prepared(tmp_path, monkeypatch) + seal, calls = owner.interest_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + if mutation == "amount_bit": + column = value.person["INT_VAL_amount"].array + i = np.flatnonzero(~column._mask)[0] + column._data[i] = np.nextafter(column._data[i], np.inf) + elif mutation == "masked_backing": + column = value.person["RINT_VAL2_amount"].array + i = np.flatnonzero(column._mask)[0] + column._data[i] = 17.0 + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("RINT_SC1") + ] = "7" + else: + value.evidence["tax_treatment_assigned"] = True + return before + + monkeypatch.setattr(owner, "interest_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_interest(parent) + + +def test_default_reader_and_explicit_original_roster_agree(tmp_path): + from test_us_current_asec_income_routing import _member_row, _write_member + + path = tmp_path / "routing.csv" + _write_member(path, [_member_row()]) + default = owner.routing._read_capture(path, rows=1) + explicit = owner.routing._read_capture( + path, + rows=1, + columns=owner.routing.READ_COLUMNS, + patterns=owner.routing.amount_patterns(), + ) + pd.testing.assert_frame_equal(default, explicit) diff --git a/packages/microcosm-build/tests/test_us_current_asec_other_disability_source.py b/packages/microcosm-build/tests/test_us_current_asec_other_disability_source.py new file mode 100644 index 000000000..6cc02923a --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_other_disability_source.py @@ -0,0 +1,1268 @@ +"""Invented ASEC other-disability evidence; pure fixtures confer no authority. + +Every literal here is made up. The tests check the two published source slots, +the workers' compensation exclusion, the distinction between an observed zero +and NIU/missing/outside-universe, conditional allocation provenance, the exact +clone transport, and the fences that refuse a mutated or foreign owner. +""" + +import copy +import hashlib +import sys +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_income_routing_source as routing +from microcosm.build.us_runtime import current_asec_other_disability_source as od +from microcosm.build.us_runtime import current_asec_retirement_detail_source as detail +from microcosm.build.us_runtime import disability_benefits as legacy +from microcosm.build.us_runtime import support_provenance as provenance + +DETAIL_EVIDENCE = { + "protocol": detail.PROTOCOL, + "source_year": routing.CURRENT_INCOME_YEAR, + "survey_year": routing.CURRENT_INCOME_YEAR + 1, + "source_admission_issued": False, +} +# Pinned by value, so a silent edit to the owner's table is caught here rather +# than read back out of the same table the assertions came from. +PUBLISHED_DISABILITY_CODES = { + 0: "NIU", + 1: "worker's compensation", + 2: "company or union disability", + 3: "federal government disability", + 4: "US military retirement disability", + 5: "state or local government employee disability", + 6: "US railroad retirement disability", + 7: "accident or disability insurance", + 8: "blacklung miners disability", + 9: "state temporary sickness", + 10: "other or don't know", +} +UNRELATED_CELLS = { + "PNSN_VAL": "700", + "ANN_VAL": "-1", + "PEN_YN": "1", + "PEN_SC1": "3", + "PEN_VAL1": "700", + "SUR_YN": "1", + "SUR_SC1": "8", + "SUR_VAL1": "900", + "SRVS_VAL": "900", + "DST_VAL1": "150", + "DBTN_VAL": "150", + "I_PENVAL1": "4", + "TPEN_VAL1": "1", + "DIS_CS": "1", + "DIS_HP": "1", + "I_DISCS": "1", + "I_DISHP": "1", + "DSAB_VAL": "999", +} + + +# Module level, with no closure and no default: the implementation fence hashes +# a function's closure cells, so a recorder that captured a growing list would +# read as a second mutation rather than as the call it is trying to observe. +_REAL_ARCHIVED = legacy.derive_us_disability_benefits_from_asec +_ARCHIVED_CALLS: list[dict] = [] + + +def _recording_archived(frame, **keywords): + _ARCHIVED_CALLS.append(dict(keywords)) + return _REAL_ARCHIVED(frame, **keywords) + + +def row(**changes): + """One invented ASEC person; a single non-workers-compensation slot.""" + return { + **{name: "0" for name in detail.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "DIS_YN": "1", + "DIS_SC1": "6", + "DIS_VAL1": "200", + **changes, + } + + +def basis(*rows): + literals = pd.DataFrame(list(rows)) + frame = detail.project_retirement_detail_literals(literals) + frame.index = pd.Index(range(10, 10 + len(frame)), name="person_id") + frame["native_person_id"] = np.arange(101, 101 + len(frame), dtype="int64") + return literals, frame + + +def project(*rows): + return od.project_other_disability(basis(*rows)[1]) + + +def one(**changes): + return project(row(**changes)).iloc[0] + + +def values(*rows): + literals, frame = basis(*rows) + literals.index = frame.index.copy() + return od.compose_other_disability( + detail.CurrentAsecRetirementDetailValues(frame, literals, dict(DETAIL_EVIDENCE)) + ) + + +def receiving(result, *, clones=2, acs_rows=0, unrelated=True): + """Two clones of each original source person, plus optional ACS rows.""" + records, person_id = [], 1 + for source, native in zip( + result.person.index, result.person.native_person_id, strict=True + ): + for clone in range(clones): + records.append((person_id, int(source), clone, int(native), "asec")) + person_id += 1 + for offset in range(acs_rows): + for clone in range(clones): + records.append((person_id, 9000 + offset, clone, 7000 + offset, "acs")) + person_id += 1 + people = pd.DataFrame( + records, + columns=[ + "person_id", + provenance.support_source_id_column("person"), + provenance.support_clone_index_column("person"), + provenance.spine_source_id_column("person"), + provenance.support_channel_column("person"), + ], + ) + for column in people.columns[:-1]: + people[column] = people[column].astype("int64") + if unrelated: + people["age"] = np.arange(len(people), dtype="int64") + 20 + people["unrelated_leaf"] = np.arange(len(people), dtype="float64") + return SimpleNamespace(person=people) + + +# -------------------------------------------------------------------------- +# The two published slots and the workers' compensation exclusion +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "first_code,first_amount,second_code,second_amount,total,reason", + [ + ("6", "200", "0", "0", 200.0, "known_positive"), + ("0", "0", "0", "0", None, "affirmed_receipt_without_reported_source"), + ("2", "300", "7", "400", 700.0, "known_positive"), + ("1", "500", "0", "0", 0.0, "known_zero_workers_compensation_only"), + ("2", "300", "1", "400", 300.0, "known_positive"), + ("1", "500", "9", "250", 250.0, "known_positive"), + ("1", "500", "1", "400", 0.0, "known_zero_workers_compensation_only"), + ], +) +def test_both_slots_sum_only_non_workers_compensation_sources( + first_code, first_amount, second_code, second_amount, total, reason +): + value = one( + DIS_SC1=first_code, + DIS_VAL1=first_amount, + DIS_SC2=second_code, + DIS_VAL2=second_amount, + ) + assert value[od.REASON_COLUMN] == reason + assert bool(value[od.KNOWN_COLUMN]) is (total is not None) + if total is None: + assert pd.isna(value[od.AMOUNT_COLUMN]) + else: + assert value[od.AMOUNT_COLUMN] == total + + +def test_the_published_source_code_table_is_pinned_by_value(): + assert dict(detail.DISABILITY_CODES) == PUBLISHED_DISABILITY_CODES + assert od.WORKERS_COMPENSATION_CODE == 1 + assert od.WORKERS_COMPENSATION_LABEL == PUBLISHED_DISABILITY_CODES[1] + assert od.WORKERS_COMPENSATION_LABEL == "worker's compensation" + + +@pytest.mark.parametrize("code", sorted(PUBLISHED_DISABILITY_CODES)) +def test_every_published_source_code_is_admitted_or_excluded_explicitly(code): + value = one(DIS_SC1=str(code), DIS_VAL1="400") + if code == 0: + # A yes receipt with a nonzero amount but no source contradicts itself. + assert value.DIS_VAL1_slot_kind == "unresolved_slot_reporting" + assert not value[od.KNOWN_COLUMN] + return + if code == od.WORKERS_COMPENSATION_CODE: + assert value.DIS_VAL1_slot_kind == "excluded_workers_compensation" + assert value[od.AMOUNT_COLUMN] == 0.0 + assert value[od.REASON_COLUMN] == "known_zero_workers_compensation_only" + return + assert value.DIS_VAL1_slot_kind == "reported_source_slot" + assert value[od.AMOUNT_COLUMN] == 400.0 + assert value.DIS_SC1_label == PUBLISHED_DISABILITY_CODES[code] + + +def test_workers_compensation_exclusion_is_bound_to_its_printed_meaning(monkeypatch): + codes = dict(detail.DISABILITY_CODES) + codes[od.WORKERS_COMPENSATION_CODE] = "renamed by an editor" + monkeypatch.setattr(detail, "SOURCE_CODES", {**detail.SOURCE_CODES, "DIS": codes}) + monkeypatch.setattr(detail, "DISABILITY_CODES", codes) + with pytest.raises(ValueError, match="PRINTED_SEMANTICS"): + one() + + +@pytest.mark.parametrize( + "amount,status,kind", + [ + ("500", "known_receipt", "excluded_workers_compensation"), + ("0", "ambiguous_recipient_zero", "excluded_workers_compensation"), + ("", "missing_amount", "unresolved_slot_reporting"), + ("abc", "invalid_amount_literal", "unresolved_slot_reporting"), + ("1000000", "invalid_amount_literal", "unresolved_slot_reporting"), + ], +) +def test_workers_compensation_excludes_on_a_readable_code_not_on_its_amount( + amount, status, kind +): + value = one(DIS_SC1="1", DIS_VAL1=amount) + assert value.DIS_VAL1_reporting_status == status + assert value.DIS_VAL1_slot_kind == kind + assert bool(value[od.KNOWN_COLUMN]) is (kind == "excluded_workers_compensation") + + +@pytest.mark.parametrize( + "amount,status,kind", + [ + ("500", "known_receipt", "excluded_workers_compensation"), + ("0", "ambiguous_recipient_zero", "excluded_workers_compensation"), + ("", "missing_amount", "unresolved_slot_reporting"), + ("abc", "invalid_amount_literal", "unresolved_slot_reporting"), + ], +) +def test_the_second_slot_excludes_on_the_same_terms(amount, status, kind): + value = one(DIS_SC1="2", DIS_VAL1="300", DIS_SC2="1", DIS_VAL2=amount) + assert value.DIS_VAL2_reporting_status == status + assert value.DIS_VAL2_slot_kind == kind + assert bool(value[od.KNOWN_COLUMN]) is (kind == "excluded_workers_compensation") + if kind == "excluded_workers_compensation": + assert value[od.AMOUNT_COLUMN] == 300.0 + + +@pytest.mark.parametrize( + "age,known", [("0", False), ("14", False), ("15", True), ("16", True)] +) +def test_the_reporting_universe_starts_at_the_printed_age(age, known): + value = one(A_AGE=age) + assert bool(value[od.KNOWN_COLUMN]) is known + assert od.REPORTING_AGE == 15 + if known: + assert value[od.AMOUNT_COLUMN] == 200.0 + else: + assert value[od.REASON_COLUMN] == "outside_age_universe" + + +def test_receipt_universe_drift_refuses(monkeypatch): + entry = detail.RECEIPT_ENTRIES["DIS_YN"] + changed = routing.ReceiptEntry(*entry[:4], "All Persons", entry[5]) + monkeypatch.setattr( + detail, "RECEIPT_ENTRIES", {**detail.RECEIPT_ENTRIES, "DIS_YN": changed} + ) + with pytest.raises(ValueError, match="PRINTED_SEMANTICS"): + one() + + +def test_a_second_slot_is_read_independently_of_the_first(): + value = one(DIS_SC1="0", DIS_VAL1="0", DIS_SC2="4", DIS_VAL2="750") + assert value.DIS_VAL1_slot_kind == "unused_source_slot" + assert value.DIS_VAL2_slot_kind == "reported_source_slot" + assert value[od.AMOUNT_COLUMN] == 750.0 + assert value.DIS_VAL1_slot_contribution == 0.0 + assert value.DIS_VAL2_slot_contribution == 750.0 + + +# -------------------------------------------------------------------------- +# Observed zero versus NIU, missing, outside universe and contradiction +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "changes,known,reason", + [ + ( + {"DIS_YN": "2", "DIS_SC1": "0", "DIS_VAL1": "0"}, + True, + "known_zero_nonreceipt", + ), + ( + {"DIS_YN": "0", "DIS_SC1": "0", "DIS_VAL1": "0"}, + False, + "niu_not_observed_zero", + ), + ({"DIS_VAL1": ""}, False, "unresolved_slot_reporting"), + ({"DIS_YN": ""}, False, "unresolved_slot_reporting"), + ({"DIS_SC1": ""}, False, "unresolved_slot_reporting"), + ({"DIS_YN": "3"}, False, "unresolved_slot_reporting"), + ({"DIS_SC1": "11"}, False, "unresolved_slot_reporting"), + ({"DIS_YN": "2", "DIS_SC1": "6"}, False, "unresolved_slot_reporting"), + ({"DIS_SC1": "6", "DIS_VAL1": "0"}, False, "unresolved_slot_reporting"), + ( + {"A_AGE": "14", "DIS_YN": "0", "DIS_SC1": "0", "DIS_VAL1": "0"}, + False, + "outside_age_universe", + ), + ( + {"A_AGE": "0", "DIS_YN": "0", "DIS_SC1": "0", "DIS_VAL1": ""}, + False, + "outside_age_universe", + ), + ], +) +def test_unobserved_slots_never_become_an_observed_zero(changes, known, reason): + value = one(**changes) + assert bool(value[od.KNOWN_COLUMN]) is known + assert value[od.REASON_COLUMN] == reason + if not known: + assert pd.isna(value[od.AMOUNT_COLUMN]) + else: + assert value[od.AMOUNT_COLUMN] == 0.0 + + +def test_a_recipient_zero_under_a_reported_source_stays_ambiguous(): + value = one(DIS_SC1="6", DIS_VAL1="0") + assert value.DIS_VAL1_reporting_status == "ambiguous_recipient_zero" + assert value.DIS_VAL1_slot_kind == "unresolved_slot_reporting" + assert pd.isna(value.DIS_VAL1_slot_contribution) + + +def test_an_affirmed_receipt_without_a_populated_source_is_not_zero(): + value = one(DIS_SC1="0", DIS_VAL1="0", DIS_SC2="0", DIS_VAL2="0") + assert value.DIS_VAL1_slot_kind == value.DIS_VAL2_slot_kind == "unused_source_slot" + assert value[od.REASON_COLUMN] == "affirmed_receipt_without_reported_source" + assert pd.isna(value[od.AMOUNT_COLUMN]) + + +def test_one_unresolved_slot_withholds_the_whole_person(): + value = one(DIS_SC1="6", DIS_VAL1="200", DIS_SC2="2", DIS_VAL2="") + assert value.DIS_VAL1_slot_kind == "reported_source_slot" + assert value.DIS_VAL2_slot_kind == "unresolved_slot_reporting" + assert not value[od.KNOWN_COLUMN] + assert pd.isna(value[od.AMOUNT_COLUMN]) + + +def test_reason_and_slot_vocabularies_stay_closed(): + frame = project( + row(), + row(DIS_SC1="1", DIS_VAL1="500"), + row(DIS_YN="2", DIS_SC1="0", DIS_VAL1="0"), + row(DIS_YN="0", DIS_SC1="0", DIS_VAL1="0"), + row(A_AGE="4", DIS_YN="0", DIS_SC1="0", DIS_VAL1="0"), + row(DIS_VAL1=""), + row(DIS_SC1="0", DIS_VAL1="0"), + ) + assert set(frame[od.REASON_COLUMN]) <= set(od.REASONS) + assert set(frame[od.REASON_COLUMN]) == set(od.REASONS) - {od.UNOBSERVED_REASON} + for name in od.AMOUNT_FIELDS: + assert set(frame[name + "_slot_kind"]) <= set(od.SLOT_KINDS) + + +# -------------------------------------------------------------------------- +# The retired arithmetic is reused, and its divergences are recorded +# -------------------------------------------------------------------------- + + +def test_the_archived_two_slot_arithmetic_is_reused_verbatim(): + frame = project( + row(DIS_SC1="2", DIS_VAL1="300", DIS_SC2="1", DIS_VAL2="400"), + row(DIS_SC1="1", DIS_VAL1="500"), + ) + expected = legacy.derive_us_disability_benefits_from_asec( + pd.DataFrame( + { + "DIS_VAL1": [300.0, 500.0], + "DIS_SC1": [2.0, 1.0], + "DIS_VAL2": [400.0, 0.0], + "DIS_SC2": [1.0, 0.0], + }, + index=frame.index, + ) + )[legacy.US_DISABILITY_BENEFITS_OUTPUT_COLUMNS[0]] + assert list(frame.other_disability_archived_arithmetic_amount) == list(expected) + assert list(frame[od.AMOUNT_COLUMN]) == list(expected) + assert frame.other_disability_archived_arithmetic_agrees.all() + + +@pytest.mark.parametrize( + "changes", + [ + {"DIS_YN": "0", "DIS_SC1": "0", "DIS_VAL1": "0"}, + {"A_AGE": "3", "DIS_YN": "0", "DIS_SC1": "0", "DIS_VAL1": "0"}, + {"DIS_SC1": "0", "DIS_VAL1": "0", "DIS_SC2": "0", "DIS_VAL2": "0"}, + ], +) +def test_archived_zeroes_are_recorded_but_never_adopted(changes): + value = one(**changes) + assert value.other_disability_archived_arithmetic_evaluable + assert value.other_disability_archived_arithmetic_amount == 0.0 + assert not value[od.KNOWN_COLUMN] + assert pd.isna(value[od.AMOUNT_COLUMN]) + assert pd.isna(value.other_disability_archived_arithmetic_agrees) + + +def test_a_contradictory_out_of_universe_row_keeps_a_positive_archived_reading(): + """The archived arithmetic did not only read zero where nobody was asked.""" + value = one(A_AGE="10") + assert value.DIS_VAL1_reporting_status == ( + "contradictory_outside_reporting_universe" + ) + assert value[od.REASON_COLUMN] == "outside_age_universe" + assert value.other_disability_archived_arithmetic_evaluable + assert value.other_disability_archived_arithmetic_amount == 200.0 + assert pd.isna(value[od.AMOUNT_COLUMN]) + assert pd.isna(value.other_disability_archived_arithmetic_agrees) + + +@pytest.mark.parametrize( + "changes", + [ + {"DIS_VAL1": "abc"}, + {"DIS_VAL1": ""}, + # Outside the published code table. The archived function accepted any + # finite number, so this replay is narrower than the retired pipeline. + {"DIS_SC1": "11"}, + {"DIS_YN": "1", "DIS_SC1": "99"}, + ], +) +def test_the_replay_is_evaluable_only_on_the_owners_parsed_fields(changes): + value = one(**changes) + assert not value.other_disability_archived_arithmetic_evaluable + assert pd.isna(value.other_disability_archived_arithmetic_amount) + assert pd.isna(value.other_disability_archived_arithmetic_agrees) + assert not value[od.KNOWN_COLUMN] + + +def test_archived_parameter_drift_refuses(monkeypatch): + monkeypatch.setattr( + legacy, + "US_DISABILITY_BENEFITS_REQUIRED_SOURCE_COLUMNS", + ("DIS_VAL1", "DIS_SC1"), + ) + with pytest.raises(ValueError, match="ARCHIVED_ROSTER"): + one() + + +def test_owner_roster_drift_refuses(monkeypatch): + monkeypatch.setattr(detail, "RETAINED_MONEY_FIELDS", ("PNSN_VAL",)) + with pytest.raises(ValueError, match="OWNER_ROSTER"): + one() + + +@pytest.mark.parametrize( + "name,replacement", + [ + ("DIS_SC1", (2, 461, 44, "6C-23", "DIS_YN > 0")), + ("DIS_SC2", (2, 463, 44, "6C-23", "All Persons aged 15+")), + ], +) +def test_source_code_universe_drift_refuses(monkeypatch, name, replacement): + entries = dict(detail.SOURCE_ENTRIES) + entries[name] = replacement + monkeypatch.setattr(detail, "SOURCE_ENTRIES", entries) + with pytest.raises(ValueError, match="PRINTED_SEMANTICS"): + one() + + +# -------------------------------------------------------------------------- +# Conditional allocation provenance and topcode censoring +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "flag,changes,status,published,family", + [ + ( + "I_DISVL1", + {}, + "not_allocated_in_flag_universe", + "published_flags_all_zero", + "not_allocated_in_flag_universe", + ), + ( + "I_DISVL2", + {}, + "outside_flag_universe", + "published_flags_all_zero", + "not_allocated_in_flag_universe", + ), + ( + "I_DISVL1", + {"I_DISVL1": "5"}, + "publisher_allocated", + "publisher_allocated", + "publisher_allocated_in_flag_universe", + ), + ( + "I_DISVL1", + {"I_DISVL1": ""}, + "allocation_flag_not_populated", + "allocation_flag_not_populated", + "allocation_flag_not_populated", + ), + ( + "I_DISSC1", + {"I_DISSC1": "3"}, + "unresolved_allocation_literal", + "unresolved_allocation_provenance", + "unresolved_allocation_provenance", + ), + ( + "I_DISYN", + {"I_DISYN": "7"}, + "publisher_allocated", + "publisher_allocated", + "publisher_allocated_in_flag_universe", + ), + ], +) +def test_allocation_flags_keep_their_conditional_universes( + flag, changes, status, published, family +): + value = one(**changes) + assert value[flag + "_allocation_status"] == status + assert value.other_disability_published_flag_origin == published + assert value.other_disability_allocation_status == family + # Provenance never becomes knownness in either direction. + assert bool(value[od.KNOWN_COLUMN]) is True + assert value[od.AMOUNT_COLUMN] == 200.0 + assert not value.other_disability_allocation_qualifies_receipt + + +def test_a_flag_outside_its_universe_does_not_allocate_this_family(): + """The owner's family reading is universe-blind; ours is not, and says so.""" + value = one(I_DISVL2="5") + assert value.I_DISVL2_flag_universe is not pd.NA + assert not value.I_DISVL2_flag_universe + assert value.I_DISVL2_allocation_status == "outside_flag_universe" + assert value.other_disability_published_flag_origin == "publisher_allocated" + assert value.other_disability_allocation_status == "not_allocated_in_flag_universe" + assert value[od.AMOUNT_COLUMN] == 200.0 + + +def test_no_flag_in_its_own_universe_is_reported_as_such(): + """Every printed universe here is ``X > 0``, so an all-zero row has none.""" + value = one(DIS_YN="0", DIS_SC1="0", DIS_VAL1="0") + assert value.other_disability_allocation_status == "no_flag_in_its_universe" + assert value.other_disability_published_flag_origin == "published_flags_all_zero" + assert value[od.REASON_COLUMN] == "niu_not_observed_zero" + + +def test_a_nonreceipt_answer_is_still_inside_the_receipt_flag_universe(): + value = one(DIS_YN="2", DIS_SC1="0", DIS_VAL1="0") + assert value.I_DISYN_flag_universe is not pd.NA + assert value.I_DISYN_allocation_status == "not_allocated_in_flag_universe" + assert value.other_disability_allocation_status == "not_allocated_in_flag_universe" + assert value[od.REASON_COLUMN] == "known_zero_nonreceipt" + + +def test_a_definite_allocation_outranks_an_unresolved_flag(): + value = one(I_DISVL1="5", I_DISSC1="3") + assert value.I_DISVL1_allocation_status == "publisher_allocated" + assert value.I_DISSC1_allocation_status == "unresolved_allocation_literal" + # The owner's universe-blind reading leads with the unresolved literal; + # this family's reading leads with the allocation it did establish. + assert ( + value.other_disability_published_flag_origin + == "unresolved_allocation_provenance" + ) + assert ( + value.other_disability_allocation_status + == "publisher_allocated_in_flag_universe" + ) + + +def test_a_missing_flag_outside_its_universe_is_not_reported_as_missing(): + value = one(I_DISVL2="") + assert value.I_DISVL2_allocation_status == "outside_flag_universe" + assert value.other_disability_allocation_status == "not_allocated_in_flag_universe" + + +def test_an_unreadable_amount_leaves_its_flag_universe_unresolved(): + value = one(DIS_VAL1="") + assert value.I_DISVL1_flag_universe is pd.NA + assert value.I_DISVL1_allocation_status == "unresolved_flag_universe" + assert not value[od.KNOWN_COLUMN] + + +def test_work_limitation_flags_are_not_income_allocation_provenance(): + assert "I_DISCS" not in od.ALLOCATION_FIELDS + assert "I_DISHP" not in od.ALLOCATION_FIELDS + plain = one() + limited = one(DIS_CS="1", DIS_HP="1", I_DISCS="4", I_DISHP="4") + assert ( + plain.other_disability_published_flag_origin + == limited.other_disability_published_flag_origin + ) + assert ( + plain.other_disability_allocation_status + == limited.other_disability_allocation_status + ) + assert limited[od.AMOUNT_COLUMN] == plain[od.AMOUNT_COLUMN] + + +@pytest.mark.parametrize( + "changes,censored", + [ + ({}, False), + ({"TDISVAL1": "1"}, True), + ({"TDISVAL1": ""}, None), + ({"DIS_SC1": "1", "DIS_VAL1": "500", "TDISVAL1": "1"}, False), + ({"DIS_YN": "2", "DIS_SC1": "0", "DIS_VAL1": "0", "TDISVAL1": "1"}, False), + ], +) +def test_topcodes_describe_only_the_dollars_this_leaf_admits(changes, censored): + value = one(**changes) + if censored is None: + assert pd.isna(value.other_disability_topcoded) + else: + assert bool(value.other_disability_topcoded) is censored + + +@pytest.mark.parametrize( + "first_flag,second_flag,censored", + [ + ("1", "", True), + ("", "1", True), + ("0", "", None), + ("", "", None), + ("0", "0", False), + ("1", "0", True), + ], +) +def test_one_readable_topcode_settles_two_admitted_slots( + first_flag, second_flag, censored +): + value = one( + DIS_SC1="2", + DIS_VAL1="300", + DIS_SC2="7", + DIS_VAL2="400", + TDISVAL1=first_flag, + TDISVAL2=second_flag, + ) + assert value[od.AMOUNT_COLUMN] == 700.0 + if censored is None: + assert pd.isna(value.other_disability_topcoded) + else: + assert bool(value.other_disability_topcoded) is censored + + +def test_an_unknown_person_has_no_topcode_reading(): + assert pd.isna(one(DIS_VAL1="", TDISVAL1="1").other_disability_topcoded) + + +# -------------------------------------------------------------------------- +# Owner agreement, mutation and unrelated-cell preservation +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("column", sorted(set(od.required_basis_columns()))) +def test_every_declared_basis_column_is_required(column): + _, frame = basis(row()) + with pytest.raises(ValueError, match="BASIS_COLUMNS"): + od.project_other_disability(frame.drop(columns=[column])) + + +def test_unrelated_source_cells_do_not_move_this_leaf(): + plain = project(row()) + noisy = project(row(**UNRELATED_CELLS)) + pd.testing.assert_frame_equal(plain, noisy) + + +def test_owner_knownness_disagreement_refuses(): + _, frame = basis(row()) + frame["DIS_VAL1_amount_known"] = ~frame.DIS_VAL1_amount_known + with pytest.raises(ValueError, match="OWNER_KNOWNNESS"): + od.project_other_disability(frame) + + +def test_money_owner_axes_are_cross_checked_when_supplied(): + _, frame = basis(row()) + for name in od.AMOUNT_FIELDS: + frame[name + "_parent_statuses"] = np.int16(2) + frame[name + "_parent_validity"] = np.int16(1) + frame[name + "_parent_zero_origin"] = np.int16(0) + projected = od.project_other_disability(frame) + assert bool(projected.other_disability_money_owner_axes_present.iloc[0]) + assert projected.DIS_VAL1_parent_validity.iloc[0] == 1 + frame["DIS_VAL1_parent_validity"] = np.int16(0) + with pytest.raises(ValueError, match="PARENT_VALIDITY"): + od.project_other_disability(frame) + + +def test_money_owner_axes_are_optional_and_declared(): + projected = project(row()) + assert not projected.other_disability_money_owner_axes_present.iloc[0] + assert "DIS_VAL1_parent_validity" not in projected + + +@pytest.mark.parametrize( + "mutation", + [ + "own_projection", + "own_reporting_age", + "own_report_prefix", + "owner_qualifier", + "owner_literal_projection", + "owner_slot_status", + "owner_capture", + "owner_amount_comparison", + "owner_source_codes", + "owner_known_statuses", + "routing_receipt_status", + "routing_read_capture", + "archived_arithmetic", + "archived_input_guard", + "archived_parameters", + ], +) +def test_composition_refuses_a_mutated_owner(monkeypatch, mutation): + literals, frame = basis(row()) + literals.index = frame.index.copy() + owner = detail.CurrentAsecRetirementDetailValues( + frame, literals, dict(DETAIL_EVIDENCE) + ) + if mutation == "own_projection": + original = od._slot_kind + monkeypatch.setattr(od, "_slot_kind", lambda *a, **k: original(*a, **k)) + elif mutation == "own_reporting_age": + monkeypatch.setattr(od, "REPORTING_AGE", 65) + elif mutation == "own_report_prefix": + monkeypatch.setattr(od, "REPORT_PREFIX", "invented_") + elif mutation in ( + "owner_qualifier", + "owner_literal_projection", + "owner_slot_status", + "owner_capture", + "owner_amount_comparison", + "routing_receipt_status", + "routing_read_capture", + "archived_input_guard", + ): + owner_module, name = { + "owner_qualifier": (detail, "qualify_current_asec_retirement_detail"), + "owner_literal_projection": (detail, "project_retirement_detail_literals"), + "owner_slot_status": (detail, "_slot_status"), + # The owner cross-checks retained amounts against the money owner, + # but the source codes this leaf reads come only from the capture. + "owner_capture": (detail, "_capture_member"), + "owner_amount_comparison": (detail, "_compare_amount"), + "routing_receipt_status": (routing, "receipt_status"), + "routing_read_capture": (routing, "_read_capture"), + "archived_input_guard": (legacy, "_strict_numeric_source"), + }[mutation] + original = getattr(owner_module, name) + monkeypatch.setattr(owner_module, name, lambda *a, **k: original(*a, **k)) + elif mutation == "owner_source_codes": + codes = dict(detail.DISABILITY_CODES) + codes[10] = "edited in place" + monkeypatch.setattr(detail, "DISABILITY_CODES", codes) + elif mutation == "owner_known_statuses": + monkeypatch.setattr(routing, "KNOWN_AMOUNT_STATUSES", ("known_receipt",)) + elif mutation == "archived_arithmetic": + original = legacy.derive_us_disability_benefits_from_asec + monkeypatch.setattr( + legacy, + "derive_us_disability_benefits_from_asec", + lambda *a, **k: original(*a, **k), + ) + else: + monkeypatch.setattr( + legacy, + "_EXPECTED_DIRECT_PARAMETERS", + {**legacy._EXPECTED_DIRECT_PARAMETERS, "workers_compensation_code": 9}, + ) + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + od.compose_other_disability(owner) + + +def test_composition_refuses_an_owner_that_changes_under_it(monkeypatch): + """Mutate the borrowed table for real, between the two seals.""" + literals, frame = basis(row()) + literals.index = frame.index.copy() + owner = detail.CurrentAsecRetirementDetailValues( + frame, literals, dict(DETAIL_EVIDENCE) + ) + real = detail.retirement_detail_values_seal + mutated = [] + + def seal_then_mutate(value): + stamp = real(value) + if not mutated: + mutated.append(True) + value.person.loc[10, "DIS_VAL1_published_amount"] = 999.0 + return stamp + + monkeypatch.setattr(detail, "retirement_detail_values_seal", seal_then_mutate) + monkeypatch.setattr(od, "_LIVE", od._live()) + with pytest.raises(ValueError, match="DETAIL_VALUES_CHANGED"): + od.compose_other_disability(owner) + assert owner.person.loc[10, "DIS_VAL1_published_amount"] == 999.0 + + +def test_the_archived_function_itself_is_called_with_its_own_parameters(monkeypatch): + """Equivalent arithmetic is not reuse; record the actual call.""" + _ARCHIVED_CALLS.clear() + monkeypatch.setattr( + legacy, "derive_us_disability_benefits_from_asec", _recording_archived + ) + monkeypatch.setattr(od, "_LIVE", od._live()) + result = values(row(DIS_SC1="2", DIS_VAL1="300")) + assert _ARCHIVED_CALLS == [ + { + "first_amount_source": "DIS_VAL1", + "first_code_source": "DIS_SC1", + "second_amount_source": "DIS_VAL2", + "second_code_source": "DIS_SC2", + "workers_compensation_code": 1, + "output_column": legacy.US_DISABILITY_BENEFITS_OUTPUT_COLUMNS[0], + } + ] + assert result.person.other_disability_archived_arithmetic_amount.iloc[0] == 300.0 + + +def test_a_shared_constant_mapping_cannot_be_retuned_in_place(monkeypatch): + """The archived parameters are another module's live dict, not a copy.""" + monkeypatch.setattr( + legacy, "_EXPECTED_DIRECT_PARAMETERS", dict(legacy._EXPECTED_DIRECT_PARAMETERS) + ) + original = dict(od._ARCHIVED) + od._ARCHIVED["first_amount_source"] = "DIS_VAL2" + try: + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + values(row()) + finally: + od._ARCHIVED.clear() + od._ARCHIVED.update(original) + # Restoring the constant reopens the fence, so the refusal was the mutation. + assert values(row()).person[od.AMOUNT_COLUMN].iloc[0] == 200.0 + + +def test_the_recorded_implementation_digest_is_the_imported_bytes(): + digest = hashlib.sha256(Path(od.__file__).read_bytes()).hexdigest() + assert od._IMPLEMENTATION_SHA256 == digest + evidence = values(row()).evidence + assert evidence["implementation_sha256"] == digest + scope = evidence["implementation_sha256_scope"] + assert "at the end of its own import" in scope + # The pin does not attest the loader's own earlier read, and says so. + assert "is not attested" in scope + + +def test_a_file_that_changed_after_import_refuses(monkeypatch): + monkeypatch.setattr(od, "_IMPLEMENTATION_SHA256", "0" * 64) + monkeypatch.setattr(od, "_LIVE", od._live()) + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + values(row()) + + +def test_the_public_qualifier_requires_the_live_preparation_owner(): + for foreign in (None, SimpleNamespace(), object()): + with pytest.raises(ValueError, match="PREPARATION_TYPE"): + od.qualify_current_asec_other_disability(foreign) + + +def test_live_preparation_qualifies_other_disability_from_real_detail_owner( + tmp_path, monkeypatch +): + from test_us_current_asec_retirement_detail_source import prepared + + parent = prepared(detail, tmp_path, monkeypatch) + result = od.qualify_current_asec_other_disability(parent) + observed = result.person.set_index("native_person_id") + assert observed.loc[105, od.AMOUNT_COLUMN] == 200 + assert bool(observed.loc[105, od.KNOWN_COLUMN]) + assert observed.loc[107, od.AMOUNT_COLUMN] == 0 + assert bool(observed.loc[107, od.KNOWN_COLUMN]) + assert not bool(observed.loc[106, od.KNOWN_COLUMN]) + assert pd.isna(observed.loc[106, od.AMOUNT_COLUMN]) + assert result.evidence["source_admission_issued"] is False + assert result.evidence["release_eligible"] is False + + +@pytest.mark.parametrize("mutation", ["semantic_constant", "bound_qualifier"]) +def test_live_preparation_refuses_implementation_changed_by_final_owner_return( + tmp_path, monkeypatch, mutation +): + from test_us_current_asec_retirement_detail_source import prepared + + parent = prepared(detail, tmp_path, monkeypatch) + checked = type(parent)._checked.__code__ + composed = od.compose_other_disability.__code__ + calls = [] + armed = [] + + def after_return(code, offset, result): + if code is composed: + armed.append(True) + elif code is checked and armed: + calls.append(True) + if mutation == "semantic_constant": + monkeypatch.setattr(od, "REPORTING_AGE", 16) + else: + monkeypatch.setattr( + detail, + "qualify_current_asec_retirement_detail", + lambda preparation: None, + ) + + # Observe actual callback returns without replacing the preparation, + # qualified detail operation, or the implementation seal under test. Local + # monitoring leaves the test runner's global no-fit guard intact. + monitor = sys.monitoring + tool_id = 4 + assert monitor.get_tool(tool_id) is None + monitor.use_tool_id(tool_id, "other-disability-final-owner-test") + monitor.register_callback(tool_id, monitor.events.PY_RETURN, after_return) + monitor.set_local_events(tool_id, checked, monitor.events.PY_RETURN) + monitor.set_local_events(tool_id, composed, monitor.events.PY_RETURN) + try: + with pytest.raises( + ValueError, match="OTHER_DISABILITY_SOURCE_IMPLEMENTATION_CHANGED" + ): + od.qualify_current_asec_other_disability(parent) + finally: + monitor.set_local_events(tool_id, checked, 0) + monitor.set_local_events(tool_id, composed, 0) + monitor.register_callback(tool_id, monitor.events.PY_RETURN, None) + monitor.free_tool_id(tool_id) + assert len(armed) == len(calls) == 1 + + +@pytest.mark.parametrize( + "mutation", + ["type", "protocol", "year", "survey_year", "admission", "native"], +) +def test_composition_refuses_a_foreign_or_unqualified_owner(mutation): + literals, frame = basis(row()) + literals.index = frame.index.copy() + evidence = dict(DETAIL_EVIDENCE) + if mutation == "protocol": + evidence["protocol"] = "microcosm.us.invented.v1" + elif mutation == "year": + evidence["source_year"] = routing.CURRENT_INCOME_YEAR - 1 + elif mutation == "survey_year": + evidence["survey_year"] = routing.CURRENT_INCOME_YEAR + elif mutation == "admission": + evidence["source_admission_issued"] = True + elif mutation == "native": + frame = frame.drop(columns=["native_person_id"]) + owner = ( + SimpleNamespace(person=frame, asec_literals=literals, evidence=evidence) + if mutation == "type" + else detail.CurrentAsecRetirementDetailValues(frame, literals, evidence) + ) + expected = { + "type": "DETAIL_OWNER_TYPE", + "native": "DETAIL_NATIVE_AXIS", + }.get(mutation, "DETAIL_EVIDENCE") + with pytest.raises(ValueError, match=expected): + od.compose_other_disability(owner) + + +def test_values_seal_covers_masked_backing_and_evidence(): + result = values(row(), row(DIS_VAL1="")) + seal = od.other_disability_values_seal(result) + assert od.other_disability_values_seal(copy.deepcopy(result)) == seal + for kind in ("masked_backing", "mask", "label", "evidence"): + changed = copy.deepcopy(result) + if kind == "masked_backing": + changed.person[od.AMOUNT_COLUMN].array._data[1] = 17.0 + elif kind == "mask": + changed.person[od.AMOUNT_COLUMN].array._mask[0] = True + elif kind == "label": + changed.person.loc[10, od.REASON_COLUMN] = "known_positive " + else: + changed.evidence["invented"] = True + assert od.other_disability_values_seal(changed) != seal + + +def test_evidence_claims_nothing_beyond_the_observed_slots(): + result = values(row(), row(DIS_YN="0", DIS_SC1="0", DIS_VAL1="0")) + evidence = result.evidence + for claim in ( + "acs_completion_assigned", + "under15_completed_with_zero", + "niu_completed_with_zero", + "unknown_completed_with_zero", + "social_security_disability_included", + "workers_compensation_included", + "ssi_eligibility_assigned", + "taxability_assigned", + "topcode_corrected", + "allocation_flags_qualify_receipt", + "model_fitted", + "clone_redraw_issued", + "source_admission_issued", + "release_eligible", + ): + assert evidence[claim] is False + assert evidence["adds_raw_member_reader"] is False + assert evidence["member_capture_owner"] == detail.PROTOCOL + assert evidence["canonical_leaf"] == legacy.US_DISABILITY_BENEFITS_OUTPUT_COLUMNS[0] + assert evidence["retirement_detail_protocol"] == detail.PROTOCOL + assert evidence["workers_compensation_label"] == "worker's compensation" + assert evidence["rows"] == 2 and evidence["known_rows"] == 1 + assert evidence["reason_counts"]["niu_not_observed_zero"] == 1 + + +# -------------------------------------------------------------------------- +# Clone transport and receiver identity +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("clones", [1, 2, 3]) +def test_one_source_row_is_copied_to_every_clone_without_a_redraw(clones): + result = values(row(), row(DIS_SC1="1", DIS_VAL1="500")) + frame = receiving(result, clones=clones) + attachment = od.attach_other_disability_columns(result, frame) + leaf = attachment.columns["person", legacy.US_DISABILITY_BENEFITS_OUTPUT_COLUMNS[0]] + assert list(leaf) == [200.0] * clones + [0.0] * clones + assert attachment.receipt["clones_per_source"] == clones + assert attachment.receipt["redraw_issued"] is False + assert attachment.receipt["draws_consumed"] == 0 + assert attachment.receipt["unknown_completed_with_zero"] is False + assert attachment.receipt["values_sha256"] == od.other_disability_values_seal( + result + ) + reason = attachment.columns["person", od.REPORT_PREFIX + "reason"] + assert ( + list(reason) + == ["known_positive"] * clones + + ["known_zero_workers_compensation_only"] * clones + ) + + +def test_unobserved_arms_stay_unknown_rather_than_zero(): + result = values(row()) + frame = receiving(result, acs_rows=2) + attachment = od.attach_other_disability_columns(result, frame) + leaf = attachment.columns["person", legacy.US_DISABILITY_BENEFITS_OUTPUT_COLUMNS[0]] + known = attachment.columns["person", od.REPORT_PREFIX + "known"] + reason = attachment.columns["person", od.REPORT_PREFIX + "reason"] + assert list(leaf)[:2] == [200.0, 200.0] + assert all(pd.isna(value) for value in list(leaf)[2:]) + assert list(known) == [True, True] + [False] * 4 + assert list(reason)[2:] == [od.UNOBSERVED_REASON] * 4 + assert attachment.receipt["unobserved_channels"] == ["acs"] + assert attachment.receipt["unobserved_rows"] == 4 + assert attachment.receipt["observed_rows"] == 2 + + +def test_attachment_requires_receiver_source_identity(): + result = values(row(), row(DIS_VAL1="300")) + frame = receiving(result) + frame.person.loc[0, provenance.spine_source_id_column("person")] = 999 + with pytest.raises(ValueError, match="RECEIVER_SOURCE_IDENTITY"): + od.attach_other_disability_columns(result, frame) + + +@pytest.mark.parametrize( + "mutation,reason", + [ + ("missing_source", "SOURCE_ROSTER"), + ("extra_source", "SOURCE_ROSTER"), + ("acs_collision", "SOURCE_ROSTER"), + ("foreign_channel", "RECEIVING_CHANNEL_ROSTER"), + ("ragged_clones", "CLONE_COUNT"), + ("repeated_clone_index", "CLONE_INDEX_SET"), + ("missing_column", "RECEIVING_AXES"), + ("float_ids", "RECEIVING_AXES"), + ("duplicate_person", "RECEIVING_AXES"), + ("clone_native_disagreement", "CLONE_SOURCE_AGREEMENT"), + ], +) +def test_attachment_refuses_a_broken_receiving_roster(mutation, reason): + result = values(row(), row(DIS_VAL1="300")) + frame = receiving(result, acs_rows=1) + people = frame.person + source_column = provenance.support_source_id_column("person") + clone_column = provenance.support_clone_index_column("person") + channel_column = provenance.support_channel_column("person") + if mutation == "missing_source": + frame.person = people.iloc[2:].reset_index(drop=True) + elif mutation == "extra_source": + people.loc[0, source_column] = 555 + elif mutation == "acs_collision": + people.loc[4, source_column] = int(result.person.index[0]) + elif mutation == "foreign_channel": + people.loc[0, channel_column] = "puf_tax_detail" + elif mutation == "ragged_clones": + frame.person = people.iloc[1:].reset_index(drop=True) + elif mutation == "repeated_clone_index": + people.loc[1, clone_column] = 0 + elif mutation == "missing_column": + frame.person = people.drop(columns=[clone_column]) + elif mutation == "float_ids": + people[source_column] = people[source_column].astype("float64") + elif mutation == "duplicate_person": + people.loc[1, "person_id"] = people.loc[0, "person_id"] + else: + # An unobserved clone pair whose two rows claim different originals. + people.loc[5, provenance.spine_source_id_column("person")] = 7777 + with pytest.raises(ValueError, match=reason): + od.attach_other_disability_columns(result, frame) + + +def test_attachment_preserves_unrelated_receiving_cells(): + result = values(row()) + frame = receiving(result, acs_rows=1) + before = frame.person.copy(deep=True) + attachment = od.attach_other_disability_columns(result, frame) + pd.testing.assert_frame_equal(frame.person, before) + attached = {name for _, name in attachment.columns} + assert not attached & set(before.columns) + assert attached == set(attachment.receipt["attached_columns"]) + assert all(entity == "person" for entity, _ in attachment.columns) + assert "native_person_id" not in attached + assert od.REPORT_PREFIX + "native_person_id" not in attached + + +def test_attached_values_are_the_qualified_rows_themselves(): + """No redraw: every attached cell is the source person's own qualified cell.""" + result = values(row(), row(DIS_SC1="2", DIS_VAL1="450")) + frame = receiving(result, clones=2) + attachment = od.attach_other_disability_columns(result, frame) + sources = pd.Index( + frame.person[provenance.support_source_id_column("person")].to_numpy() + ) + for name in result.person.columns: + if name == "native_person_id": + continue + attached = attachment.columns["person", od.attached_name(name)] + expected = result.person[name].reindex(sources) + pd.testing.assert_series_equal( + attached.reset_index(drop=True), + expected.reset_index(drop=True), + check_names=False, + ) + + +def test_clone_transport_follows_identity_not_row_order(): + result = values(row(), row(DIS_SC1="2", DIS_VAL1="450")) + frame = receiving(result, clones=2) + leaf = legacy.US_DISABILITY_BENEFITS_OUTPUT_COLUMNS[0] + ordered = od.attach_other_disability_columns(result, frame) + shuffled_people = frame.person.iloc[[3, 0, 2, 1]].reset_index(drop=True) + shuffled = od.attach_other_disability_columns( + result, SimpleNamespace(person=shuffled_people) + ) + by_person = dict( + zip(shuffled_people.person_id, shuffled.columns["person", leaf], strict=True) + ) + for person_id, value in zip( + frame.person.person_id, ordered.columns["person", leaf], strict=True + ): + assert by_person[person_id] == value + + +def test_attachment_refuses_an_implementation_changed_by_the_receiver(monkeypatch): + """The receiver's own table() runs after the entry check.""" + result = values(row()) + people = receiving(result).person + + class HostileReceiver: + def table(self, name): + monkeypatch.setattr(od, "OUTPUT", "invented_leaf") + return people + + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + od.attach_other_disability_columns(result, HostileReceiver()) + + +def test_attachment_refuses_a_result_constructor_that_swaps_its_payload(monkeypatch): + """The receiver's own table() can reach the constructor called after it.""" + result = values(row()) + people = receiving(result).person + original = od.OtherDisabilityAttachment.__init__ + + def hijacked(self, columns, receipt): + original(self, dict(columns), dict(receipt)) + + class HostileReceiver: + def table(self, name): + monkeypatch.setattr(od.OtherDisabilityAttachment, "__init__", hijacked) + return people + + with pytest.raises(ValueError, match="ATTACHMENT_PAYLOAD_CHANGED"): + od.attach_other_disability_columns(result, HostileReceiver()) + + +def test_a_replaced_result_constructor_is_refused_before_it_is_called(monkeypatch): + result = values(row()) + frame = receiving(result) + original = od.OtherDisabilityAttachment.__init__ + monkeypatch.setattr( + od.OtherDisabilityAttachment, + "__init__", + lambda self, columns, receipt: original(self, columns, receipt), + ) + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + od.attach_other_disability_columns(result, frame) + + +def test_a_replaced_values_constructor_is_refused(monkeypatch): + literals, frame = basis(row()) + literals.index = frame.index.copy() + owner = detail.CurrentAsecRetirementDetailValues( + frame, literals, dict(DETAIL_EVIDENCE) + ) + original = od.CurrentAsecOtherDisabilityValues.__init__ + monkeypatch.setattr( + od.CurrentAsecOtherDisabilityValues, + "__init__", + lambda self, person, evidence: original(self, person, evidence), + ) + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + od.compose_other_disability(owner) + + +def test_a_copied_values_object_attaches_identically(): + result = values(row(), row(DIS_SC1="2", DIS_VAL1="450")) + frame = receiving(result) + original = od.attach_other_disability_columns(result, frame) + copied = od.attach_other_disability_columns(copy.deepcopy(result), frame) + assert original.receipt == copied.receipt + assert set(original.columns) == set(copied.columns) + for key, series in original.columns.items(): + pd.testing.assert_series_equal(series, copied.columns[key]) + + +def test_a_foreign_values_object_is_refused(): + result = values(row()) + frame = receiving(result) + foreign = SimpleNamespace(person=result.person, evidence=result.evidence) + with pytest.raises(ValueError, match="VALUES_TYPE"): + od.attach_other_disability_columns(foreign, frame) + + +def test_attachment_refuses_a_mutated_implementation(monkeypatch): + result = values(row()) + frame = receiving(result) + monkeypatch.setattr(od, "WORKERS_COMPENSATION_CODE", 2) + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + od.attach_other_disability_columns(result, frame) + + +def test_attached_names_keep_the_canonical_leaf_and_one_report_prefix(): + result = values(row()) + attachment = od.attach_other_disability_columns(result, receiving(result)) + names = sorted(name for _, name in attachment.columns) + leaf = legacy.US_DISABILITY_BENEFITS_OUTPUT_COLUMNS[0] + assert leaf in names + assert all(name == leaf or name.startswith(od.REPORT_PREFIX) for name in names) + assert not any( + name.startswith(od.REPORT_PREFIX + "other_disability") for name in names + ) diff --git a/packages/microcosm-build/tests/test_us_current_asec_property_basis.py b/packages/microcosm-build/tests/test_us_current_asec_property_basis.py new file mode 100644 index 000000000..db4500db2 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_property_basis.py @@ -0,0 +1,552 @@ +"""Pure invented descriptions; no source files, qualification, models or engines.""" + +import copy + +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +from microcosm.build.us_runtime.current_asec_property_basis import ( + build_asec_property_basis, +) +from microcosm.build.us_runtime.property_income_constants import ( + PROPERTY_COMPONENTS, + PROPERTY_REPORTED_TOTAL, +) + + +def code(frame, name, values, statuses=None): + frame[name + "_code"] = pd.array(values, dtype="Int16") + frame[name + "_literal_status"] = pd.array( + statuses or ["in_printed_range"] * len(values), dtype="string" + ) + + +def amount(frame, name, values, statuses=None): + if statuses is None: + statuses = ["known_nonreceipt" if v == 0 else "known_receipt" for v in values] + frame[name + "_published_amount"] = pd.array(values, dtype="Float64") + frame[name + "_literal_status"] = pd.array( + ["in_printed_range"] * len(values), dtype="string" + ) + frame[name + "_reporting_status"] = pd.array(statuses, dtype="string") + known = np.isin( + statuses, ["known_receipt", "known_nonreceipt", "observed_zero_component"] + ) + frame[name + "_amount_known"] = known + frame[name + "_amount"] = frame[name + "_published_amount"].where(known) + + +def inputs(n=3): + index = pd.Index(np.arange(n, dtype=np.int64) + 101, name="person_id") + base = pd.DataFrame( + { + "native_person_id": np.arange(n, dtype=np.int64) + 901, + "source_age": np.full(n, 40.0), + }, + index=index, + ) + interest, routing, dividend = (base.copy() for _ in range(3)) + amount(interest, "INT_VAL", [100.0] * n) + amount(interest, "TRDINT_VAL", [100.0] * n) + code(interest, "RINT_YN", [2] * n) + for k in (1, 2): + amount(interest, f"RINT_VAL{k}", [0.0] * n) + code(interest, f"RINT_SC{k}", [0] * n) + interest["TRINT_VAL1_literal"] = pd.array(["0"] * n, dtype="string") + interest["allocation_origin"] = pd.array( + ["publisher_allocated"] * n, dtype="string" + ) + routing["net_property_known_amount"] = np.full(n, -110.0) + routing["net_property_reporting_status"] = pd.array( + ["known_receipt"] * n, dtype="string" + ) + code(routing, "other_income_receipt", [2] * n) + code(routing, "other_income_category", [0] * n) + routing["other_income_reporting_status"] = pd.array( + ["known_nonreceipt"] * n, dtype="string" + ) + routing["other_income_routing_status"] = pd.array( + ["niu_category"] * n, dtype="string" + ) + amount(dividend, "DIV_VAL", [0.0] * n) + code(dividend, "DIV_YN", [2] * n) + code(dividend, "SUR_YN", [2] * n) + code(dividend, "SUR_SC1", [0] * n) + code(dividend, "SUR_SC2", [0] * n) + dividend["survivor_property_route_clear"] = pd.array([True] * n, dtype="boolean") + membership = pd.Series( + np.arange(n, dtype=np.int64) // 2 + 41, index=index, name="household_id" + ) + households = np.unique(membership) + weights = pd.Series( + np.arange(len(households), dtype=np.float64) * 3 + 2, + index=pd.Index(households, name="household_id"), + ) + return dict( + interest=interest, + income_routing=routing, + dividend=dividend, + original_household_membership=membership, + original_household_design_weights=weights, + ) + + +@pytest.mark.parametrize("rental,total", [(-110, -10), (-100, 0), (-40, 60), (50, 150)]) +def test_signed_anchors_preserve_positive_interest_and_losses(rental, total): + source = inputs() + source["income_routing"]["net_property_known_amount"] = float(rental) + before = copy.deepcopy(source) + result = build_asec_property_basis(**source) + assert result.person[PROPERTY_COMPONENTS[0]].eq(100).all() + assert result.person[PROPERTY_COMPONENTS[3]].eq(rental).all() + assert result.person[PROPERTY_REPORTED_TOTAL].eq(total).all() + assert result.person.joint_component_fit_eligible.all() + assert result.person.retirement_structural_zero_slot_count.eq(0).all() + assert ( + result.provenance["interest.allocation_origin"].eq("publisher_allocated").all() + ) + for name in ("interest", "income_routing", "dividend"): + assert_frame_equal(source[name], before[name]) + result.provenance.iloc[0, 0] = -9 + assert source["interest"].iloc[0, 0] == 901 + + +def test_named_unused_slot_derivation_keeps_published_niu_unknown(): + source = inputs(1) + i = source["interest"] + code(i, "RINT_YN", [1]) + code(i, "RINT_SC1", [4]) + amount(i, "RINT_VAL1", [20]) + amount(i, "RINT_VAL2", [0], ["unreported_account_slot"]) + amount(i, "INT_VAL", [120]) + result = build_asec_property_basis(**source) + row = result.person.iloc[0] + assert row.property_retirement_interest == 20 + assert row.joint_component_fit_eligible + assert row.retirement_structural_zero_slot_count == 1 + assert row.retirement_slot2_derived_unused_zero + assert ( + row.retirement_interest_derivation + == "declared_account_sum_with_unused_slot_zero" + ) + assert pd.isna(result.provenance.iloc[0]["interest.RINT_VAL2_amount"]) + assert result.provenance.iloc[0]["interest.RINT_VAL2_published_amount"] == 0 + + +@pytest.mark.parametrize( + "case", + [ + "zero_declared", + "missing_code", + "missing_amount", + "all_unused", + "niu_receipt", + "no_with_account", + ], +) +def test_unresolved_retirement_never_completes_unknown_slots(case): + source = inputs(1) + i = source["interest"] + code(i, "RINT_YN", [1]) + code(i, "RINT_SC1", [4]) + amount(i, "RINT_VAL1", [20]) + amount(i, "RINT_VAL2", [0], ["unreported_account_slot"]) + if case == "zero_declared": + code(i, "RINT_SC2", [4]) + amount(i, "RINT_VAL2", [0], ["ambiguous_recipient_zero"]) + elif case == "missing_code": + code(i, "RINT_SC2", [None], ["missing"]) + elif case == "missing_amount": + amount(i, "RINT_VAL2", [np.nan], ["missing_amount"]) + elif case == "all_unused": + code(i, "RINT_SC1", [0]) + amount(i, "RINT_VAL1", [0], ["unreported_account_slot"]) + elif case == "niu_receipt": + code(i, "RINT_YN", [0]) + else: + code(i, "RINT_YN", [2]) + row = build_asec_property_basis(**source).person.iloc[0] + assert np.isnan(row.property_retirement_interest) + assert row.reported_total_eligible + assert not row.joint_component_fit_eligible + assert row.retirement_structural_zero_slot_count == 0 + + +def test_two_active_accounts_sum_without_a_tax_label(): + source = inputs(1) + i = source["interest"] + code(i, "RINT_YN", [1]) + for n, value in ((1, 20), (2, 30)): + code(i, f"RINT_SC{n}", [n]) + amount(i, f"RINT_VAL{n}", [value]) + amount(i, "INT_VAL", [150]) + result = build_asec_property_basis(**source) + assert result.person.property_retirement_interest.iloc[0] == 50 + assert result.person.joint_component_fit_eligible.iloc[0] + assert ( + result.person.retirement_interest_derivation.iloc[0] == "declared_account_sum" + ) + + +def test_reported_aggregate_discrepancy_is_unprojected_and_excluded(): + source = inputs(1) + amount(source["interest"], "INT_VAL", [110]) + result = build_asec_property_basis(**source) + row = result.person.iloc[0] + assert row.property_reported_total == 0 + assert row.property_component_sum == -10 + assert row.interest_component_discrepancy == 10 + assert row.reported_minus_component_total == 10 + assert row.reported_total_eligible and not row.joint_component_fit_eligible + assert result.exclusions.interest_discrepancy_nonzero.iloc[0] + assert not any("cause" in column for column in result.person) + + +@pytest.mark.parametrize( + "receipt,category,status,route,clear,overlap", + [ + (1, 20, "known_receipt", "reported_category", True, False), + (1, 2, "ambiguous_recipient_zero", "reported_category", True, False), + (1, 5, "known_receipt", "reported_category", False, True), + (1, 8, "known_receipt", "reported_category", False, True), + (1, 19, "known_receipt", "reported_category", False, False), + (1, 0, "known_receipt", "receipt_without_category", False, False), + (2, 5, "known_nonreceipt", "category_without_receipt", False, False), + (0, 0, "niu", "niu_category", False, False), + ( + None, + None, + "missing_receipt_literal", + "missing_category_literal", + False, + False, + ), + ], +) +def test_other_income_routes_are_strict_and_never_added( + receipt, category, status, route, clear, overlap +): + source = inputs(1) + r = source["income_routing"] + code( + r, + "other_income_receipt", + [receipt], + ["missing" if receipt is None else "in_printed_range"], + ) + code( + r, + "other_income_category", + [category], + ["missing" if category is None else "in_printed_range"], + ) + r["other_income_reporting_status"] = status + r["other_income_routing_status"] = route + result = build_asec_property_basis(**source) + assert bool(result.person.reported_total_eligible.iloc[0]) == clear + assert bool(result.exclusions.other_income_possible_property.iloc[0]) == overlap + assert result.person.property_reported_total.iloc[0] == -10 + + +@pytest.mark.parametrize( + "receipt,first,second,declared", + [ + (1, 1, 0, True), + (1, 0, 9, True), + (1, 8, 0, False), + (1, 8, None, False), + (1, 10, 0, None), + (1, 1, None, None), + (1, 0, 0, None), + (2, 0, 0, True), + (2, 8, 0, None), + (0, 0, 0, None), + ], +) +def test_survivor_routes_keep_overlap_and_unknown_exclusions( + receipt, first, second, declared +): + source = inputs(1) + d = source["dividend"] + for name, value in (("SUR_YN", receipt), ("SUR_SC1", first), ("SUR_SC2", second)): + code(d, name, [value], ["missing" if value is None else "in_printed_range"]) + d["survivor_property_route_clear"] = pd.array([declared], dtype="boolean") + result = build_asec_property_basis(**source) + full_clear = receipt == 2 and declared is True + assert bool(result.person.reported_total_eligible.iloc[0]) is full_clear + assert bool(result.person.joint_component_fit_eligible.iloc[0]) is full_clear + assert bool(result.person.survivor_visible_routes_clear.iloc[0]) is ( + declared is True + ) + assert bool(result.exclusions.survivor_additional_sources_unresolved.iloc[0]) is ( + receipt == 1 + ) + assert bool(result.exclusions.survivor_possible_property.iloc[0]) is ( + declared is False + ) + + +def test_visible_survivor_clearance_does_not_clear_unobserved_extra_sources(): + source = inputs(3) + d = source["dividend"] + code(d, "SUR_YN", [1, 1, 2]) + code(d, "SUR_SC1", [1, 8, 0]) + d["survivor_property_route_clear"] = pd.array([True, False, True], dtype="boolean") + before = d.copy(deep=True) + result = build_asec_property_basis(**source) + assert result.person.survivor_visible_routes_clear.tolist() == [True, False, True] + assert result.person.survivor_full_scope_clear.tolist() == [False, False, True] + assert result.person.joint_component_fit_eligible.tolist() == [False, False, True] + assert result.person.property_reported_total.eq(-10).all() + reason = result.summary.loc["excluded:survivor_additional_sources_unresolved"] + assert reason.person_count == 2 and reason.household_count == 1 + assert reason.design_weighted_person_mass == 4 + assert reason.union_household_design_mass == 2 + assert result.exclusions.survivor_possible_property.tolist() == [False, True, False] + assert_frame_equal(source["dividend"], before) + assert_frame_equal( + result.provenance.filter(like="dividend."), before.add_prefix("dividend.") + ) + + +@pytest.mark.parametrize( + "axis", ["person", "native", "age", "duplicate_native", "duplicate_person"] +) +def test_both_identity_axes_and_age_must_match_exactly(axis): + source = inputs() + d = source["dividend"] + if axis == "person": + source["dividend"] = d.iloc[::-1] + elif axis == "native": + d["native_person_id"] = d.native_person_id.to_numpy()[::-1] + elif axis == "age": + d.iloc[0, d.columns.get_loc("source_age")] = 41 + elif axis == "duplicate_native": + d["native_person_id"] = 901 + else: + d.index = pd.Index([101, 101, 103], name="person_id") + with pytest.raises(ValueError): + build_asec_property_basis(**source) + + +def test_joint_permutation_and_household_weight_order_preserve_result(): + source = inputs() + original = build_asec_property_basis(**source) + for key in ( + "interest", + "income_routing", + "dividend", + "original_household_membership", + ): + source[key] = source[key].iloc[[2, 0, 1]] + source["original_household_design_weights"] = source[ + "original_household_design_weights" + ].iloc[::-1] + permuted = build_asec_property_basis(**source) + assert_frame_equal(original.person, permuted.person.loc[original.person.index]) + assert_frame_equal(original.summary, permuted.summary) + + +def test_weighted_exclusions_report_person_and_union_household_mass(): + source = inputs() + source["interest"].loc[101, "INT_VAL_amount"] = 110 + result = build_asec_property_basis(**source) + all_rows = result.summary.loc["all"] + excluded = result.summary.loc["excluded_joint_component_fit"] + assert all_rows.design_weighted_person_mass == 9 + assert all_rows.union_household_design_mass == 7 + assert excluded.person_count == 1 + assert excluded.design_weighted_person_mass == 2 + assert excluded.union_household_design_mass == 2 + + +@pytest.mark.parametrize( + "case", + [ + "extra_household", + "missing_household", + "duplicate_household", + "bad_person_order", + "negative_weight", + "infinite_weight", + ], +) +def test_only_exact_original_household_design_mapping_is_admitted(case): + source = inputs() + weights = source["original_household_design_weights"] + if case == "extra_household": + weights.loc[999] = 4 + elif case == "missing_household": + source["original_household_design_weights"] = weights.iloc[:1] + elif case == "duplicate_household": + weights.index = pd.Index([41, 41], name="household_id") + elif case == "bad_person_order": + source["original_household_membership"] = source[ + "original_household_membership" + ].iloc[::-1] + else: + weights.iloc[0] = -1 if case == "negative_weight" else np.inf + with pytest.raises(ValueError): + build_asec_property_basis(**source) + + +def test_empty_and_zero_weight_records_are_retained(): + empty = build_asec_property_basis(**inputs(0)) + assert len(empty.person) == 0 + assert empty.summary.person_count.eq(0).all() + source = inputs() + source["original_household_design_weights"].iloc[:] = 0 + result = build_asec_property_basis(**source) + assert len(result.person) == 3 and result.person.joint_component_fit_eligible.all() + assert result.summary.design_weighted_person_mass.eq(0).all() + + +def test_under15_unknowns_are_excluded_without_analytic_zero(): + source = inputs(1) + for key in ("interest", "income_routing", "dividend"): + source[key]["source_age"] = 14.0 + i = source["interest"] + for name in ("INT_VAL", "TRDINT_VAL", "RINT_VAL1", "RINT_VAL2"): + amount(i, name, [0], ["outside_reporting_universe"]) + code(i, "RINT_YN", [0]) + d = source["dividend"] + amount(d, "DIV_VAL", [0], ["outside_reporting_universe"]) + code(d, "DIV_YN", [0]) + code(d, "SUR_YN", [0]) + d["survivor_property_route_clear"] = pd.array([None], dtype="boolean") + r = source["income_routing"] + r["net_property_known_amount"] = np.nan + r["net_property_reporting_status"] = "outside_reporting_universe" + result = build_asec_property_basis(**source) + assert result.person[list(PROPERTY_COMPONENTS)].isna().all().all() + assert result.exclusions.under15.iloc[0] + assert not result.person.reported_total_eligible.iloc[0] + + +@pytest.mark.parametrize( + "case", + [ + "nonfinite", + "negative_interest", + "false_knownness", + "dividend_no_positive", + "rent_receipt_zero", + "survivor_false_clear", + "overflow", + ], +) +def test_invalid_descriptions_and_overflow_refuse(case): + source = inputs(1) + if case in ("nonfinite", "negative_interest"): + source["interest"].loc[101, "TRDINT_VAL_amount"] = ( + np.inf if case == "nonfinite" else -1 + ) + elif case == "false_knownness": + source["interest"]["INT_VAL_amount_known"] = False + elif case == "dividend_no_positive": + amount(source["dividend"], "DIV_VAL", [10]) + elif case == "rent_receipt_zero": + source["income_routing"]["net_property_known_amount"] = 0.0 + elif case == "survivor_false_clear": + source["dividend"]["survivor_property_route_clear"] = pd.array( + [False], dtype="boolean" + ) + else: + amount(source["interest"], "INT_VAL", [1.7e308]) + source["income_routing"]["net_property_known_amount"] = 1.7e308 + with pytest.raises((ValueError, FloatingPointError)): + build_asec_property_basis(**source) + + +def test_weight_summaries_are_stable_under_widely_different_weight_permutation(): + source = inputs() + source["original_household_membership"] = pd.Series( + [41, 42, 43], index=source["interest"].index, dtype="int64" + ) + source["original_household_design_weights"] = pd.Series( + [1e16, 1.0, 1.0], index=pd.Index([41, 42, 43], name="household_id") + ) + original = build_asec_property_basis(**source) + for key in ( + "interest", + "income_routing", + "dividend", + "original_household_membership", + ): + source[key] = source[key].iloc[::-1] + reordered = build_asec_property_basis(**source) + assert_frame_equal(original.summary, reordered.summary) + assert original.summary.loc["all", "design_weighted_person_mass"] == 1e16 + 2 + + +def test_fully_qualified_nonreceipt_zeros_remain_eligible(): + source = inputs(1) + for name in ("INT_VAL", "TRDINT_VAL"): + amount(source["interest"], name, [0]) + source["income_routing"]["net_property_known_amount"] = 0.0 + source["income_routing"]["net_property_reporting_status"] = "known_nonreceipt" + row = build_asec_property_basis(**source).person.iloc[0] + assert row[list(PROPERTY_COMPONENTS)].eq(0).all() + assert row.property_reported_total == 0 and row.joint_component_fit_eligible + + +@pytest.mark.parametrize("field", ["DIV_VAL", "RNT_VAL"]) +def test_receipt_yes_zero_stays_unknown_instead_of_eligible(field): + source = inputs(1) + if field == "DIV_VAL": + amount(source["dividend"], field, [0], ["ambiguous_recipient_zero"]) + code(source["dividend"], "DIV_YN", [1]) + else: + source["income_routing"]["net_property_known_amount"] = np.nan + source["income_routing"]["net_property_reporting_status"] = ( + "receipt_with_net_zero" + ) + result = build_asec_property_basis(**source) + assert np.isnan(result.person.property_reported_total.iloc[0]) + assert not result.person.reported_total_eligible.iloc[0] + assert not result.person.joint_component_fit_eligible.iloc[0] + + +def test_actual_pure_projector_column_composition_with_invented_literals(): + from test_us_current_asec_income_routing import _pure_rows, _set_amount + + from microcosm.build.us_runtime import current_asec_dividend_source as dividend + from microcosm.build.us_runtime import current_asec_income_routing_source as routing + from microcosm.build.us_runtime import current_asec_interest_source as interest + + source = inputs(1) + raw_interest = {name: "0" for name in interest.READ_COLUMNS} + raw_interest.update( + A_AGE="40", + INT_YN="1", + INT_VAL="120", + TRDINT_VAL="100", + RINT_YN="1", + RINT_SC1="4", + RINT_VAL1="20", + ) + raw_dividend = {name: "0" for name in dividend.READ_COLUMNS} + raw_dividend.update(A_AGE="40", DIV_YN="2", SUR_YN="2") + raw_routing, ages = _pure_rows(1, [40], RNT_YN=["1"], OI_YN=["2"]) + _set_amount(raw_routing, "RNT_VAL", [-120]) + projected = { + "interest": interest.project_interest_literals(pd.DataFrame([raw_interest])), + "income_routing": routing.project_income_routing(raw_routing, ages), + "dividend": dividend.project_dividend_literals(pd.DataFrame([raw_dividend])), + } + for name, frame in projected.items(): + frame.index = source[name].index + frame["native_person_id"] = source[name].native_person_id + source[name] = frame + result = build_asec_property_basis(**source) + row = result.person.iloc[0] + assert row.property_reported_total == 0 + assert row[list(PROPERTY_COMPONENTS)].tolist() == [100, 20, 0, -120] + assert row.joint_component_fit_eligible + assert row.retirement_slot2_derived_unused_zero + assert ( + result.provenance.iloc[0]["interest.RINT_VAL2_reporting_status"] + == "unreported_account_slot" + ) diff --git a/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py b/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py new file mode 100644 index 000000000..b98fef380 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py @@ -0,0 +1,693 @@ +"""Pure retirement candidate support over actual projectors and invented literals.""" + +import dataclasses +import importlib +import json + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_income_routing_source as routing +from microcosm.build.us_runtime import current_asec_retirement_detail_source as detail + + +@pytest.fixture +def owner(): + return importlib.import_module( + "microcosm.build.us_runtime.current_asec_retirement_basis" + ) + + +def options(owner, **changes): + return owner.RetirementCandidateAssumptions( + **{ + "pension_annuity_regularity": "unresolved", + "disability_pension_eligibility": "unresolved", + "survivor_annuity_overlap": "unresolved", + "withdrawal_regularity_netting": "unresolved", + "aggregate_accounting": "exact_visible_balance_only", + **changes, + } + ) + + +def inputs(owner, changes=({},), **assumptions): + rows = [] + for i, change in enumerate(changes): + row = { + **{n: "0" for n in detail.READ_COLUMNS}, + **{n: "0" for n in routing.RECEIPT_ENTRIES}, + **{n: "0" for n in routing.ACCOUNT_ENTRIES}, + **{n: "0" for n in routing.AMOUNT_FIELDS}, + "PERIDNUM": str(i + 1).zfill(22), + "PH_SEQ": str(i // 2 + 1), + "A_LINENO": str(i % 2 + 1), + "A_AGE": "40", + "OI_OFF": "0", + "PEN_YN": "2", + "DIS_YN": "2", + "SUR_YN": "2", + "ANN_YN": "2", + "DST_YN_YNG": "2", + "OI_YN": "2", + **change, + } + rows.append(row) + raw = pd.DataFrame(rows) + ages = np.array([int(r["A_AGE"]) for r in rows], dtype="int64") + values, statuses = {}, {} + for n in routing.AMOUNT_FIELDS: + data, labels = [], [] + for r in rows: + token = r[n] + value = np.nan if token == "" else float(token) + code = ( + routing.money.CodebookStatus.MISSING_NULL + if token == "" + else routing.money.CodebookStatus.DECLARED_NIU + if n == "ANN_VAL" and value == -1 + else routing.money.CodebookStatus.ZERO_NONE_OR_NIU + if value == 0 + else routing.money.CodebookStatus.AMOUNT_NONZERO + ) + data.append( + 0.0 if code == routing.money.CodebookStatus.DECLARED_NIU else value + ) + labels.append(int(code)) + values[n], statuses[n] = ( + np.array(data, dtype="float64"), + np.array(labels, dtype="u1"), + ) + arrays = { + "amounts": values, + "statuses": statuses, + "allocations": { + n: ([0] * len(rows), ["in_printed_range"] * len(rows)) + for n in routing.ALLOCATION_ENTRIES + }, + **{ + n: [r[n] for r in rows] + for n in (*routing.RECEIPT_ENTRIES, *routing.ACCOUNT_ENTRIES, "OI_OFF") + }, + } + projected = { + "retirement_detail": detail.project_retirement_detail_literals(raw), + "income_routing": routing.project_income_routing(arrays, ages), + } + index = pd.Index(np.arange(len(rows), dtype="int64") + 2**53 + 19, name="person_id") + for table in projected.values(): + table.index = index + table["native_person_id"] = np.arange(len(rows), dtype="int64") + 17 + membership = pd.Series(np.arange(len(rows), dtype="int64") // 2 + 101, index=index) + households = pd.Index(membership.unique(), dtype="int64", name="household_id") + return { + **projected, + "original_household_membership": membership, + "original_household_design_weights": pd.Series( + np.arange(len(households)) + 2.0, index=households + ), + "assumptions": options(owner, **assumptions), + } + + +def build(owner, changes=({},), **assumptions): + return owner.build_asec_retirement_basis(**inputs(owner, changes, **assumptions)) + + +def test_named_options_are_required_canonical_and_never_source_facts(owner): + value = options(owner) + assert json.loads(value.to_bytes())["pension_annuity_regularity"] == "unresolved" + assert ( + dataclasses.replace( + value, pension_annuity_regularity="assume_regular" + ).to_bytes() + != value.to_bytes() + ) + assert ( + dataclasses.replace( + value, disability_pension_eligibility="assume_qualifying" + ).to_bytes() + != value.to_bytes() + ) + with pytest.raises(TypeError): + owner.RetirementCandidateAssumptions() + for field, bad in ( + ("pension_annuity_regularity", True), + ("aggregate_accounting", "clip"), + ("withdrawal_regularity_netting", "all_regular"), + ): + with pytest.raises(ValueError): + options(owner, **{field: bad}) + + +@pytest.mark.parametrize( + "code,route", + [(1, "candidate"), (6, "candidate"), (7, "railroad"), (8, "unresolved")], +) +def test_balanced_pension_routes_and_named_regularity(owner, code, route): + row = {"PEN_YN": "1", "PEN_SC1": str(code), "PEN_VAL1": "100", "PNSN_VAL": "100"} + result = build(owner, [row]) + selected = result.slots.query("family == 'pension' and slot == 1").iloc[0] + assert selected.route == route and selected.source_known_amount == 100 + p = result.person.iloc[0] + assert p.pension_candidate_lower == 0 + assert p.pension_candidate_upper == (0 if code == 7 else 100) + assumed = build( + owner, [row], pension_annuity_regularity="assume_regular" + ).person.iloc[0] + assert assumed.pension_candidate_lower == (100 if code in (1, 6) else 0) + assert not p.point_identified and not p.fiscal_outputs_produced + assert ( + result.slots.query("family == 'pension' and slot == 2") + .iloc[0] + .structural_zero_comparison + ) + + +@pytest.mark.parametrize( + "total,status", + [(150, "additional_scope_unresolved"), (80, "contradictory_accounting")], +) +def test_unbalanced_pension_never_clips_or_allocates_residual(owner, total, status): + p = build( + owner, + [{"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": str(total)}], + ).person.iloc[0] + assert p.pension_accounting_difference == total - 100 + assert p.pension_status == status + assert np.isnan(p.pension_candidate_upper) + assert p.pension_observed_candidate_subtotal == 100 + assert not p.candidate_basis_eligible + + +@pytest.mark.parametrize("code", [1, 5, 8, 9, 10]) +def test_positive_survivor_never_claims_hidden_source_clearance(owner, code): + result = build( + owner, + [ + { + "SUR_YN": "1", + "SUR_SC1": str(code), + "SUR_VAL1": "100", + "SRVS_VAL": "100", + "ANN_YN": "1", + "ANN_VAL": "40", + } + ], + ) + p = result.person.iloc[0] + assert p.survivor_accounting_difference == 0 + assert p.survivor_status == "additional_survivor_scope_unresolved" + assert np.isnan(p.survivor_candidate_upper) and not p.candidate_basis_eligible + assert bool(result.exclusions.survivor_annuity_overlap.iloc[0]) == (code == 9) + assert not p.point_identified + + +@pytest.mark.parametrize( + "code,route", + [ + (2, "candidate"), + (5, "candidate"), + (6, "railroad"), + (9, "other_compensation"), + (10, "unresolved"), + ], +) +def test_disability_does_not_infer_eligibility_from_health_answers(owner, code, route): + row = { + "DIS_YN": "1", + "DIS_SC1": str(code), + "DIS_VAL1": "100", + "DSAB_VAL": "100", + "DIS_HP": "1", + "DIS_CS": "1", + } + result = build(owner, [row]) + p = result.person.iloc[0] + assert ( + result.slots.query("family == 'disability' and slot == 1").iloc[0].route + == route + ) + assert p.disability_candidate_lower == 0 + assert p.disability_candidate_upper == ( + 100 if route in ("candidate", "unresolved") else 0 + ) + changed = build(owner, [{**row, "DIS_HP": "2", "DIS_CS": "2"}]) + pd.testing.assert_frame_equal(result.person, changed.person, check_exact=True) + assumed = build( + owner, [row], disability_pension_eligibility="assume_qualifying" + ).person.iloc[0] + assert assumed.disability_candidate_lower == (100 if route == "candidate" else 0) + + +@pytest.mark.parametrize( + "amount,receipt,known", + [("-1", "0", False), ("0", "1", True), ("0", "2", True), ("40", "1", True)], +) +def test_annuity_niu_zero_and_regular_scenario_are_distinct( + owner, amount, receipt, known +): + source = inputs(owner, [{"ANN_YN": receipt, "ANN_VAL": amount}]) + p = owner.build_asec_retirement_basis(**source).person.iloc[0] + assert np.isfinite(p.annuity_candidate_upper) == known + if known: + assert p.annuity_candidate_lower == 0 and p.annuity_candidate_upper == float( + amount + ) + source["assumptions"] = options( + owner, pension_annuity_regularity="assume_regular" + ) + assert owner.build_asec_retirement_basis(**source).person.iloc[ + 0 + ].annuity_candidate_lower == float(amount) + + +@pytest.mark.parametrize("age", [57, 58]) +def test_distribution_account_is_not_frequency_or_taxability(owner, age): + young = age < 58 + suffix = "_YNG" if young else "" + row = { + "A_AGE": str(age), + "DST_YN_YNG": "1" if young else "0", + "DST_YN": "0" if young else "1", + "DST_SC1" + suffix: "4", + "DST_VAL1" + suffix: "100", + "DBTN_VAL": "0" if young else "100", + } + p = build(owner, [row]).person.iloc[0] + assert p.distribution_candidate_lower == 0 and p.distribution_candidate_upper == 100 + assert p.distribution_account_4_amount == 100 + assert not p.point_identified and not p.fiscal_outputs_produced + + +@pytest.mark.parametrize( + "change", + [ + {"DST_SC1_YNG": ""}, + {"DST_SC1_YNG": "9"}, + {"DST_VAL1": "1"}, + {"DST_YN": "2"}, + {"DST_SC2_YNG": "3"}, + ], +) +def test_distribution_unreadable_or_offroute_composition_stays_unknown(owner, change): + row = {"DST_YN_YNG": "1", "DST_SC1_YNG": "4", "DST_VAL1_YNG": "100", **change} + p = build(owner, [row]).person.iloc[0] + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_basis_eligible + + +@pytest.mark.parametrize( + "code,clear", + [(2, False), (13, False), (19, False), (1, True), (8, True), (20, True)], +) +def test_other_income_not_silently_added(owner, code, clear): + p = build( + owner, [{"OI_YN": "1", "OI_OFF": str(code), "OI_VAL": "100"}] + ).person.iloc[0] + assert bool(p.other_income_scope_clear) is clear + assert p.other_income_reported_amount == 100 + assert ( + (p.retirement_candidate_upper == 0) + if clear + else np.isnan(p.retirement_candidate_upper) + ) + + +@pytest.mark.parametrize("age", [0, 14]) +def test_under15_retains_evidence_without_analytic_zero(owner, age): + result = build( + owner, + [ + { + "A_AGE": str(age), + "PEN_YN": "0", + "ANN_YN": "0", + "DIS_YN": "0", + "SUR_YN": "0", + "DST_YN_YNG": "0", + "OI_YN": "0", + } + ], + ) + p = result.person.iloc[0] + assert all( + np.isnan(p[n]) + for n in p.index + if n.endswith(("_candidate_lower", "_candidate_upper")) + ) + assert result.exclusions.under15.iloc[0] and not p.candidate_basis_eligible + + +@pytest.mark.parametrize( + "change", [{"PEN_YN": ""}, {"PEN_SC1": ""}, {"PEN_VAL1": ""}, {"PEN_VAL1": "0"}] +) +def test_source_unknowns_and_ambiguous_zeros_remain_visible(owner, change): + row = { + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + **change, + } + p = build(owner, [row]).person.iloc[0] + assert np.isnan(p.pension_candidate_upper) and not p.candidate_basis_eligible + + +@pytest.mark.parametrize( + "defect", + [ + "person_order", + "native_id", + "age", + "shared_amount", + "known_mask", + "comparison", + "bad_weight", + "membership", + ], +) +def test_incompatible_descriptions_refuse(owner, defect): + source = inputs(owner, [{}, {}]) + d, r = source["retirement_detail"], source["income_routing"] + if defect == "person_order": + source["income_routing"] = r.iloc[::-1] + elif defect == "native_id": + r.iloc[0, r.columns.get_loc("native_person_id")] += 100 + elif defect == "age": + r.iloc[0, r.columns.get_loc("source_age")] += 1 + elif defect == "shared_amount": + r.iloc[0, r.columns.get_loc("pension_annuity_pension_source_total")] += 1 + elif defect == "known_mask": + d.iloc[0, d.columns.get_loc("PEN_VAL1_amount_known")] = False + elif defect == "comparison": + d.iloc[0, d.columns.get_loc("pension_total_minus_visible_slots")] = 1 + elif defect == "bad_weight": + source["original_household_design_weights"].iloc[0] = np.nan + else: + source["original_household_membership"] = source[ + "original_household_membership" + ].iloc[::-1] + with pytest.raises(ValueError): + owner.build_asec_retirement_basis(**source) + + +def test_exact_permutation_detachment_and_design_mass(owner): + source = inputs( + owner, + [{}, {"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": "80"}, {}], + ) + before = { + k: v.copy(deep=True) + for k, v in source.items() + if isinstance(v, (pd.DataFrame, pd.Series)) + } + result = owner.build_asec_retirement_basis(**source) + assert result.summary.loc["all", "design_weighted_person_mass"] == 7 + assert result.summary.loc["all", "union_household_design_mass"] == 5 + assert ( + result.summary.loc["candidate_basis_eligible", "design_weighted_person_mass"] + == 5 + ) + assert ( + result.summary.loc["candidate_basis_eligible", "union_household_design_mass"] + == 5 + ) + for name in ( + "retirement_detail", + "income_routing", + "original_household_membership", + ): + source[name] = source[name].iloc[::-1] + permuted = owner.build_asec_retirement_basis(**source) + pd.testing.assert_frame_equal( + permuted.person.loc[result.person.index], result.person, check_exact=True + ) + pd.testing.assert_frame_equal(permuted.summary, result.summary, check_exact=True) + result.provenance.iloc[0, 0] = "detached" + for name, original in before.items(): + current = source[name].loc[original.index] + if isinstance(original, pd.DataFrame): + pd.testing.assert_frame_equal(current, original, check_exact=True) + else: + pd.testing.assert_series_equal(current, original, check_exact=True) + + +def test_unknown_account_total_is_preserved_without_candidate_admission(owner): + source = inputs( + owner, [{"DST_YN_YNG": "1", "DST_SC1_YNG": "", "DST_VAL1_YNG": "100"}] + ) + assert source["income_routing"].retirement_distribution_known_amount.iloc[0] == 100 + p = owner.build_asec_retirement_basis(**source).person.iloc[0] + assert p.distribution_source_known_amount == 100 + assert np.isnan(p.distribution_candidate_upper) + assert not p.distribution_account_composition_known + + +def test_allocation_and_disclosure_flags_are_provenance_not_filters(owner): + row = {"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": "100"} + before = build(owner, [row]) + changed = build(owner, [{**row, "I_PENVAL1": "4", "TPEN_VAL1": "1"}]) + pd.testing.assert_frame_equal(before.person, changed.person, check_exact=True) + pd.testing.assert_frame_equal(before.summary, changed.summary, check_exact=True) + assert changed.provenance["retirement_detail.I_PENVAL1_code"].iloc[0] == 4 + + +def test_outside_universe_unreadable_and_contradictory_evidence_stay_distinct(owner): + unresolved = build( + owner, [{"A_AGE": "14", "PEN_YN": "0", "PEN_VAL1": ""}] + ).person.iloc[0] + contradictory = build( + owner, + [ + { + "A_AGE": "14", + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + } + ], + ).person.iloc[0] + assert unresolved.pension_status == "unresolved_outside_reporting_universe" + assert contradictory.pension_status == "contradictory_outside_reporting_universe" + assert np.isnan(unresolved.pension_candidate_upper) and np.isnan( + contradictory.pension_candidate_upper + ) + + +@pytest.mark.parametrize("total", [150, 50]) +def test_distribution_main_aggregate_difference_blocks_candidate_interval(owner, total): + result = build( + owner, + [ + { + "A_AGE": "60", + "DST_YN_YNG": "0", + "DST_YN": "1", + "DST_SC1": "4", + "DST_VAL1": "100", + "DBTN_VAL": str(total), + } + ], + ) + p = result.person.iloc[0] + assert p.distribution_accounting_difference == total - 100 + assert p.distribution_source_known_amount == 100 + assert p.distribution_account_4_amount == 100 + assert ( + result.provenance["retirement_detail.DBTN_VAL_published_amount"].iloc[0] + == total + ) + assert np.isnan(p.distribution_candidate_lower) + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_interval_available and not p.candidate_basis_eligible + assert p.distribution_status == ( + "additional_scope_unresolved" if total > 100 else "contradictory_accounting" + ) + + +@pytest.mark.parametrize("token", ["", "malformed"]) +@pytest.mark.parametrize( + "age,field,retained", + [ + (40, "DST_YN", "receipt_58"), + (40, "DST_SC1", "slot1_account"), + (40, "DST_SC2", "slot2_account"), + (60, "DST_YN_YNG", "receipt_young"), + (60, "DST_SC1_YNG", "slot1_young_account"), + (60, "DST_SC2_YNG", "slot2_young_account"), + ], +) +def test_unreadable_offroute_distribution_literals_prevent_admission( + owner, age, field, retained, token +): + young = age < 58 + suffix = "_YNG" if young else "" + result = build( + owner, + [ + { + "A_AGE": str(age), + "DST_YN_YNG": "1" if young else "0", + "DST_YN": "0" if young else "1", + "DST_SC1" + suffix: "4", + "DST_VAL1" + suffix: "100", + "DBTN_VAL": "0" if young else "100", + field: token, + } + ], + ) + p = result.person.iloc[0] + assert ( + result.provenance[ + "income_routing.retirement_distribution_" + retained + "_literal" + ].iloc[0] + == token + ) + assert not p.distribution_account_composition_known + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_basis_eligible + + +@pytest.mark.parametrize("age,field", [(40, "DST_VAL1"), (60, "DST_VAL1_YNG")]) +def test_unreadable_offroute_distribution_amount_is_not_assumed_zero(owner, age, field): + young = age < 58 + suffix = "_YNG" if young else "" + p = build( + owner, + [ + { + "A_AGE": str(age), + "DST_YN_YNG": "1" if young else "0", + "DST_YN": "0" if young else "1", + "DST_SC1" + suffix: "4", + "DST_VAL1" + suffix: "100", + "DBTN_VAL": "0" if young else "100", + field: "", + } + ], + ).person.iloc[0] + assert not p.distribution_account_composition_known + assert np.isnan(p.distribution_candidate_upper) + + +def test_account_subtotals_are_evidence_when_receipt_interval_is_unknown(owner): + p = build( + owner, [{"DST_YN_YNG": "", "DST_SC1_YNG": "4", "DST_VAL1_YNG": "100"}] + ).person.iloc[0] + assert p.distribution_account_composition_known + assert p.distribution_account_4_amount == 100 + assert np.isnan(p.distribution_source_known_amount) + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_interval_available + + +@pytest.mark.parametrize( + "family,prefix,total,code1,code2,lower,upper,status", + [ + ("pension", "PEN", "PNSN_VAL", 1, 8, 60, 100, "candidate_under_assumptions"), + ("pension", "PEN", "PNSN_VAL", 1, 7, 60, 60, "candidate_under_assumptions"), + ( + "disability", + "DIS", + "DSAB_VAL", + 2, + 10, + 60, + 100, + "candidate_under_assumptions", + ), + ("disability", "DIS", "DSAB_VAL", 2, 1, 60, 60, "candidate_under_assumptions"), + ("disability", "DIS", "DSAB_VAL", 6, 1, 0, 0, "route_outside_only"), + ], +) +def test_two_slot_family_routes_preserve_candidate_and_unresolved_bounds( + owner, family, prefix, total, code1, code2, lower, upper, status +): + p = build( + owner, + [ + { + prefix + "_YN": "1", + prefix + "_SC1": str(code1), + prefix + "_SC2": str(code2), + prefix + "_VAL1": "60", + prefix + "_VAL2": "40", + total: "100", + } + ], + pension_annuity_regularity="assume_regular", + disability_pension_eligibility="assume_qualifying", + ).person.iloc[0] + assert p[family + "_candidate_lower"] == lower + assert p[family + "_candidate_upper"] == upper + assert p[family + "_status"] == status + assert p[family + "_accounting_difference"] == 0 + assert ( + p[family + "_observed_candidate_subtotal"] + + p[family + "_observed_unresolved_subtotal"] + == upper + ) + + +@pytest.mark.parametrize( + "family,total", + [("pension", "PNSN_VAL"), ("disability", "DSAB_VAL"), ("survivor", "SRVS_VAL")], +) +def test_no_receipt_with_nonzero_total_and_zero_slots_remains_contradictory( + owner, family, total +): + p = build(owner, [{total: "100"}]).person.iloc[0] + assert p[family + "_status"] == "contradictory_accounting" + assert p[family + "_accounting_difference"] == 100 + assert p[family + "_observed_candidate_subtotal"] == 0 + assert np.isnan(p[family + "_candidate_upper"]) + assert not p.candidate_basis_eligible + + +def test_equal_bounds_under_explicit_scenarios_do_not_claim_identification(owner): + row = { + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + "DIS_YN": "1", + "DIS_SC1": "2", + "DIS_VAL1": "40", + "DSAB_VAL": "40", + "ANN_YN": "1", + "ANN_VAL": "20", + } + unresolved = build(owner, [row]).person.iloc[0] + p = build( + owner, + [row], + pension_annuity_regularity="assume_regular", + disability_pension_eligibility="assume_qualifying", + ).person.iloc[0] + assert ( + unresolved.retirement_candidate_lower == 0 + and unresolved.retirement_candidate_upper == 160 + ) + assert not unresolved.candidate_point_under_assumptions + assert p.retirement_candidate_lower == p.retirement_candidate_upper == 160 + assert p.candidate_point_under_assumptions and p.candidate_basis_eligible + assert not p.point_identified and not p.fiscal_outputs_produced + + +def test_diagnostic_household_mass_deduplicates_only_selected_households(owner): + row = {"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": "150"} + result = build(owner, [row, row, {}, row, {}]) + selected = result.summary.loc["diagnostic:pension_candidate_unresolved"] + assert selected.person_count == 3 and selected.household_count == 2 + assert selected.design_weighted_person_mass == 7 + assert selected.union_household_design_mass == 5 + + +def test_design_weights_refuse_an_unselected_household(owner): + source = inputs(owner, [{}]) + source["original_household_design_weights"].loc[999] = 4.0 + with pytest.raises(ValueError, match="DESIGN_MEMBERSHIP"): + owner.build_asec_retirement_basis(**source) diff --git a/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py b/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py new file mode 100644 index 000000000..21f6fe1b5 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py @@ -0,0 +1,459 @@ +"""Retirement detail source observations; no tax or regularity model.""" + +import copy +import hashlib +import importlib +import shutil +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + + +@pytest.fixture +def owner(): + return importlib.import_module( + "microcosm.build.us_runtime.current_asec_retirement_detail_source" + ) + + +def row(owner, **changes): + return { + **{name: "0" for name in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + "DIS_YN": "1", + "DIS_SC1": "6", + "DIS_VAL1": "200", + "DSAB_VAL": "200", + "SUR_YN": "1", + "SUR_SC1": "8", + "SUR_VAL1": "300", + "SRVS_VAL": "450", + **changes, + } + + +def project(owner, **changes): + return owner.project_retirement_detail_literals( + pd.DataFrame([row(owner, **changes)]) + ).iloc[0] + + +def test_source_components_and_extra_survivor_total_remain_distinct(owner): + value = project(owner) + assert value.PEN_VAL1_amount == 100 + assert value.DIS_VAL1_amount == 200 + assert value.SUR_VAL1_amount == 300 + assert value.SRVS_VAL_published_amount == 450 + assert value.survivor_total_minus_visible_slots == 150 + assert not value.survivor_visible_slots_exhaustive + assert not value.taxability_assigned + assert not value.regularity_assigned + assert not value.acs_retirement_component_assigned + + +@pytest.mark.parametrize( + "receipt,code,amount,status,known", + [ + ("1", "1", "100", "known_receipt", True), + ("1", "1", "0", "ambiguous_recipient_zero", False), + ("1", "0", "0", "unreported_source_slot", False), + ("1", "0", "100", "contradictory_source_slot", False), + ("2", "0", "0", "known_nonreceipt", True), + ("2", "1", "0", "contradictory_source_slot", False), + ("2", "0", "100", "contradictory_no_nonzero", False), + ("0", "0", "0", "niu", False), + ("", "1", "100", "missing_receipt_literal", False), + ("1", "", "100", "unresolved_source_slot", False), + ], +) +@pytest.mark.parametrize("family", ["PEN", "DIS", "SUR"]) +def test_slot_knownness_requires_receipt_and_source( + owner, family, receipt, code, amount, status, known +): + value = project( + owner, + **{family + "_YN": receipt, family + "_SC1": code, family + "_VAL1": amount}, + ) + assert value[family + "_VAL1_reporting_status"] == status + assert bool(value[family + "_VAL1_amount_known"]) is known + + +@pytest.mark.parametrize("age", ["0", "14"]) +def test_under15_has_no_analytical_zero(owner, age): + value = project(owner, A_AGE=age, PEN_YN="0", PEN_SC1="0", PEN_VAL1="0") + assert value.PEN_VAL1_reporting_status == "outside_reporting_universe" + assert not value.PEN_VAL1_amount_known + + +@pytest.mark.parametrize( + "token,status", + [ + ("", "missing"), + ("-1", "malformed"), + ("1.2", "malformed"), + ("1000000", "malformed"), + ("NaN", "malformed"), + ], +) +def test_new_amount_literal_domains_are_strict(owner, token, status): + value = project(owner, SUR_VAL1=token) + assert value.SUR_VAL1_literal == token + assert value.SUR_VAL1_literal_status == status + assert not value.SUR_VAL1_amount_known + + +def test_public_amount_mapping_is_detached_and_values_immutable(owner): + entries = owner.amount_entries() + expected = entries["SUR_VAL1"] + entries["SUR_VAL1"] = entries["PEN_VAL1"] + assert owner.amount_entries()["SUR_VAL1"] == expected + + +def test_reference_ira_annuity_and_aggregate_do_not_acquire_new_meanings(owner): + value = project(owner, ANN_VAL="-1", DST_VAL1="100", DST_VAL2="50", DBTN_VAL="120") + assert value.ANN_VAL_published_amount == -1 + assert value.distribution_total_minus_main_slots == -30 + assert not value.regularity_assigned and not value.taxability_assigned + + +def test_full_physical_seal_covers_masked_backing_and_metadata(owner): + raw = pd.DataFrame([row(owner, SUR_VAL1="0")]) + result = owner.CurrentAsecRetirementDetailValues( + owner.project_retirement_detail_literals(raw), raw, {} + ) + seal = owner.retirement_detail_values_seal(result) + assert owner.retirement_detail_values_seal(copy.deepcopy(result)) == seal + for kind in ("masked_backing", "mask", "literal", "evidence"): + changed = copy.deepcopy(result) + if kind == "masked_backing": + changed.person.SUR_VAL1_amount.array._data[0] = 17.0 + elif kind == "mask": + changed.person.SUR_VAL1_amount.array._mask[0] = False + elif kind == "literal": + changed.asec_literals.loc[0, "SUR_VAL1"] = "1" + else: + changed.evidence["invented"] = True + assert owner.retirement_detail_values_seal(changed) != seal + + +@pytest.mark.parametrize("field", ["PEN_YN", "PEN_SC1", "PEN_VAL1"]) +def test_unreadable_under15_values_are_unresolved_not_contradictions(owner, field): + value = project(owner, **{"A_AGE": "14", field: ""}) + assert value.PEN_VAL1_reporting_status == "unresolved_outside_reporting_universe" + assert not value.PEN_VAL1_amount_known + + +def test_source_allocation_domains_do_not_inherit_full_header_range(owner): + value = project(owner, I_PENSC1="4", I_PENVAL1="4", TPEN_VAL1="1", I_SURVL2="") + assert value.I_PENSC1_literal_status == "outside_printed_range" + assert value.I_PENVAL1_literal_status == "in_printed_range" + assert value.I_SURVL2_literal_status == "missing" + assert value.PEN_VAL1_amount == 100 + assert value.TPEN_VAL1_code == 1 + assert ( + next(e for e in owner.ALLOCATION_ENTRIES if e[0] == "I_SURVL2")[4] + == "SURV_VAL2 > 0" + ) + + +@pytest.mark.parametrize("change", ["bound", "duplicate", "missing", "new_retained"]) +def test_live_money_roster_is_exact_for_existing_fields_only(owner, change): + entries = owner.amount_entries() + fields = [ + SimpleNamespace( + name=name, + entity="person", + grain="person", + column=name, + minimum=entries[name].encoded_minimum, + maximum=entries[name].encoded_maximum, + zero_semantics=entries[name].zero_semantics, + ) + for name in owner.RETAINED_MONEY_FIELDS + ] + owner._domain_agreement( + SimpleNamespace(bindings=SimpleNamespace(spec=SimpleNamespace(fields=fields))) + ) + if change == "bound": + fields[0].maximum += 1 + elif change == "duplicate": + fields.append(fields[0]) + elif change == "missing": + fields.pop() + else: + fields.append(SimpleNamespace(name="SRVS_VAL")) + with pytest.raises(ValueError): + owner._domain_agreement( + SimpleNamespace( + bindings=SimpleNamespace(spec=SimpleNamespace(fields=fields)) + ) + ) + + +def retirement_arguments(owner, tmp_path, monkeypatch, *, annuity_niu=False): + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + zeros = { + name: "0" + for name in ( + *owner.AMOUNT_FIELDS, + *owner.RECEIPT_ENTRIES, + *owner.SOURCE_ENTRIES, + ) + } + literals = { + 105: row(owner, I_PENVAL1="4", ANN_VAL="-1" if annuity_niu else "0"), + 106: row(owner, **zeros, A_AGE="14"), + 107: row( + owner, + **{**zeros, "A_AGE": "70", "PEN_YN": "2", "DIS_YN": "2", "SUR_YN": "2"}, + ), + 108: row(owner, PEN_VAL1="0", SUR_VAL1="0"), + } + changes = {} + for name in (*owner.RETAINED_MONEY_FIELDS, "A_AGE"): + values = people[name].to_numpy(copy=True) + for person_id, literal in literals.items(): + values[ids == person_id] = int(literal[name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in (*owner.RETAINED_MONEY_FIELDS, "A_AGE"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + for name in ( + set(owner.READ_COLUMNS) + - set(owner.routing.COORDINATE_COLUMNS) + - set(owner.RETAINED_MONEY_FIELDS) + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row(owner))[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "retirement-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(owner, tmp_path, monkeypatch): + return owner.routing.source.prepare_authenticated_survey_population( + **retirement_arguments(owner, tmp_path, monkeypatch) + ) + + +def test_actual_annuity_niu_preserves_literal_and_normalized_owner( + owner, tmp_path, monkeypatch +): + parent = owner.routing.source.prepare_authenticated_survey_population( + **retirement_arguments(owner, tmp_path, monkeypatch, annuity_niu=True) + ) + result = owner.qualify_current_asec_retirement_detail(parent) + values = result.person.set_index("native_person_id") + money = owner.routing.money + assert values.loc[105, "ANN_VAL_literal"] == "-1" + assert values.loc[105, "ANN_VAL_published_amount"] == -1 + assert values.loc[105, "ANN_VAL_parent_validity"] == 1 + assert ( + values.loc[105, "ANN_VAL_parent_statuses"] == money.CodebookStatus.DECLARED_NIU + ) + assert values.loc[105, "ANN_VAL_parent_zero_origin"] == money.ZeroOrigin.NOT_ZERO + assert values.loc[107, "ANN_VAL_literal"] == "0" + assert ( + values.loc[107, "ANN_VAL_parent_statuses"] != money.CodebookStatus.DECLARED_NIU + ) + literals = result.asec_literals + assert literals.loc[literals.ANN_VAL.eq("-1")].shape[0] == 1 + + capture = owner._capture_member + + def substitute_dollar_zero(*args): + raw = capture(*args) + raw.loc[raw.ANN_VAL.eq("-1"), "ANN_VAL"] = "0" + return raw + + monkeypatch.setattr(owner, "_capture_member", substitute_dollar_zero) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + + +@pytest.mark.parametrize("literal,niu_status", [("-1", False), ("0", True)]) +def test_annuity_literal_and_owner_niu_status_must_agree(owner, literal, niu_status): + money = owner.routing.money + status = ( + money.CodebookStatus.DECLARED_NIU + if niu_status + else money.CodebookStatus.ZERO_NONE_OR_NIU + ) + field = money.MoneyField( + "ANN_VAL", + np.array([0.0], dtype=np.float64).tobytes(), + np.array([status], dtype="u1").tobytes(), + np.array([1], dtype="u1").tobytes(), + np.array([0], dtype="u1").tobytes(), + ) + ready = SimpleNamespace(field=lambda name: field) + with pytest.raises(ValueError): + owner._compare_amount( + ready, np.array([0], dtype=np.int64), "ANN_VAL", [literal] + ) + + +def test_actual_owner_qualifies_new_literals_without_fabricating_money_fields( + owner, tmp_path, monkeypatch +): + parent = prepared(owner, tmp_path, monkeypatch) + compare, compared = owner._compare_amount, [] + + def seen(ready, positions, name, literals): + compared.append(name) + assert name not in owner.INDEPENDENT_LITERAL_FIELDS + return compare(ready, positions, name, literals) + + monkeypatch.setattr(owner, "_compare_amount", seen) + result = owner.qualify_current_asec_retirement_detail(parent) + assert tuple(compared) == owner.RETAINED_MONEY_FIELDS + values = result.person.set_index("native_person_id") + assert values.loc[105, "PEN_VAL1_amount"] == 100 + assert values.loc[105, "survivor_total_minus_visible_slots"] == 150 + assert values.loc[107, "SUR_VAL1_amount"] == 0 + assert not values.loc[106, "SUR_VAL1_amount_known"] + assert not values.loc[108, "SUR_VAL1_amount_known"] + assert not result.evidence["source_admission_issued"] + assert not result.evidence["taxability_assigned"] + assert owner.retirement_detail_values_seal( + owner.qualify_current_asec_retirement_detail(parent) + ) == owner.retirement_detail_values_seal(result) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(copy.copy(parent)) + + +@pytest.mark.parametrize("field", ["PERIDNUM", "A_AGE", "DIS_VAL1"]) +def test_actual_owner_requires_retained_identity_and_money( + owner, tmp_path, monkeypatch, field +): + parent = prepared(owner, tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if field == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(field)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + + +def test_actual_owner_pins_new_survivor_dollars_to_original_member( + owner, tmp_path, monkeypatch +): + parent = prepared(owner, tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(root, pin): + # Only the invented fixture is touched. Even an unretained money field + # must match the exact already admitted source-member bytes. + path = root / "asec" / pin[1] + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + raw["SRVS_VAL"] = "999" + raw.to_csv(path, index=False) + return capture(root, pin) + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "literal", "evidence"] +) +def test_actual_owner_final_seal_covers_returned_values( + owner, tmp_path, monkeypatch, mutation +): + parent = prepared(owner, tmp_path, monkeypatch) + seal, calls = owner.retirement_detail_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + array = value.person.SUR_VAL1_amount.array + if mutation == "amount_bit": + i = np.flatnonzero(~array._mask)[0] + array._data[i] = np.nextafter(array._data[i], np.inf) + elif mutation == "masked_backing": + array._data[np.flatnonzero(array._mask)[0]] = 17.0 + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("SUR_YN") + ] = "2" + else: + value.evidence["taxability_assigned"] = True + return before + + monkeypatch.setattr(owner, "retirement_detail_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_retirement_detail(parent) + + +def test_actual_owner_requalifies_parent_after_last_capture( + owner, tmp_path, monkeypatch +): + parent = prepared(owner, tmp_path, monkeypatch) + capture, calls = owner._capture_member, [] + + def changed(*args): + raw = capture(*args) + table = parent._checked()[2].frame.person + table.iloc[0, table.columns.get_loc("age")] += 1 + calls.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + assert calls diff --git a/packages/microcosm-build/tests/test_us_current_asec_usual_hours.py b/packages/microcosm-build/tests/test_us_current_asec_usual_hours.py new file mode 100644 index 000000000..ca933e220 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_usual_hours.py @@ -0,0 +1,160 @@ +"""All-age source-code semantics on invented literals; no source/data admission.""" + +import pytest +from test_us_current_survey_hours import donor + +from microcosm.build.us_runtime import current_asec_usual_hours as asec +from microcosm.build.us_runtime import current_survey_hours as common + + +@pytest.mark.parametrize("age", [15, 16, 40, 80, 85]) +@pytest.mark.parametrize("hours", [1, 40, 99]) +def test_preserves_source_hours_without_an_earnings_predictor(age, hours): + row = donor(A_AGE=str(age), HRSWK=str(hours), MARSUPWT="0", FL_665="0", I_HRSWK="9") + result = asec.recode_asec_usual_hours(row) + assert result.hours == float(hours) + assert dict(result.raw) == row + assert result.provenance == "asec_positive_source_hours" + assert dict(result.allocation_flags)["FL_665"] == 0 + assert dict(result.allocation_flags)["I_HRSWK"] == 9 + assert result.policy is None and result.donor_key is None + + +@pytest.mark.parametrize("age", [0, 14, 30]) +def test_all_niu_remains_unresolved_and_is_not_observed_zero(age): + row = donor( + A_AGE=str(age), HRSWK="0", WKSWORK="0", WORKYN="0", WTEMP="0", WRK_CK="0" + ) + result = asec.recode_asec_usual_hours(row) + assert result.hours is None + assert result.provenance == ( + "asec_source_universe_unavailable" + if age < 15 + else "asec_work_history_unresolved" + ) + assert dict(result.raw) == row + + +@pytest.mark.parametrize("temporary", ["0", "2"]) +def test_final_nonwork_can_support_zero_even_when_temporary_answer_is_niu(temporary): + row = donor(positive=False, A_AGE="55", WTEMP=temporary) + result = asec.recode_asec_usual_hours(row) + assert result.hours == 0.0 + assert result.provenance == "asec_source_nonwork_completion" + assert dict(result.raw)["HRSWK"] == "0" + + +@pytest.mark.parametrize("temporary", ["0", "1"]) +def test_initial_no_is_not_a_final_no(temporary): + result = asec.recode_asec_usual_hours( + donor(A_AGE="40", WORKYN="2", WTEMP=temporary) + ) + assert result.hours == 20.0 + + +@pytest.mark.parametrize( + "changes", + [ + {"A_AGE": "14"}, + {"A_AGE": "86"}, + {"HRSWK": "100"}, + {"HRSWK": ""}, + {"WKSWORK": "0"}, + {"WRK_CK": "2"}, + {"WORKYN": "0"}, + {"WORKYN": "2", "WTEMP": "2"}, + {"I_HRSWK": "2"}, + {"FL_665": "4"}, + {"MARSUPWT": "-1"}, + {"HRSWK": 40}, + ], +) +def test_invalid_codes_or_inconsistent_work_history_refuse(changes): + with pytest.raises(ValueError, match="NATIVE_HOURS_"): + asec.recode_asec_usual_hours(donor(**changes)) + + +@pytest.mark.parametrize("positive", [False, True]) +def test_age15_observation_agrees_with_donor_recoder_without_issuing_authority( + positive, +): + row = donor(positive=positive) + assert asec.recode_asec_usual_hours(row) == common.recode_asec_age15_hours(row) + + +def test_unanswered_child_work_history_cannot_be_completed_from_an_earnings_value(): + row = donor( + A_AGE="14", + HRSWK="0", + WKSWORK="0", + WORKYN="0", + WTEMP="0", + WRK_CK="0", + WSAL_VAL="1000", + ) + result = asec.recode_asec_usual_hours(row) + assert result.hours is None + assert result.policy is None + + +def completion_row(**changes): + return donor(WSAL_VAL="0", SEMP_VAL="0", FRSE_VAL="0", **changes) + + +def child(**changes): + row = completion_row( + A_AGE="14", HRSWK="0", WKSWORK="0", WORKYN="0", WTEMP="0", WRK_CK="0" + ) + row.update(changes) + return row + + +def test_child_completion_is_explicit_and_preserves_niu_and_earnings_evidence(): + row = child() + with pytest.raises(ValueError, match="UNDER15_POLICY_REQUIRED"): + asec.propose_asec_usual_hours(row) + result = asec.propose_asec_usual_hours(row, under15_policy=common.UNDER15_POLICY) + assert result.hours == 0.0 + assert result.provenance == "under15_explicit_modeled_zero" + assert result.policy == common.UNDER15_POLICY + assert dict(result.raw) == row + assert asec.recode_asec_usual_hours(row).hours is None + + +@pytest.mark.parametrize("field", asec.EARNINGS_FIELDS) +@pytest.mark.parametrize("value", ["1", "-1", "", "unknown"]) +def test_child_earnings_conflicts_or_unknowns_refuse(field, value): + with pytest.raises(ValueError, match="NATIVE_HOURS_"): + asec.propose_asec_usual_hours( + child(**{field: value}), under15_policy=common.UNDER15_POLICY + ) + + +@pytest.mark.parametrize("field", asec.EARNINGS_FIELDS) +def test_absent_earnings_column_cannot_support_child_zero(field): + row = child() + del row[field] + with pytest.raises(ValueError, match="SOURCE_FIELDS"): + asec.propose_asec_usual_hours(row, under15_policy=common.UNDER15_POLICY) + + +def test_complete_proposal_refuses_unresolved_adult_even_with_zero_earnings(): + with pytest.raises(ValueError, match="UNRESOLVED_ADULT"): + asec.propose_asec_usual_hours( + child(A_AGE="30"), under15_policy=common.UNDER15_POLICY + ) + + +@pytest.mark.parametrize("age", [15, 35, 85]) +def test_observed_adult_hours_do_not_depend_on_earnings_or_child_policy(age): + row = completion_row(A_AGE=str(age), HRSWK="99") + row.update(WSAL_VAL="100000", SEMP_VAL="-1000", FRSE_VAL="0") + result = asec.propose_asec_usual_hours(row) + assert result.hours == 99.0 + assert result.policy is None + assert dict(result.raw) == row + + +def test_unknown_completion_policy_refuses(): + with pytest.raises(ValueError, match="UNDER15_POLICY"): + asec.propose_asec_usual_hours(child(), under15_policy="automatic") diff --git a/packages/microcosm-build/tests/test_us_current_property_income_sources.py b/packages/microcosm-build/tests/test_us_current_property_income_sources.py new file mode 100644 index 000000000..351e0e8cb --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_property_income_sources.py @@ -0,0 +1,439 @@ +"""Invented original-source composition; no PUF, QRF, native data or engine.""" + +import copy +import hashlib +import json +import shutil +from dataclasses import fields, is_dataclass, replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_property_income_sources as owner +from microcosm.build.us_runtime import graph_survey_population as graph +from microcosm.fit import model_input +from microcosm.frame import Frame, WeightKind, Weights + + +def _detached(value): + # Avoid Python 3.14 copy's __slotnames__ class-cache mutation on core + # slotted types. Real constructors preserve the same physical contents. + if isinstance(value, Frame): + return Frame( + {name: value.table(name).copy(deep=True) for name in value.entities}, + value.schema, + { + name: _detached(value.weights_for(name)) + for name in value.weighted_entities + }, + value.strata.copy(deep=True), + mass_log=value.mass_log, + metadata=value.metadata, + ) + if isinstance(value, Weights): + return Weights(value.values.copy(), value.kind) + if isinstance(value, pd.DataFrame): + return value.copy(deep=True) + if is_dataclass(value): + return type(value)( + **{ + field.name: _detached(getattr(value, field.name)) + for field in fields(value) + } + ) + if type(value) is tuple: + return tuple(_detached(item) for item in value) + if type(value) is list: + return [_detached(item) for item in value] + if type(value) is dict: + return {key: _detached(item) for key, item in value.items()} + return copy.deepcopy(value) + + +def source_arguments(tmp_path, monkeypatch): + import test_us_survey_population_preparation as fixture + from test_us_asec_coverage_authentication import _changed_parent + from test_us_current_asec_income_routing import routing_arguments + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + original = fixture._person + + def person(*args, **kwargs): + row = original(*args, **kwargs) + row.update(FINTP="0", FRETP="0", INTP="100", RETP="0") + if row["SERIALNO"] == "2024HU0000001" and int(row["SPORDER"]) == 1: + row.update(INTP="-800", ADJINC="1000133") + elif row["SERIALNO"] == "2024HU0000002": + row["INTP"] = "bad" + elif row["SERIALNO"] == "2024GQ0000001": + row.update(AGEP="14", INTP="", RETP="", WAGP="", SEMP="") + return row + + monkeypatch.setattr(fixture, "_person", person) + arguments = routing_arguments(tmp_path, monkeypatch) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + literals = {} + for pid, age in zip(ids, people.A_AGE, strict=True): + literals[int(pid)] = { + **{ + name: "0" + for name in (*owner.interest.READ_COLUMNS, *owner.dividend.READ_COLUMNS) + }, + "A_AGE": str(int(age)), + "INT_YN": "2" if age >= 15 else "0", + "RINT_YN": "2" if age >= 15 else "0", + "DIV_YN": "2" if age >= 15 else "0", + "SUR_YN": "2" if age >= 15 else "0", + } + literals[105].update( + INT_VAL="140", + TRDINT_VAL="100", + INT_YN="1", + RINT_YN="1", + RINT_SC1="4", + RINT_VAL1="40", + DIV_VAL="140", + DIV_YN="1", + ) + literals[107].update( + INT_VAL="500", + TRDINT_VAL="500", + INT_YN="1", + DIV_VAL="500", + DIV_YN="1", + SUR_YN="1", + SUR_SC2="8", + ) + changes = {} + for name in ("INT_VAL", "DIV_VAL"): + values = people[name].to_numpy(copy=True) + for pid in (105, 106, 107, 108): + values[ids == pid] = int(literals[pid][name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in set( + (*owner.interest.READ_COLUMNS, *owner.dividend.READ_COLUMNS) + ) - set(owner.routing.COORDINATE_COLUMNS): + raw[name] = [ + literals[int(updated.loc[key, "person_id"])][name] + for key in raw.PERIDNUM + ] + # Historical parent values remain their own current-money observations. + for name in ("INT_VAL", "DIV_VAL"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "property-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return {**arguments, "store_root": tmp_path / "property-survey-store"} + + +@pytest.fixture(scope="module") +def actual(tmp_path_factory): + patch = pytest.MonkeyPatch() + root = tmp_path_factory.mktemp("property-source-composition") + try: + arguments = source_arguments(root, patch) + live = graph.run_authenticated_survey_population( + **arguments, clones=True, return_values=True + ) + frame_seal = owner._frame_seal(live.preparation._checked()[2].frame) + result = owner.qualify_current_property_income_sources( + live.preparation, live.allocated_population, live.clone_population + ) + assert owner._frame_seal(live.preparation._checked()[2].frame) == frame_seal + (root / "composition-metadata.json").write_text( + json.dumps( + { + "scope": "invented source-composition fixture; no model or native data", + "survey_stage_order": list(live.compiled.order), + "source_people": result.source_frame.n("person"), + "source_households": result.source_frame.n("household"), + "allocated_people": live.allocated_population.frame.n("person"), + "clone_people": live.clone_population.frame.n("person"), + "eligible_donors": len(result.donor_columns), + "eligible_recipients": len(result.recipient_columns), + }, + indent=2, + ) + + "\n" + ) + yield live, result, arguments + finally: + patch.undo() + + +def test_actual_source_composition_preserves_design_and_signed_anchor(actual): + live, result, _ = actual + assert len(result.origins) == 9 + assert len(result.donor_columns) == 1 + assert len(result.recipient_columns) == 3 + assert result.donor_frame.weights_for("household").kind is WeightKind.DESIGN + assert ( + live.allocated_population.frame.weights_for("household").kind + is WeightKind.IMPORTANCE + ) + row = result.donor_columns.iloc[0] + assert row.property_ordinary_interest == 100 + assert row.property_retirement_interest == 40 + assert row.property_dividends == 140 + assert row.property_broad_receipts == -400 + assert row.property_reported_total == -120 + selected = result.recipient_columns.property_reported_total + assert (selected < 0).any() + amount = np.float64(-800) * (np.float64(1000133) / 1000000) + anchor = result.recipient_diagnostics + exact_id = anchor.index[ + anchor.SERIALNO.eq("2024HU0000001") & anchor.SPORDER.eq("1") + ][0] + assert selected.loc[exact_id].view("uint64") == amount.view("uint64") + pd.testing.assert_frame_equal( + model_input.decode_recipient_matrix(result.recipient_matrix).features, + result.recipient_columns, + ) + assert result.recipient_diagnostics.excluded_under15.sum() == 1 + assert result.recipient_diagnostics.excluded_unknown_anchor.sum() == 2 + assert result.evidence["source_admission_issued"] is False + assert not result.evidence["model_fitted"] + assert not result.evidence["clone_attachment_performed"] + assert json.loads(result.origin_document)["persons"]["rows"] + + +def _values(result): + return ( + result.shared_predictors, + result.acs_anchor_values, + result.asec_interest_values, + result.asec_routing_values, + result.asec_dividend_values, + ) + + +@pytest.mark.parametrize( + "target", + [ + "asec_person", + "asec_native", + "acs_person", + "acs_native", + "donor_features", + "recipient_matrix", + "origin_document", + ], +) +def test_source_composition_refuses_axis_mismatch(actual, target): + _, result, _ = actual + values = list(_detached(_values(result))) + origin = result.origin_document + if target == "asec_person": + values[2].person.index = values[2].person.index[::-1] + elif target == "asec_native": + values[3].person["native_person_id"] = values[ + 3 + ].person.native_person_id.to_numpy()[::-1] + elif target == "acs_person": + values[1].anchors.index = values[1].anchors.index[::-1] + elif target == "acs_native": + values[1].anchors["native_person_id"] = values[ + 1 + ].anchors.native_person_id.to_numpy()[::-1] + elif target == "donor_features": + values[0].donor_columns.index = values[0].donor_columns.index[::-1] + elif target == "recipient_matrix": + matrix = model_input.decode_recipient_matrix(values[0].matrix) + features = matrix.features.iloc[::-1] + values[0] = replace( + values[0], + matrix=model_input.encode_recipient_matrix( + features, entity="person", entity_ids=features.index.to_numpy() + ), + ) + else: + document = json.loads(origin) + document["persons"]["rows"].reverse() + origin = owner.shared.codec.encode_json(document) + with pytest.raises(ValueError, match="PROPERTY_INCOME_SOURCES"): + owner._compose(tuple(values), origin) + + +@pytest.mark.parametrize( + "target", + [ + "basis", + "nullable_storage", + "donor_columns", + "recipient_columns", + "origin_document", + "source_frame", + "projection", + ], +) +def test_complete_description_seal_detects_every_returned_surface(actual, target): + _, result, _ = actual + before = owner.property_income_sources_seal(result) + changed = _detached(result) + if target == "basis": + changed.donor_basis.person.iloc[ + 0, changed.donor_basis.person.columns.get_loc("property_reported_total") + ] += 1 + elif target == "nullable_storage": + array = changed.acs_anchor_values.anchors.property_income_amount.array + position = np.flatnonzero(array._mask)[0] + array._data[position] = 321.0 + elif target == "donor_columns": + changed.donor_columns.iloc[0, 0] += 1 + elif target == "recipient_columns": + changed.recipient_columns.iloc[0, 0] += 1 + elif target == "origin_document": + changed = replace(changed, origin_document=changed.origin_document + b" ") + elif target == "source_frame": + changed.source_frame.person.iloc[ + 0, changed.source_frame.person.columns.get_loc("age") + ] += 1 + else: + changed = replace(changed, projection=changed.projection + b" ") + assert owner.property_income_sources_seal(changed) != before + assert owner.property_income_sources_seal(result) == before + + +def test_no_eligible_recipient_is_explicit_without_empty_matrix_or_zero_fill(actual): + _, result, _ = actual + values = list(_detached(_values(result))) + anchors = values[1].anchors + anchors["property_income_known"] = False + anchors["property_income_amount"] = pd.array([None] * len(anchors), dtype="Float64") + anchors["property_income_status"] = "missing_source_amount" + rebuilt = owner._compose(tuple(values), result.origin_document) + assert rebuilt.recipient_frame is None and rebuilt.recipient_matrix is None + assert len(rebuilt.recipient_columns) == 0 + assert len(rebuilt.recipient_diagnostics) == 5 + + +@pytest.mark.parametrize("target", ["columns", "geography_config"]) +def test_final_io_cannot_change_already_composed_values(actual, monkeypatch, target): + live, result, _ = actual + # The first test already ran every real source owner. Here the same detached + # values isolate the final real preparation I/O fence; no new issuer or + # source admission is mocked. Any unchecked descriptive replay still has to + # survive the real original-owner check and final full-value seal. + values = list(_detached(_values(result))) + config = None + if target == "geography_config": + config = owner.shared.host.survey_budget.geography.AtomicSurveyReconstruction( + "/invented/support.json", + "a" * 64, + ( + ("district", "invented-district"), + ("population", "invented-population"), + ("puma", "invented-puma"), + ), + 1, + ) + # This is only the option-lifetime fault injection. No support file is + # opened or admitted, and no geography reconstruction is claimed. + values[0] = replace(values[0], geography_config_payload=config.to_bytes()) + for module, name, value in ( + (owner.shared, "qualify_current_survey_predictors", values[0]), + (owner.acs, "qualify_current_acs_income_anchors", values[1]), + (owner.interest, "qualify_current_asec_interest", values[2]), + (owner.routing, "qualify_current_asec_income_routing", values[3]), + (owner.dividend, "qualify_current_asec_dividend", values[4]), + ): + monkeypatch.setattr(module, name, lambda *a, _value=value, **kw: _value) + original = owner._compose + completed = [] + + def compose(*args): + value = original(*args) + completed.append(value) + return value + + monkeypatch.setattr(owner, "_compose", compose) + open_path = Path.open + mutated = [] + + def open_during_final_io(path, *args, **kwargs): + stream = open_path(path, *args, **kwargs) + if completed and not mutated: + if target == "columns": + completed[0].donor_columns.iloc[0, 0] += 1 + else: + object.__setattr__(config, "seed", 2) + mutated.append(str(path)) + return stream + + # Keep the issuer method itself untouched: replacing it is rightly refused + # by its own live-code seal before the intended last-I/O fence is reached. + monkeypatch.setattr(Path, "open", open_during_final_io) + reason = ( + "FINAL_DERIVED_VALUES_CHANGED" + if target == "columns" + else "FINAL_GEOGRAPHY_CONFIG_CHANGED" + ) + with pytest.raises(ValueError, match=reason): + owner.qualify_current_property_income_sources( + live.preparation, + live.allocated_population, + live.clone_population, + geography_config=config, + ) + + assert len(mutated) == 1 + + +def test_unissued_plain_object_refuses_without_source_io(): + with pytest.raises(ValueError, match="PREPARATION_TYPE"): + owner.qualify_current_property_income_sources(object(), None, None) + + +def test_constructor_weight_copy_preserves_source_live_identity(): + before = owner.shared.source._live() + copied = _detached(Weights(np.array([1.0]), WeightKind.DESIGN)) + assert copied.values.tolist() == [1.0] + assert owner.shared.source._live() == before + + +def test_unissued_preparation_copy_refuses_before_composition(actual): + live, _, _ = actual + before = owner.shared.source._live() + # Copy only the descriptive payload into an unissued identity. Avoid the + # unrelated stdlib copy class-cache side effect in this ownership control. + copied = object.__new__(type(live.preparation)) + object.__setattr__(copied, "payload", live.preparation.payload) + with pytest.raises(ValueError): + owner.qualify_current_property_income_sources( + copied, + live.allocated_population, + live.clone_population, + ) + assert owner.shared.source._live() == before diff --git a/packages/microcosm-build/tests/test_us_current_survey_amounts.py b/packages/microcosm-build/tests/test_us_current_survey_amounts.py new file mode 100644 index 000000000..89f92aecb --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_amounts.py @@ -0,0 +1,316 @@ +"""Invented reporting bases and clone joins; no native data or country engine.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_unemployment_source as uc +from microcosm.build.us_runtime import current_survey_amounts as values +from microcosm.fit import model_input +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + + +@pytest.mark.parametrize( + "amount,age,token,label,canonical", + [ + (100, 40, "1", "known_receipt", 100), + (0, 40, "2", "known_nonreceipt", 0), + (0, 40, "1", "ambiguous_recipient_zero", None), + (0, 40, "0", "niu", None), + (0, 14, "0", "outside_reporting_universe", None), + (0, 14, "2", "contradictory_outside_reporting_universe", None), + (100, 14, "1", "contradictory_outside_reporting_universe", None), + (100, 40, "2", "contradictory_no_positive", None), + (100, 40, "0", "contradictory_niu_positive", None), + (0, 40, "", "missing_receipt_literal", None), + (100, 40, "9", "unrecognized_receipt_literal", None), + (100, 40, "yes", "unrecognized_receipt_literal", None), + (100, 40, "01", "unrecognized_receipt_literal", None), + (np.nan, 40, "1", "missing_amount", None), + ], +) +def test_uc_raw_amount_and_reporting_knownness_are_separate( + amount, age, token, label, canonical +): + basis = uc.reporting_basis( + np.array([amount], dtype="float64"), np.array([age], dtype="float64"), [token] + ) + row = basis.iloc[0] + assert row.reporting_status == label + assert row.receipt_literal == token + assert bool(row.source_reporting_universe) == (age >= 15) + assert bool(row.canonical_amount_known) == (canonical is not None) + if canonical is None: + assert np.isnan(row.canonical_amount) + else: + assert row.canonical_amount == canonical + assert ( + np.isnan(row.source_amount) if np.isnan(amount) else row.source_amount == amount + ) + + +@pytest.mark.parametrize( + "amount,age", [(-1, 40), (100000, 40), (np.inf, 40), (0, -1), (0, 40.5)] +) +def test_uc_refuses_out_of_dictionary_numeric_domain(amount, age): + with pytest.raises(ValueError): + uc.reporting_basis( + np.array([amount], dtype="float64"), np.array([age], dtype="float64"), ["2"] + ) + + +def test_uc_bounded_literal_reader_and_duplicate_join_refusal(tmp_path): + path = tmp_path / "invented.csv" + header = ",".join(uc.READ_COLUMNS) + "\n" + row = "0000000000000000000001,1,1,40,0,2\n" + path.write_text(header + row) + result = uc._read_capture(path, rows=1) + assert result.iloc[0].UC_YN == "2" + path.write_text(header + row + row) + with pytest.raises(ValueError, match="DUPLICATE"): + uc._read_capture(path, rows=2) + path.write_text(header + row.replace(",0,2", ",0," + "9" * 65)) + with pytest.raises(ValueError, match="TOKEN_BOUND"): + uc._read_capture(path, rows=1) + + +def test_money_literal_join_retains_selected_and_unselected_missingness(): + field = uc.money.MoneyField( + "UC_VAL", + np.array([0.0, 0.0, 40.0], dtype="= 16 else "", + "WKL": "1" if age >= 16 else "", + "FWKHP": "0", + "WAGP": "0", + "SEMP": "0", + **changes, + } + + +def donor(number=1, positive=True, **changes): + return { + "PERIDNUM": str(number).zfill(22), + "PH_SEQ": str(number), + "A_LINENO": "1", + "A_AGE": "15", + "HRSWK": "20" if positive else "0", + "WKSWORK": "10" if positive else "0", + "WORKYN": "1" if positive else "2", + "WTEMP": "0" if positive else "2", + "WRK_CK": "1" if positive else "2", + "MARSUPWT": "100", + "I_HRSWK": "0", + "I_WKSWK": "0", + "I_WORKYN": "0", + "I_WTEMP": "0", + "FL_665": "1", + **changes, + } + + +def propose(rows, donors=(), **kwargs): + return hours.propose_acs_usual_hours(rows, donors=donors, **kwargs) + + +@pytest.mark.parametrize("value", ["40", "99"]) +@pytest.mark.parametrize( + "flag,kind", + [("0", "not_allocated"), ("1", "allocated"), ("", "allocation_unknown")], +) +def test_source_observations_and_allocation_survive(value, flag, kind): + row = acs(WKHP=value, FWKHP=flag) + result = hours.recode_acs_usual_hours(row) + assert result.hours == float(value) + assert result.provenance == "acs_native_hours_" + kind + assert dict(result.raw) == row + + +def test_nonwork_is_source_completion_not_observed_zero_and_esr_is_irrelevant(): + observed = acs(ESR="3", hours_worked_last_week="0") + assert hours.recode_acs_usual_hours(observed).hours == 40 + result = hours.recode_acs_usual_hours(acs(WKHP="", WKL="2", WAGP="500")) + assert result.hours == 0 + assert result.provenance == "acs_source_nonwork_completion" + + +@pytest.mark.parametrize("age", [0, 14, 15]) +def test_survey_universe_absence_is_not_zero(age): + result = hours.recode_acs_usual_hours(acs(age)) + assert result.hours is None + assert result.provenance == "acs_source_universe_unavailable" + + +@pytest.mark.parametrize( + "changes", + [ + {"WKHP": "0"}, + {"WKHP": "1.5"}, + {"WKHP": "b"}, + {"WKHP": "100"}, + {"FWKHP": "2"}, + {"WKL": "4"}, + {"WKL": "2"}, + {"AGEP": ""}, + {"AGEP": "15"}, + {"WAGP": "NaN"}, + {"SERIALNO": "2024HU1"}, + {"SPORDER": "0"}, + {"SPORDER": True}, + ], +) +def test_invalid_acs_literals_and_contradictions_refuse(changes): + with pytest.raises(ValueError, match="NATIVE_HOURS_"): + hours.recode_acs_usual_hours(acs(**changes)) + + +def test_missing_header_is_not_a_blank_answer(): + row = acs() + del row["WKHP"] + with pytest.raises(ValueError, match="SOURCE_FIELDS"): + hours.recode_acs_usual_hours(row) + + +def test_policy_is_explicit_and_unknown_earnings_stay_unknown(): + row = acs(14, WAGP="", SEMP="") + with pytest.raises(ValueError, match="UNDER15_POLICY_REQUIRED"): + propose([row]) + p = propose([row], under15_policy=hours.UNDER15_POLICY).proposals[0] + assert p.hours == 0 and p.provenance == "under15_explicit_modeled_zero" + assert p.unknown_earnings == ("WAGP", "SEMP") + assert dict(p.raw)["WKHP"] == "" + + +@pytest.mark.parametrize("changes", [{"WAGP": "1"}, {"SEMP": "-1"}, {"SEMP": "1"}]) +def test_under15_policy_refuses_earnings_signal(changes): + with pytest.raises(ValueError, match="UNDER15_EARNINGS"): + propose([acs(14, **changes)], under15_policy=hours.UNDER15_POLICY) + + +def test_age15_and_unresolved_adults_never_get_a_generic_fallback(): + with pytest.raises(ValueError, match="AGE15_DONORS_REQUIRED"): + propose([acs(15)], donors=[donor()]) + with pytest.raises(ValueError, match="AGE15_DONORS_REQUIRED"): + propose([acs(15)], age15_policy=hours.AGE15_POLICY) + with pytest.raises(ValueError, match="UNRESOLVED_ADULT"): + propose([acs(16, WKHP="")], donors=[donor()], age15_policy=hours.AGE15_POLICY) + + +def test_asec_temporary_followup_is_work_and_flags_remain_separate(): + row = donor(WORKYN="2", WTEMP="1", I_HRSWK="9", FL_665="3") + p = hours.recode_asec_age15_hours(row) + assert p.hours == 20 and p.provenance == "asec_positive_source_hours" + result = propose([acs(15, FWKHP="")], [row], age15_policy=hours.AGE15_POLICY) + draw = result.proposals[0] + assert draw.provenance == "age15_weighted_empirical_draw" + assert draw.donor_key == p.key and draw.donor_allocation_flags == p.allocation_flags + assert draw.allocation_flags == (("FWKHP", None),) + assert not result.source_authenticated + assert not result.complete_donor_cohort_authenticated + assert not result.release_qualified + + +@pytest.mark.parametrize( + "changes", + [ + {"A_AGE": "14"}, + {"A_AGE": "16"}, + {"MARSUPWT": "0"}, + {"MARSUPWT": "1.5"}, + {"HRSWK": "100"}, + {"WKSWORK": "0"}, + {"WRK_CK": "2"}, + {"WORKYN": "2", "WTEMP": "2"}, + {"I_HRSWK": "2"}, + {"FL_665": "4"}, + {"PERIDNUM": "123"}, + ], +) +def test_unqualified_asec_domains_refuse(changes): + with pytest.raises(ValueError, match="NATIVE_HOURS_"): + hours.recode_asec_age15_hours(donor(**changes)) + + +def test_asec_zero_is_nonwork_completion_and_initial_no_is_not_sufficient(): + p = hours.recode_asec_age15_hours(donor(positive=False)) + assert p.hours == 0 and p.provenance == "asec_source_nonwork_completion" + with pytest.raises(ValueError, match="ASEC_WORK_CONTRADICTION"): + hours.recode_asec_age15_hours(donor(positive=False, WTEMP="1")) + + +@pytest.mark.parametrize("positive", [False, True]) +def test_complete_supplement_nonresponse_is_preserved_not_rejected(positive): + # Census2025 FL_665: 0=complete supplement nonresponse; valid source status, + # not proof of observed hours or independently qualified donor eligibility. + # https://api.census.gov/data/2025/cps/asec/mar/variables/FL_665.json + row = donor(positive=positive, FL_665="0", I_HRSWK="1") + recoded = hours.recode_asec_age15_hours(row) + assert dict(recoded.raw)["FL_665"] == "0" + assert dict(recoded.allocation_flags)["FL_665"] == 0 + batch = propose([acs(15)], [row], age15_policy=hours.AGE15_POLICY) + draw = batch.proposals[0] + assert draw.hours == (20 if positive else 0) + assert dict(draw.donor_allocation_flags)["FL_665"] == 0 + assert draw.provenance == "age15_weighted_empirical_draw" + assert not batch.complete_donor_cohort_authenticated + assert not batch.release_qualified + + +def test_temporary_niu_is_not_an_explicit_no_against_final_work(): + result = hours.recode_asec_age15_hours(donor(WORKYN="2", WTEMP="0", WRK_CK="1")) + assert result.hours == 20 + assert dict(result.raw)["WTEMP"] == "0" + assert result.provenance == "asec_positive_source_hours" + + +def test_weighted_cdf_boundaries_preserve_actual_donor_and_provenance(): + donors = [donor(1, False, MARSUPWT="100"), donor(2, True, MARSUPWT="300")] + for uniform, expected in [ + (0, 0), + (np.nextafter(0.25, 0), 0), + (0.25, 20), + (np.nextafter(1.0, 0), 20), + ]: + with patch.object(hours, "native_hours_uniform", return_value=uniform): + p = propose([acs(15)], donors, age15_policy=hours.AGE15_POLICY).proposals[0] + assert p.hours == expected + assert ( + p.donor_key == hours.recode_asec_age15_hours(donors[int(expected > 0)]).key + ) + + +def test_permutation_batches_and_nonidentity_earnings_do_not_change_draws(): + rows = [acs(15, SERIALNO=f"2024HU{i:07d}") for i in range(1, 25)] + donors = [donor(2), donor(1, False), donor(3, I_HRSWK="1")] + before = deepcopy((rows, donors)) + full = propose(rows, donors, age15_policy=hours.AGE15_POLICY) + shuffled = propose( + rows[::-1], donors[1:] + donors[:1], age15_policy=hours.AGE15_POLICY + ) + assert full.proposals == shuffled.proposals[::-1] + batches = tuple( + p + for batch in (rows[:7], rows[7:]) + for p in propose(batch, donors, age15_policy=hours.AGE15_POLICY).proposals + ) + assert batches == full.proposals + changed = propose( + [{**r, "WAGP": "500", "SEMP": "-10"} for r in rows], + donors, + age15_policy=hours.AGE15_POLICY, + ) + assert [(p.hours, p.donor_key) for p in changed.proposals] == [ + (p.hours, p.donor_key) for p in full.proposals + ] + assert (rows, donors) == before + + +def test_cdf_matches_reviewed_weight_unit_conversion_order(): + donors = [donor(1, False, MARSUPWT="117"), donor(2, True, MARSUPWT="192637")] + weights = np.asarray([117, 192637], dtype=float) / 100.0 + weights /= np.max(weights) + boundary = np.cumsum(weights / weights.sum())[0] + for uniform, expected in ((np.nextafter(boundary, 0), 0), (boundary, 20)): + with patch.object(hours, "native_hours_uniform", return_value=uniform): + p = propose([acs(15)], donors, age15_policy=hours.AGE15_POLICY).proposals[0] + assert p.hours == expected + + +def test_native_key_is_not_row_rank_and_zero_padding_cannot_duplicate_identity(): + a = hours.recode_acs_usual_hours(acs(SPORDER="02")).key + b = hours.recode_acs_usual_hours(acs(SPORDER="2")).key + assert a == b + assert hours.native_hours_uniform(a) == hours.native_hours_uniform(b) + assert a.household == "2024HU0000001" + assert hours.native_hours_uniform(a) != hours.native_hours_uniform( + replace(a, line=10) + ) + with pytest.raises(ValueError, match="DUPLICATE_RECIPIENT_KEY"): + propose([acs(SPORDER="02"), acs(SPORDER="2")]) + with pytest.raises(ValueError, match="KEY_TYPE"): + replace(a, line=True) + + +@pytest.mark.parametrize( + "second", [donor(), donor(2, PH_SEQ="1"), donor(2, PERIDNUM="1".zfill(22))] +) +def test_duplicate_donor_native_identities_refuse(second): + with pytest.raises(ValueError, match="DUPLICATE_"): + propose([acs(15)], [donor(), second], age15_policy=hours.AGE15_POLICY) + + +def test_maximum_hash_cannot_make_uniform_one(): + class Maximum: + def digest(self): + return b"\xff" * 8 + + with patch.object(hours.hashlib, "blake2b", return_value=Maximum()): + assert ( + hours.native_hours_uniform(hours.recode_acs_usual_hours(acs()).key) + == (2**53 - 1) / 2**53 + ) + + +def test_pure_proposal_does_not_read_files_or_claim_authority(): + with patch("builtins.open", side_effect=AssertionError("I/O forbidden")): + result = propose([acs()]) + document = asdict(result) + assert document["source_authenticated"] is False + assert document["complete_donor_cohort_authenticated"] is False + assert document["release_qualified"] is False + assert hours.TARGET == "weekly_hours_worked_before_lsr" diff --git a/packages/microcosm-build/tests/test_us_current_survey_hours_source.py b/packages/microcosm-build/tests/test_us_current_survey_hours_source.py new file mode 100644 index 000000000..eacd17597 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_hours_source.py @@ -0,0 +1,501 @@ +"""Real survey issuers and captures over invented archives, without an engine.""" + +import hashlib +import io +import json +import shutil +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_survey_hours_source as owner + + +def hours_acs_person(original_person, *, gq_age_changes_afterward=False): + def person(*args, **kwargs): + row = original_person(*args, **kwargs) + under16 = int(row["AGEP"]) < 16 or ( + gq_age_changes_afterward and row["SERIALNO"] == "2024GQ0000001" + ) + row.update(WKHP="" if under16 else "40", WKL="" if under16 else "1", FWKHP="0") + return row + + return person + + +def add_hours_source_fields(arguments, monkeypatch): + """Set wholly invented hours literals before any source owner is issued.""" + from microcosm.build.us_runtime import asec_person_income_source as restoration + + folder = arguments["source_dir"] / "asec" + pins, paths = [], {} + for year, member, archive, *_ in owner.original.asec._MEMBER_PINS: + path = folder / member + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + raw["MARSUPWT"] = "10000" + for name in owner.asec_hours.EARNINGS_FIELDS: + if name not in raw: + raw[name] = "0" + raw.loc[raw.A_AGE.map(int).lt(15), name] = "0" + for name in ( + "HRSWK", + "WKSWORK", + "WORKYN", + "WTEMP", + "WRK_CK", + *owner.hours.ALLOCATION_FLAGS, + ): + raw[name] = "0" + for position, age in enumerate(raw.A_AGE.map(int)): + if age >= 15: + raw.loc[position, ["HRSWK", "WKSWORK", "WORKYN", "WRK_CK"]] = [ + "12", + "30", + "1", + "1", + ] + # Preserve a valid complete-supplement-nonresponse flag as a distinct fact. + raw["FL_665"] = "0" + raw.to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.original.asec, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + restored = folder.parent.parent / "hours-restored-money" + restoration.restore_asec_person_income_source( + folder / "parent.h5", + folder / "household-attachment.h5", + member_paths=paths, + output_dir=restored, + ) + shutil.copyfile( + restored / restoration.CHECKPOINT_FILENAME, + folder / "person-income-attachment.h5", + ) + # A private fixture cardinality seam, before any owner is issued. Production + # remains bound to 2174 keys from the complete income-2024 ASEC coverage. + monkeypatch.setattr(owner, "_EXPECTED_DONORS", 1) + monkeypatch.setattr(owner, "_LIVE", owner._live()) + return arguments + + +def source_arguments(tmp_path, monkeypatch): + import test_us_child_property_income_source_owner as fixture + import test_us_survey_population_preparation as preparation_fixture + + monkeypatch.setattr( + preparation_fixture, + "_person", + hours_acs_person(preparation_fixture._person, gq_age_changes_afterward=True), + ) + full, partial, donor = fixture.source_arguments(tmp_path, monkeypatch) + add_hours_source_fields(full, monkeypatch) + for member in ( + *(pin[1] for pin in owner.original.asec._MEMBER_PINS), + "person-income-attachment.h5", + ): + shutil.copyfile( + full["source_dir"] / "asec" / member, + partial["source_dir"] / "asec" / member, + ) + return full, partial, donor + + +def qualify(preparation): + return owner.qualify_current_survey_hours( + preparation, + age15_policy=owner.hours.AGE15_POLICY, + under15_policy=owner.hours.UNDER15_POLICY, + ) + + +@pytest.fixture(scope="module") +def actual(tmp_path_factory): + root = tmp_path_factory.mktemp("hours-source-owner") + with pytest.MonkeyPatch.context() as patch: + full_args, partial_args, donor = source_arguments(root, patch) + full = owner.source.prepare_authenticated_survey_population(**full_args) + partial = owner.source.prepare_authenticated_survey_population(**partial_args) + full_seal = owner.source._frame_identity(full._checked()[2].frame) + partial_seal = owner.source._frame_identity(partial._checked()[2].frame) + values, smaller = qualify(full), qualify(partial) + yield SimpleNamespace( + full=full, partial=partial, values=values, smaller=smaller, donor=donor + ) + assert owner.source._frame_identity(full._checked()[2].frame) == full_seal + assert owner.source._frame_identity(partial._checked()[2].frame) == partial_seal + + +def test_full_donor_roster_is_preserved_outside_selected_support(actual): + left, right = actual.values, actual.smaller + left.validate() + right.validate() + assert not right.asec_selected_raw.A_AGE.eq("15").any() + pd.testing.assert_frame_equal(left.donor_raw, right.donor_raw) + assert left.donor_raw.PERIDNUM.tolist() == [str(actual.donor - 100).zfill(22)] + assert left.donor_raw.FL_665.tolist() == ["0"] + assert left.proposals.supplied_donors == right.proposals.supplied_donors == 1 + all_proposals = {p.key: p for p in left.proposals.proposals} + assert all(p == all_proposals[p.key] for p in right.proposals.proposals) + teenager = next(p for p in left.proposals.proposals if dict(p.raw)["AGEP"] == "15") + assert teenager.hours == 12.0 + assert teenager.policy == owner.hours.AGE15_POLICY + assert teenager.donor_key.person == str(actual.donor - 100).zfill(22) + assert dict(teenager.donor_allocation_flags)["FL_665"] == 0 + assert dict(teenager.raw)["WKHP"] == "" + assert not left.proposals.source_authenticated + evidence = json.loads(left.receipt) + assert evidence["projection_scope"] == "selected_original_survey_people" + assert evidence["full_age15_donor_rows"] == 1 + assert evidence["selected_acs_rows"] == len(left.proposals.proposals) + assert not evidence["all_person_engine_input_qualified"] + assert evidence["asec_own_arm_hours_qualified"] + assert evidence["selected_original_person_hours_qualified"] + assert not evidence["source_admission_issued"] and not evidence["release_eligible"] + + +def test_both_original_arms_have_complete_ordered_hours_without_mutating_frame(actual): + value = actual.values + assert value.person_hours.index.equals(value.origins.index) + assert not value.person_hours[owner.hours.TARGET].isna().any() + assert owner.hours.TARGET not in value.source_frame.person + asec_ids = value.origins.index[value.origins.source.eq("asec")] + for pid, proposal in zip(asec_ids, value.asec_proposals.proposals, strict=True): + row = value.asec_selected_raw.loc[pid] + assert proposal.key == owner.hours._asec_key(row) + assert value.person_hours.at[pid, owner.hours.TARGET] == proposal.hours + assert value.person_hours.at[pid, "hours_provenance"] == proposal.provenance + if int(row.A_AGE) < 15: + assert proposal.hours == 0.0 + assert proposal.policy == owner.hours.UNDER15_POLICY + assert proposal.provenance == "under15_explicit_modeled_zero" + assert dict(proposal.raw)["HRSWK"] == "0" + else: + assert proposal.hours == 12.0 + assert proposal.policy is None + expected = value.person_hours.loc[actual.smaller.person_hours.index] + pd.testing.assert_frame_equal(actual.smaller.person_hours, expected) + evidence = json.loads(value.receipt) + assert evidence["selected_person_rows"] == len(value.person_hours) + assert evidence["asec_completion_protocol"] == owner.asec_hours.COMPLETION_PROTOCOL + + +def test_nested_asec_proposal_mutation_refuses(actual): + proposal = actual.values.asec_proposals.proposals[0] + previous = proposal.hours + try: + object.__setattr__(proposal, "hours", 99.0) + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + actual.values.validate() + finally: + object.__setattr__(proposal, "hours", previous) + actual.values.validate() + + +def test_asec_completion_policy_change_cannot_reuse_source_owner(actual, monkeypatch): + with monkeypatch.context() as patch: + patch.setattr(owner.asec_hours, "COMPLETION_PROTOCOL", "different") + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + actual.values.validate() + actual.values.validate() + + +def test_numeric_person_hours_mutation_refuses(actual): + table = actual.values.person_hours + pid = table.index[0] + previous = table.at[pid, owner.hours.TARGET] + try: + table.at[pid, owner.hours.TARGET] = previous + 1 + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + actual.values.validate() + finally: + table.at[pid, owner.hours.TARGET] = previous + actual.values.validate() + + +def receiving_people(qualified): + import numpy as np + + provenance = owner.attachment.provenance + origins = qualified.origins + original = np.tile(origins.index.to_numpy(), 2) + selected = origins.reindex(original) + return ( + pd.DataFrame( + { + "person_id": np.arange(len(original), dtype="int64") + 100000, + provenance.support_source_id_column("person"): original, + provenance.support_clone_index_column("person"): np.repeat( + [0, 1], len(origins) + ).astype("int64"), + provenance.spine_source_id_column( + "person" + ): selected.native_person_id.to_numpy(), + provenance.support_channel_column("person"): pd.array( + selected.source.to_numpy(), dtype=owner.STRING + ), + } + ) + .iloc[::-1] + .reset_index(drop=True) + ) + + +def test_clone_transport_preserves_original_draw_and_provenance_under_reordering( + actual, +): + value = actual.values + people = receiving_people(value) + before = people.copy(deep=True) + result = owner.borrow_cloned_hours_columns(value, people) + source_id = owner.attachment.provenance.support_source_id_column("person") + for column in value.person_hours: + expected = value.person_hours[column].reindex(people[source_id]) + expected.index = pd.Index(people.person_id.to_numpy(), name="person_id") + pd.testing.assert_series_equal(result["person", column], expected) + pd.testing.assert_frame_equal(people, before) + result["person", owner.hours.TARGET].iloc[0] = 97.0 + value.validate() + + +@pytest.mark.parametrize( + "change", ["source", "native", "channel", "clone", "collision"] +) +def test_clone_transport_refuses_wrong_identity_or_owned_output(actual, change): + people = receiving_people(actual.values) + p = owner.attachment.provenance + if change == "collision": + people[owner.hours.TARGET] = 40.0 + elif change == "channel": + people.loc[0, p.support_channel_column("person")] = "foreign" + else: + column = { + "source": p.support_source_id_column("person"), + "native": p.spine_source_id_column("person"), + "clone": p.support_clone_index_column("person"), + }[change] + people.loc[0, column] = 99999 + with pytest.raises(ValueError): + owner.borrow_cloned_hours_columns(actual.values, people) + + +def test_clone_transport_checks_receiving_mutation_after_final_source_io( + actual, monkeypatch +): + people = receiving_people(actual.values) + original = owner.QualifiedSurveyHoursProposals.validate + calls = SimpleNamespace(count=0) + + def validation(value): + original(value) + calls.count += 1 + if calls.count == 2: + people.loc[0, "person_id"] += 1 + + # Change the method only before this test's accepted implementation profile; + # the resulting callback simulates mutation during final foreign-owner I/O. + with monkeypatch.context() as patch: + patch.setattr(owner.QualifiedSurveyHoursProposals, "validate", validation) + patch.setattr(owner, "_LIVE", owner._live()) + with pytest.raises(ValueError, match="RECEIVING_CHANGED"): + owner.borrow_cloned_hours_columns(actual.values, people) + actual.values.validate() + + +@pytest.mark.parametrize("which", ["owner", "callback", "preparation", "receipt"]) +def test_copies_callbacks_and_receipts_cannot_replace_the_live_owner(actual, which): + value = actual.values + if which == "owner": + with pytest.raises(ValueError, match="PROJECTION_OBJECT_CHANGED"): + replace(value).validate() + elif which == "preparation": + fake = object.__new__(type(actual.full)) + object.__setattr__(fake, "payload", actual.full.payload) + with pytest.raises(ValueError): + qualify(fake) + else: + field = "_revalidate" if which == "callback" else "receipt" + old = getattr(value, field) + try: + object.__setattr__( + value, field, (lambda _: None) if which == "callback" else b"{}" + ) + with pytest.raises(ValueError): + value.validate() + finally: + object.__setattr__(value, field, old) + value.validate() + + +@pytest.mark.parametrize( + "field,column", + [ + ("acs_raw", "WKHP"), + ("asec_selected_raw", "HRSWK"), + ("donor_raw", "HRSWK"), + ("origins", "source"), + ("person_hours", "hours_provenance"), + ("asec_selected_raw", "WSAL_VAL"), + ], +) +def test_mutated_source_projection_refuses(actual, field, column): + table = getattr(actual.values, field) + position = table.index[0] + previous = table.at[position, column] + try: + table.at[position, column] = "changed" + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + actual.values.validate() + finally: + table.at[position, column] = previous + actual.values.validate() + + +@pytest.mark.parametrize("field", ["hours", "donor_key", "donor_allocation_flags"]) +def test_nested_proposal_mutation_refuses(actual, field): + proposal = next( + p for p in actual.values.proposals.proposals if p.donor_key is not None + ) + previous = getattr(proposal, field) + try: + object.__setattr__( + proposal, + field, + {"hours": 99.0, "donor_key": None, "donor_allocation_flags": ()}[field], + ) + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + actual.values.validate() + finally: + object.__setattr__(proposal, field, previous) + actual.values.validate() + + +def test_final_io_cannot_mutate_then_hide_a_projection(actual, monkeypatch): + value = actual.values + pid = value.donor_raw.index[0] + previous = value.donor_raw.at[pid, "HRSWK"] + read_bytes = Path.read_bytes + triggered = [] + + def mutate_after_read(path): + data = read_bytes(path) + if not triggered: + triggered.append(True) + value.donor_raw.at[pid, "HRSWK"] = "99" + return data + + try: + with monkeypatch.context() as patch: + patch.setattr(Path, "read_bytes", mutate_after_read) + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + value.validate() + assert triggered == [True] + finally: + value.donor_raw.at[pid, "HRSWK"] = previous + value.validate() + + +@pytest.mark.parametrize( + "field", ["acs_raw", "donor_raw", "proposals", "asec_proposals", "person_hours"] +) +def test_equal_detached_projection_cannot_replace_owned_identity(actual, field): + value = actual.values + previous = getattr(value, field) + detached = ( + replace(previous) + if field in ("proposals", "asec_proposals") + else previous.copy(deep=True) + ) + try: + object.__setattr__(value, field, detached) + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + value.validate() + finally: + object.__setattr__(value, field, previous) + value.validate() + + +def test_retained_source_coverage_mutation_refuses(actual): + callback = actual.values._revalidate + captured = dict( + zip( + callback.__code__.co_freevars, + (c.cell_contents for c in callback.__closure__), + strict=True, + ) + ) + coverage = captured["coverage"] + previous = coverage._body + try: + object.__setattr__(coverage, "_body", previous + b"changed") + with pytest.raises(ValueError): + actual.values.validate() + finally: + object.__setattr__(coverage, "_body", previous) + actual.values.validate() + + +def test_changed_draw_policy_cannot_reuse_an_issued_projection(actual, monkeypatch): + with monkeypatch.context() as patch: + patch.setattr(owner.hours, "SEED", owner.hours.SEED + 1) + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + actual.values.validate() + actual.values.validate() + + +@pytest.mark.parametrize("policy", ["age15_policy", "under15_policy"]) +def test_model_policies_are_explicit_before_any_source_access(policy): + options = dict( + age15_policy=owner.hours.AGE15_POLICY, under15_policy=owner.hours.UNDER15_POLICY + ) + options[policy] = None + with pytest.raises(ValueError, match="POLICY"): + owner.qualify_current_survey_hours(object(), **options) + + +@pytest.mark.parametrize( + "change,code", + [ + ("header", "SOURCE_HEADER"), + ("shape", "SOURCE_ROW_SHAPE"), + ("duplicate", "DUPLICATE_SELECTED_SOURCE_KEY"), + ("bound", "SOURCE_ROW_SHAPE"), + ], +) +def test_scanner_exhausts_and_rejects_malformed_or_duplicate_records(change, code): + columns = owner.hours.ACS_FIELDS + row = ["2024HU0000001", "1", "30", "40", "1", "0", "100", "0"] + header = list(columns) + rows = [row] + maximum = 2 + if change == "header": + header[-1] = header[-2] + elif change == "shape": + rows.append(row[:-1]) + elif change == "duplicate": + rows.append(row) + else: + maximum = 0 + raw = (",".join(header) + "\n" + "".join(",".join(r) + "\n" for r in rows)).encode() + with pytest.raises(ValueError, match=code): + owner._scan( + io.BytesIO(raw), + survey="acs", + wanted={(row[0], "1", 1)}, + selected={}, + donors={}, + maximum=maximum, + ) diff --git a/packages/microcosm-build/tests/test_us_current_survey_household_roles.py b/packages/microcosm-build/tests/test_us_current_survey_household_roles.py new file mode 100644 index 000000000..bad87234b --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_household_roles.py @@ -0,0 +1,551 @@ +"""Invented values through household-role classification and exact clone binding. + +Private fixture projections assert no source authority. Actual owners have a +separate test file and separately bounded runtime proposal. +""" + +import ast +import builtins +import hashlib +import importlib.util +import sys +from io import BytesIO +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_survey_household_roles as roles +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +DEFAULT = ( + (10, 2, "asec", 1, 1, 0), + (20, 3, "acs", 21, 2, 0), + (30, 5, "asec", 6, 0, 1), +) + + +def _table(specs=DEFAULT): + rows = [] + for _original, native, survey, code, state, reason in specs: + observation = ( + roles.asec_household_role_observation(state, reason) + if survey == "asec" + else roles.acs_household_role_observation(state, code) + ) + rows.append( + { + roles.SURVEY_COLUMN: survey, + roles.NATIVE_ID_COLUMN: native, + roles.CODE_COLUMN: code, + roles.CODE_KNOWN_COLUMN: code + in ( + roles.demographic.A_EXPRRP.named_codes + if survey == "asec" + else roles.ACS_PRINTED_RELATIONSHIP_DOMAIN + ), + roles.ALLOCATION_KNOWN_COLUMN: False, + **observation, + } + ) + return roles._observation_table(rows, [row[0] for row in specs]) + + +def _frame(people): + people = people.copy(deep=True) + tables = {"person": people} + for entity in US_SCHEMA.group_entities: + people[US_SCHEMA.membership_column(entity)] = people.person_id.to_numpy() + tables[entity] = pd.DataFrame( + {US_SCHEMA.entity_id_column(entity): np.sort(people.person_id.to_numpy())} + ) + return Frame( + tables, + US_SCHEMA, + {"household": Weights(np.ones(len(people)), WeightKind.DESIGN)}, + ) + + +def _qualified(table=None): + table = _table() if table is None else table + frame = _frame(pd.DataFrame({"person_id": table.index.to_numpy()})) + origins = table[[roles.SURVEY_COLUMN, roles.NATIVE_ID_COLUMN]].copy(deep=True) + projection = roles._projection_bytes(table) + receipt = roles.source._encode( + { + "protocol": roles.PROTOCOL, + "contract_sha256": roles.contract_sha256(), + "projection_sha256": roles._sha(projection), + **roles._counts(table), + "source_admission_issued": False, + "release_eligible": False, + }, + maximum=roles.MAX_RECEIPT_BYTES, + ) + return roles.QualifiedCurrentSurveyHouseholdRoles( + frame, origins, table, projection, receipt + ) + + +def _receiving(table, *, incumbent=None, base=0, dtype="boolean"): + pairs = [(int(original), clone) for original in table.index for clone in (0, 1)][ + ::-1 + ] + people = pd.DataFrame( + { + "person_id": np.arange(base + 1, base + 1 + len(pairs), dtype=np.int64), + roles.support_source_id_column("person"): np.array( + [p for p, _ in pairs], dtype=np.int64 + ), + roles.support_clone_index_column("person"): np.array( + [c for _, c in pairs], dtype=np.int64 + ), + roles.spine_source_id_column("person"): np.array( + [table.at[p, roles.NATIVE_ID_COLUMN] for p, _ in pairs], dtype=np.int64 + ), + roles.support_channel_column("person"): pd.array( + [table.at[p, roles.SURVEY_COLUMN] for p, _ in pairs], dtype="string" + ), + "unrelated_amount": np.arange(len(pairs), dtype=np.float64), + "census_block_geoid": pd.array( + ["060750101001001"] * len(pairs), dtype="string" + ), + } + ) + if incumbent is not None: + values = [incumbent.get(original, pd.NA) for original, _ in pairs] + people[roles.CANONICAL_COLUMN] = pd.array(values, dtype=dtype) + return SimpleNamespace(person=people) + + +@pytest.mark.parametrize( + "state,reason,head", + [ + (1, 0, True), + (2, 0, False), + (0, 1, None), + (0, 2, None), + (0, 3, None), + (0, 4, None), + ], +) +def test_asec_preserves_authenticated_owner_state_and_unknownness(state, reason, head): + result = roles.asec_household_role_observation(state, reason) + assert result[roles.VALUE_COLUMN] is head + assert result[roles.VALUE_KNOWN_COLUMN] is (head is not None) + assert result[roles.UNBOUND_REASON_COLUMN] == reason + assert result[roles.ORIGIN_COLUMN] == roles.ASEC_ORIGIN + + +@pytest.mark.parametrize("state,reason", [(1, 2), (2, 3), (0, 0), (3, 0), (-1, 1)]) +def test_asec_inconsistent_or_unnamed_owner_states_refuse(state, reason): + with pytest.raises(ValueError, match="ASEC_STATE"): + roles.asec_household_role_observation(state, reason) + + +@pytest.mark.parametrize("state,code", [(1, 21), (2, 20), (2, 37), (2, 38), (3, 20)]) +def test_acs_state_must_agree_with_named_reference_and_universe(state, code): + with pytest.raises(ValueError, match="CODE_DISAGREE"): + roles.acs_household_role_observation(state, code) + + +@pytest.mark.parametrize( + "codes,expected", + [ + ((21, 20), (2, 1)), + ((21, 22), (0, 0)), + ((20, 20), (0, 0)), + ((20, 99), (0, 0)), + ((37, 38), (3, 3)), + ((20, 37), (0, 0)), + ], +) +def test_acs_uses_whole_household_named_role_and_universe(codes, expected): + serial = "2024HU0000001" + states, _ = roles._acs_states( + {serial: {2}}, {(serial, 1): str(codes[0]), (serial, 2): str(codes[1])} + ) + # A retained subset may not discard the other published household member. + assert tuple(states[serial, line][0] for line in (1, 2)) == expected + for line, state in enumerate(expected, 1): + value = roles.acs_household_role_observation(state, codes[line - 1]) + assert value[roles.VALUE_COLUMN] is (None if state in (0, 3) else state == 1) + + +@pytest.mark.parametrize( + "token,code,known", + [ + ("20", 20, True), + ("37", 37, True), + ("38", 38, True), + ("99", 99, False), + ("", -1, False), + ("20.0", -1, False), + ("020", -1, False), + (" 20", -1, False), + ], +) +def test_relationship_literal_knownness_does_not_invent_a_role(token, code, known): + assert roles.acs_relationship_token(token) == (code, known) + + +@pytest.mark.parametrize("bad", [True, 1.0, "1", None]) +def test_owner_state_is_an_exact_integer(bad): + with pytest.raises(ValueError, match="ASEC_STATE_TYPE"): + roles.asec_household_role_observation(bad, 0) + + +def test_group_quarters_codes_remain_descriptive_without_headship(): + table = _table(((10, 2, "acs", 37, 3, 5), (20, 3, "acs", 38, 3, 5))) + assert table[roles.CODE_KNOWN_COLUMN].all() + assert not table[roles.VALUE_KNOWN_COLUMN].any() + assert table[roles.VALUE_COLUMN].isna().all() + qualified = _qualified(table) + receiving = _receiving(table) + result = roles.household_role_columns_for_population(qualified, receiving) + assert result["person", roles.CANONICAL_COLUMN].isna().all() + + +def test_exact_integer_identity_fans_to_both_clones_and_preserves_unowned_cells(): + maximum = np.iinfo(np.int64).max + table = _table( + ( + (maximum - 2, 2, "asec", 1, 1, 0), + (maximum - 1, 3, "acs", 21, 2, 0), + (maximum, 5, "asec", 6, 0, 1), + ) + ) + qualified = _qualified(table) + receiving = _receiving(table, incumbent={maximum - 1: False}, base=2**53 + 1) + before = receiving.person.copy(deep=True) + result = roles.household_role_columns_for_population(qualified, receiving)[ + "person", roles.CANONICAL_COLUMN + ] + original = before[roles.support_source_id_column("person")].to_numpy() + assert result.index.equals(pd.Index(before.person_id, name="person_id")) + assert result[original == maximum - 2].all() + assert not result[original == maximum - 1].any() + assert result[original == maximum].isna().all() + pd.testing.assert_frame_equal(before, receiving.person, check_exact=True) + + +@pytest.mark.parametrize("known", [False, True]) +def test_unsupported_incumbent_refuses_without_erasing_the_original(known): + qualified = _qualified() + receiving = _receiving(qualified.rows, incumbent={30: known}) + before = receiving.person.copy(deep=True) + with pytest.raises(ValueError, match="UNBOUND_INCUMBENT"): + roles.household_role_columns_for_population(qualified, receiving) + pd.testing.assert_frame_equal(before, receiving.person, check_exact=True) + audit = roles.household_role_reconciliation(qualified, receiving) + assert audit["incumbent_known_qualified_unbound"] == 2 + assert audit["binding_would_refuse"] + + +def test_conflicting_incumbent_refuses_and_remains_visible_to_reconciliation(): + qualified = _qualified() + receiving = _receiving(qualified.rows, incumbent={10: False}) + before = receiving.person.copy(deep=True) + with pytest.raises(ValueError, match="CANONICAL_CONFLICT"): + roles.household_role_columns_for_population(qualified, receiving) + audit = roles.household_role_reconciliation(qualified, receiving) + assert audit["conflicting_cells"] == 2 and audit["binding_would_refuse"] + pd.testing.assert_frame_equal(before, receiving.person, check_exact=True) + + +def test_nonnullable_bool_incumbent_preserves_storage_and_all_known_values(): + qualified = _qualified(_table(DEFAULT[:2])) + receiving = _receiving( + qualified.rows, incumbent={10: True, 20: False}, dtype="bool" + ) + before = receiving.person.copy(deep=True) + assert roles.canonical_dtype_token(receiving) == "bool" + result = roles.household_role_columns_for_population(qualified, receiving)[ + "person", roles.CANONICAL_COLUMN + ] + expected = before.set_index("person_id")[roles.CANONICAL_COLUMN] + pd.testing.assert_series_equal(result, expected, check_exact=True) + assert result.dtype == np.dtype("bool") + pd.testing.assert_frame_equal(before, receiving.person, check_exact=True) + + +def test_nonnullable_bool_cannot_supply_an_unsupported_source_role(): + qualified = _qualified() + receiving = _receiving( + qualified.rows, incumbent={10: True, 20: False, 30: False}, dtype="bool" + ) + before = receiving.person.copy(deep=True) + # A genuine bool column has no unknown incumbents. Its unsupported known + # cell therefore refuses before the later unresolved-bool defense. + with pytest.raises(ValueError, match="UNBOUND_INCUMBENT"): + roles.household_role_columns_for_population(qualified, receiving) + pd.testing.assert_frame_equal(before, receiving.person, check_exact=True) + + +@pytest.mark.parametrize( + "dtype,reason", + [("int64", "INCUMBENT_DTYPE$"), ("object", "INCUMBENT_DTYPE_NOT_DECLARABLE$")], +) +def test_invalid_incumbent_storage_refuses_without_boolean_coercion(dtype, reason): + qualified = _qualified(_table(DEFAULT[:2])) + receiving = _receiving(qualified.rows, incumbent={10: True, 20: False}, dtype=dtype) + before = receiving.person.copy(deep=True) + with pytest.raises(ValueError, match=reason): + roles.household_role_columns_for_population(qualified, receiving) + pd.testing.assert_frame_equal(before, receiving.person, check_exact=True) + + +def test_asec_current_published_maximum_line_is_a_coordinate_not_a_role(): + # The 2025 dictionary prints A_LINENO 01:16 (physical page 22). + origins = pd.DataFrame( + { + "source": ["asec"], + "raw_native_household_id": ["12345"], + "raw_native_person_id": ["0000000000000000000001"], + "native_line_numeric_original": ["16"], + }, + index=pd.Index([101], dtype="int64", name="person_id"), + ) + assert roles._asec_keys(origins) == {(12345, "0000000000000000000001", 16): 101} + + +@pytest.mark.parametrize( + "column", + [ + "support_source_id_person", + "support_clone_index_person", + "spine_source_id_person", + ], +) +def test_clone_identity_cannot_be_float_even_when_values_are_exact(column): + # Resolve actual shared names explicitly; no hand-maintained identity aliases. + columns = { + "support_source_id_person": roles.support_source_id_column("person"), + "support_clone_index_person": roles.support_clone_index_column("person"), + "spine_source_id_person": roles.spine_source_id_column("person"), + } + qualified = _qualified() + receiving = _receiving(qualified.rows) + receiving.person[columns[column]] = receiving.person[columns[column]].astype(float) + with pytest.raises(ValueError, match="CLONE_AXIS_DTYPE"): + roles.household_role_columns_for_population(qualified, receiving) + + +def test_public_receipt_contains_only_closed_aggregate_fields(): + qualified = _qualified() + receiving = _receiving(qualified.rows) + receipt = roles.household_role_binding_receipt( + qualified, + receiving, + receiving_version="invented.receiving", + declared_rewrite=False, + ) + assert set(receipt) == roles.PUBLIC_RECEIPT_KEYS + assert receipt["filled_cells"] == 4 and receipt["unresolved_cells"] == 2 + assert not receipt["source_admission_issued"] + assert not receipt["unallocated_observation_claim"] + assert receipt["relationship_allocation_provenance"] == "unresolved" + + +@pytest.mark.parametrize( + "name", + ["VALUE_COLUMN", "CODE_COLUMN", "ACS_OBSERVATION_YEAR", "MAX_HOUSEHOLD_MEMBERS"], +) +def test_live_contract_seals_individual_column_and_period_bindings(monkeypatch, name): + before = roles._live() + monkeypatch.setattr( + roles, name, "changed" if type(getattr(roles, name)) is str else 2099 + ) + assert roles._live() != before + + +def test_caller_projection_cannot_substitute_for_the_retained_preparation(): + with pytest.raises(ValueError, match="PREPARATION_TYPE"): + roles.qualify_current_survey_household_roles(_qualified()) + + +@pytest.mark.parametrize( + "defect,reason", + [ + ("header", "SOURCE_HEADER"), + ("duplicate", "DUPLICATE_SELECTED_ACS_ROW"), + ("line", "SOURCE_LINE"), + ("width", "SOURCE_ROW_SHAPE"), + ("count", "SOURCE_ROW_SHAPE"), + ("token", "SOURCE_TOKEN_BOUND"), + ], +) +def test_literal_acs_roster_requires_a_complete_well_formed_member(defect, reason): + serial = "2024HU0000001" + header = "SERIALNO,SPORDER,RELSHIPP" + rows = [f"{serial},1,20", f"{serial},2,25"] + maximum = 2 + if defect == "header": + header = "SERIALNO,SPORDER,SPORDER" + elif defect == "duplicate": + rows[1] = rows[0] + elif defect == "line": + rows[1] = f"{serial},0,25" + elif defect == "width": + rows[1] += ",unrequested" + elif defect == "count": + maximum = 1 + else: + rows[1] = f"{serial},2," + "2" * 65 + stream = BytesIO((header + "\n" + "\n".join(rows) + "\n").encode()) + with pytest.raises(ValueError, match=reason): + roles._scan_acs_relationships( + stream, serials={serial: {2}}, selected={}, maximum=maximum + ) + + +@pytest.mark.parametrize("defect", ["native", "survey", "clone", "original"]) +def test_clone_join_refuses_a_changed_identity_without_mutating_receiving(defect): + qualified = _qualified() + receiving = _receiving(qualified.rows) + if defect == "native": + receiving.person.loc[0, roles.spine_source_id_column("person")] = 99 + elif defect == "survey": + receiving.person.loc[0, roles.support_channel_column("person")] = "acs" + elif defect == "clone": + receiving.person.loc[0, roles.support_clone_index_column("person")] = 0 + else: + receiving.person.loc[0, roles.support_source_id_column("person")] = 99 + before = receiving.person.copy(deep=True) + with pytest.raises(ValueError, match="CLONE_"): + roles.household_role_columns_for_population(qualified, receiving) + pd.testing.assert_frame_equal(before, receiving.person, check_exact=True) + + +@pytest.mark.parametrize( + "name,expected", + [ + ( + "_UNBOUND_DEMOGRAPHICS", + "5e05f66bcf34401a3581df8abf0335ea1f34145e952dc635e64b17d21300bbda", + ), + ( + "UNBOUND_DEMOGRAPHIC_COLUMNS", + "fe968c959d177f46f0a242973f855789b37db036e2987e9981978c37ee28aaac", + ), + ], +) +def test_shared_declarations_preserve_the_original_assignment_ast(name, expected): + # Fingerprints of the two assignments in the reviewed b967 composed stage. + tree = ast.parse(Path(roles.demographic_contract.__file__).read_bytes()) + assert all(isinstance(node, (ast.Expr, ast.Assign)) for node in tree.body) + assignments = { + node.targets[0].id: node for node in tree.body if isinstance(node, ast.Assign) + } + assert set(assignments) == {"_UNBOUND_DEMOGRAPHICS", "UNBOUND_DEMOGRAPHIC_COLUMNS"} + assert hashlib.sha256(ast.dump(assignments[name]).encode()).hexdigest() == expected + + +def test_composed_stage_reexports_the_same_shared_objects_without_redeclaration(): + path = Path(roles.__file__).with_name("graph_composed_asec_binding.py") + tree = ast.parse(path.read_bytes()) + names = {"_UNBOUND_DEMOGRAPHICS", "UNBOUND_DEMOGRAPHIC_COLUMNS"} + reexports = [ + node + for node in tree.body + if isinstance(node, ast.ImportFrom) + and node.module == "survey_demographic_contract" + and node.level == 1 + ] + assert {alias.name for node in reexports for alias in node.names} == names + assert not any( + isinstance(node, ast.Name) + and isinstance(node.ctx, ast.Store) + and node.id in names + for node in ast.walk(tree) + ) + # Execute only those actual two imports, never the composed module body. + namespace = {"__package__": roles.__package__} + exec( + compile(ast.Module(body=reexports, type_ignores=[]), str(path), "exec"), + namespace, + ) + for name in names: + assert namespace[name] is getattr(roles.demographic_contract, name) + + +@pytest.mark.parametrize( + "module_name", + ["current_survey_household_roles", "graph_current_survey_household_roles"], +) +def test_actual_module_body_imports_without_the_legacy_composed_runtime( + monkeypatch, module_name +): + path = Path(roles.__file__).with_name(module_name + ".py") + alias = roles.__package__ + "._import_boundary_" + module_name + module = ModuleType(alias) + module.__package__ = roles.__package__ + module.__file__ = str(path) + original_import = builtins.__import__ + blocked = ( + "microcosm.build.us_runtime.graph_composed_asec_binding", + "microcosm.build.us_runtime.graph_composed_population", + "microcosm.build.us_runtime.stacked_spine", + "microcosm.build.us_runtime.multispine_pool", + "microcosm.build.us_runtime.spine_agreement", + "microcosm.build.spec_engine", + "jsonschema", + "policyengine_us", + "policyengine_uk", + ) + attempted = [] + + def checked_import(name, globals=None, locals=None, fromlist=(), level=0): + package = (globals or {}).get("__package__", "") + target = ( + importlib.util.resolve_name("." * level + name, package) if level else name + ) + candidates = (target, *(target + "." + item for item in fromlist or ())) + attempted.extend(candidates) + assert not any( + candidate == forbidden or candidate.startswith(forbidden + ".") + for candidate in candidates + for forbidden in blocked + ), candidates + return original_import(name, globals, locals, fromlist, level) + + monkeypatch.setitem(sys.modules, alias, module) + monkeypatch.setattr(builtins, "__import__", checked_import) + # Real source body, without loader cache probes or a child interpreter. + exec(compile(path.read_bytes(), str(path), "exec"), module.__dict__) + owner = module if module_name == "current_survey_household_roles" else module.roles + assert owner.demographic_contract is roles.demographic_contract + assert roles.demographic_contract in owner._implementation_modules() + assert not hasattr(owner, "composed") + assert attempted + + +@pytest.mark.parametrize( + "name", ["_UNBOUND_DEMOGRAPHICS", "UNBOUND_DEMOGRAPHIC_COLUMNS"] +) +def test_live_contract_binds_both_complete_shared_declarations(monkeypatch, name): + before = roles._live() + value = getattr(roles.demographic_contract, name) + # Removing the other leaf still must invalidate the complete shared binding. + monkeypatch.setattr(roles.demographic_contract, name, value[1:]) + assert roles._live() != before + + +def test_source_seal_binds_the_shared_contract_file(monkeypatch): + path = Path(roles.demographic_contract.__file__) + before = dict(roles._source_bytes()) + assert roles.demographic_contract.__name__ in before + read_bytes = Path.read_bytes + + def changed(candidate): + value = read_bytes(candidate) + return value + b"\n# invented byte mutation\n" if candidate == path else value + + monkeypatch.setattr(Path, "read_bytes", changed) + after = dict(roles._source_bytes()) + assert after.pop(roles.demographic_contract.__name__) != before.pop( + roles.demographic_contract.__name__ + ) + assert after == before diff --git a/packages/microcosm-build/tests/test_us_current_survey_household_roles_graph.py b/packages/microcosm-build/tests/test_us_current_survey_household_roles_graph.py new file mode 100644 index 000000000..3e028e1e0 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_household_roles_graph.py @@ -0,0 +1,672 @@ +"""The household-role fragment on a tiny real graph: invented values, actual executor. + +The Frame, the source bytes and the qualified values here are invented and +claim no source authority. Everything they are run through — compile_graph, +run_graph, the ContentStore, population patching and the replay comparison — +is the real machinery, so cold and required execution, artifact identity and +complete receiving-state preservation are checked against it rather than +against a stub. +""" + +import json +from dataclasses import replace +from types import SimpleNamespace + +import pytest +from test_us_current_survey_household_roles import ( + _frame, + _qualified, + _receiving, + _table, +) + +from microcosm.build.us_runtime import current_survey_household_roles as roles +from microcosm.build.us_runtime import graph_current_survey_household_roles as graph +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + ArtifactValue, + Capabilities, + ContentStore, + Determinism, + Graph, + GraphError, + KernelBase, + KernelRegistry, + KernelResult, + Node, + NodeRejectedError, + NumericScope, + Owned, + SourceRef, + StructuralDelta, + WeightTransition, + compile_graph, + run_graph, +) +from microcosm.graph import population as populations + +RECEIVING = "invented.receiving" +ALLOCATED = "invented.allocated" +ORDERING_TYPE = ArtifactType("invented.household_roles_ordering", 1) + + +def _ordering_edge(producer=RECEIVING): + return ArtifactInput("ordering", producer, "ordering", ORDERING_TYPE) + + +def _invented_source(receiving, people): + class InventedSource(KernelBase): + ref = "invented.household_roles_receiving@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return "a" * 64 # Test-only fixed producer for the invented Frame. + + def run(self, context): + return KernelResult( + frame=receiving, artifacts={"ordering": b"invented ordering input"} + ) + + node = Node( + RECEIVING, + InventedSource.ref, + sources=(graph.SOURCE_NAME,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned("person", column, graph._dtype(people[column])) + for column in people + if column != "person_id" + ), + artifact_outputs=(ArtifactOutput("ordering", ORDERING_TYPE),), + ) + return node, InventedSource() + + +def _graph_case(tmp_path, incumbent=None, *, owner_mutation=None): + qualified = _qualified() + people = _receiving(qualified.rows, incumbent=incumbent).person + receiving = _frame(people) + create, source_kernel = _invented_source(receiving, people) + after = _ordering_edge() + state = SimpleNamespace(calls=[]) + expected_seal = graph.household_role_projection_seal(qualified) + + def current(): + assert graph.household_role_projection_seal(qualified) == expected_seal + state.calls.append(True) + if state.mutate is not None: + state.mutate(len(state.calls), state) + + state.mutate = owner_mutation + state.current = current + + nodes, kernels = graph.current_survey_household_roles_kernels( + qualified, + receiving, + receiving_version=create.id, + host_pins={}, + after=after, + host_edges=(), + require_current=current, + ) + state.kernels = kernels + compiled = compile_graph( + Graph( + "us", + (SourceRef(graph.SOURCE_NAME, "raw-bytes-v1"),), + (create, *nodes), + ) + ) + registry = KernelRegistry() + registry.register(source_kernel) + for kernel in kernels: + registry.register(kernel) + path = tmp_path / "invented-source.txt" + path.write_bytes(b"invented household-role fixture only") + store = ContentStore(tmp_path / "store") + arguments = { + "sources": {graph.SOURCE_NAME: path}, + "store": store, + "kernels": registry, + } + cold = run_graph(compiled, **arguments) + observed = {} + by_node = {} + + def capture(node_id, population): + observed[population.version] = population + by_node[node_id] = population + + warm = run_graph( + compiled, **arguments, resume="require", _population_observer=capture + ) + return SimpleNamespace( + qualified=qualified, + people=people, + receiving=receiving, + by_node=by_node, + nodes=nodes, + compiled=compiled, + cold=cold, + warm=warm, + store=store, + calls=state.calls, + observed=observed, + ) + + +def test_actual_graph_cold_and_required_bind_both_clones_and_keep_everything_else( + tmp_path, +): + import pandas as pd + + case = _graph_case(tmp_path) + assert len(case.compiled.order) == 3 + assert case.cold.key == case.warm.key + assert all(receipt.hit for receipt in case.warm.nodes.values()) + # Before and after each of the two cold fragment nodes; replay invokes none. + assert len(case.calls) == 4 + output = case.warm.population(RECEIVING) + pd.testing.assert_frame_equal( + output.person[case.receiving.person.columns], + case.receiving.person, + check_exact=True, + ) + column = output.person[roles.CANONICAL_COLUMN] + assert str(column.dtype) == "boolean" + original = output.person[roles.support_source_id_column("person")].to_numpy() + values = column.to_numpy(na_value=pd.NA) + assert set(values[original == 10]) == {True} + assert set(values[original == 20]) == {False} + assert all(value is pd.NA for value in values[original == 30]) + projection = case.warm.population(graph.SOURCE_NODE) + assert set(roles.COLUMNS) <= set(projection.person.columns) + assert roles.CANONICAL_COLUMN not in projection.person + + +def test_private_codes_stay_in_the_typed_artifact_and_the_public_receipt_is_closed( + tmp_path, +): + case = _graph_case(tmp_path) + binding = case.warm.node(graph.BIND_NODE) + document = json.loads(case.store.load_bytes(binding.opaque_artifacts["binding"])) + assert set(document) == roles.PUBLIC_RECEIPT_KEYS + assert document["relationship_allocation_provenance"] == "unresolved" + assert document["equivalent_allocation_evidence_claim"] is False + assert document["source_admission_issued"] is False + assert document["release_eligible"] is False + projection = case.warm.node(graph.SOURCE_NODE) + payload = case.store.load_bytes(projection.opaque_artifacts["projection"]) + assert payload == case.qualified.projection + # The rowwise codes live only in the private typed artifact. + assert roles.CODE_COLUMN in payload.decode() + assert roles.CODE_COLUMN not in json.dumps(document) + + +def test_actual_graph_independent_complete_reconstruction(tmp_path): + case = _graph_case(tmp_path) + expected = {RECEIVING: populations.Population.from_frame(case.receiving, RECEIVING)} + for node_id in case.compiled.order: + if node_id == RECEIVING: + continue + node = case.compiled.graph.node(node_id) + artifacts = {} + for edge in node.artifact_inputs: + producer = case.warm.node(edge.producer) + key = producer.opaque_artifacts[edge.artifact] + artifacts[edge.name] = ArtifactValue( + case.store.load_bytes(key), + edge.type, + key, + producer.key, + NumericScope(), + ) + version = case.compiled.versions[node_id] + expected[version] = graph.expected_survey_household_roles_population( + node_id, + expected.get(version), + qualified=case.qualified, + node=node, + artifacts=artifacts, + ) + for version, population in expected.items(): + graph.replay.same_replayed_population(population, case.observed[version]) + + +def test_materialized_verifier_accepts_the_run_and_refuses_a_wrong_declaration( + tmp_path, +): + case = _graph_case(tmp_path) + node = case.compiled.graph.node(graph.BIND_NODE) + # The population the bind node actually received, not a rebuilt lookalike. + incoming = case.by_node[RECEIVING] + receipt = graph.verify_materialized_survey_household_roles( + case.observed[RECEIVING], incoming, qualified=case.qualified, node=node + ) + assert set(receipt) == roles.PUBLIC_RECEIPT_KEYS + assert receipt["declared_rewrite"] is False + assert receipt["filled_cells"] == 4 and receipt["unresolved_cells"] == 2 + # A declaration that disagrees with the population it is applied to would + # let a create silently overwrite an incumbent leaf; it must refuse here. + wrong = replace(node, params={**dict(node.params), "declared_rewrite": True}) + with pytest.raises(ValueError, match="RECEIVING_DECLARATION"): + graph.verify_materialized_survey_household_roles( + case.observed[RECEIVING], incoming, qualified=case.qualified, node=wrong + ) + + +def test_projection_artifact_bytes_cannot_be_replaced(): + qualified = _qualified() + receiving = _frame(_receiving(qualified.rows).person) + bind = graph.current_survey_household_roles_nodes( + qualified, + receiving, + receiving_version="country.receiving", + host_pins={}, + after=_ordering_edge("country.amounts"), + host_edges=(), + )[1] + artifacts = { + graph.PROJECTION_ALIAS: ArtifactValue( + b"{}", graph.PROJECTION_TYPE, "0" * 64, "1" * 64, NumericScope() + ), + "ordering": ArtifactValue( + b"x", ORDERING_TYPE, "2" * 64, "3" * 64, NumericScope() + ), + } + with pytest.raises(ValueError, match="ARTIFACT_PAYLOAD"): + graph._check_artifacts(bind, artifacts, qualified, {}) + with pytest.raises(ValueError, match="ARTIFACT_ROSTER"): + graph._check_artifacts(bind, {}, qualified, {}) + + +def test_host_edge_pins_are_checked_against_the_incoming_artifact(): + qualified = _qualified() + receiving = _frame(_receiving(qualified.rows).person) + edge = ArtifactInput( + "preparation", "survey_population.create", "preparation", ArtifactType("p", 2) + ) + pins = { + "preparation": { + "producer_key": "a" * 64, + "artifact_key": "b" * 64, + "payload_sha256": "c" * 64, + } + } + projection = graph.current_survey_household_roles_nodes( + qualified, + receiving, + receiving_version="country.receiving", + host_pins=pins, + after=_ordering_edge("country.amounts"), + host_edges=(edge,), + )[0] + assert edge in projection.artifact_inputs + artifacts = { + "ordering": ArtifactValue( + b"x", ORDERING_TYPE, "2" * 64, "3" * 64, NumericScope() + ), + "preparation": ArtifactValue( + b"payload", edge.type, "b" * 64, "a" * 64, NumericScope() + ), + } + with pytest.raises(ValueError, match="HOST_EDGE_PIN"): + graph._check_artifacts(projection, artifacts, qualified, pins) + + +def test_qualified_mutation_changes_the_seal_without_touching_a_fresh_value(): + qualified = _qualified() + fresh = _qualified() + seal = graph.household_role_projection_seal(qualified) + assert graph.household_role_projection_seal(fresh) == seal + qualified.rows.loc[10, roles.CODE_COLUMN] = 0 + with pytest.raises(ValueError, match="PROJECTION_BINDING"): + graph.household_role_projection_seal(qualified) + assert graph.household_role_projection_seal(fresh) == seal + + +def test_declared_rewrite_preserves_known_incumbents_through_the_real_patch(): + """The rewrite path goes through the executor's own patch, not a private writer.""" + import pandas as pd + + qualified = _qualified() + people = _receiving(qualified.rows, incumbent={20: False}).person + receiving = _frame(people) + node = graph.current_survey_household_roles_nodes( + qualified, + receiving, + receiving_version=RECEIVING, + host_pins={}, + after=_ordering_edge("country.amounts"), + host_edges=(), + )[1] + assert node.outputs[0].rewrite and node.outputs[0].dtype == "boolean" + incoming = populations.Population.from_frame(receiving, RECEIVING) + result = graph._result(qualified, node, receiving.person) + patched = populations.patch(incoming, node, result) + column = patched.frame.person[roles.CANONICAL_COLUMN] + original = patched.frame.person[roles.support_source_id_column("person")].to_numpy() + values = column.to_numpy(na_value=pd.NA) + assert set(values[original == 20]) == {False} # Incumbent preserved. + assert set(values[original == 10]) == {True} # Source-known null filled. + assert all(value is pd.NA for value in values[original == 30]) + assert patched.owners[("person", roles.CANONICAL_COLUMN)] == graph.BIND_NODE + # The owned leaf is the one column this node is allowed to change; every + # other column of every entity must survive the rewrite untouched. + for entity in receiving.entities: + before = receiving.table(entity) + unowned = [c for c in before.columns if c != roles.CANONICAL_COLUMN] + pd.testing.assert_frame_equal( + patched.frame.table(entity)[unowned], before[unowned], check_exact=True + ) + assert list(patched.frame.person.columns) == list(receiving.person.columns) + + +def test_a_rewrite_needs_a_receiving_version_with_a_base(): + """A host that adopts the rewrite must open a version below its CREATE node.""" + qualified = _qualified() + people = _receiving(qualified.rows, incumbent={20: False}).person + receiving = _frame(people) + create, _kernel = _invented_source(receiving, people) + nodes = graph.current_survey_household_roles_nodes( + qualified, + receiving, + receiving_version=create.id, + host_pins={}, + after=_ordering_edge(), + host_edges=(), + ) + assert nodes[1].outputs[0].rewrite + with pytest.raises(GraphError, match="rewrite"): + compile_graph( + Graph( + "us", + (SourceRef(graph.SOURCE_NAME, "raw-bytes-v1"),), + (create, *nodes), + ) + ) + + +def test_country_host_edges_name_the_real_typed_ancestry(): + edges = graph.country_host_edges() + assert {edge.name for edge in edges} == { + "preparation", + "allocation", + "frame_context", + } + assert all(type(edge) is ArtifactInput for edge in edges) + assert {edge.producer for edge in edges} == { + "survey_population.create", + "survey_population.allocate", + } + + +def test_projection_branch_keeps_the_original_support_and_adds_only_source_columns(): + qualified = _qualified() + projected = graph._projection_frame(qualified) + assert type(projected) is Frame + person = projected.person + assert set(roles.COLUMNS) <= set(person.columns) + assert roles.CANONICAL_COLUMN not in person + assert person.person_id.tolist() == qualified.rows.index.tolist() + for entity in qualified.source_frame.entities: + assert set(projected.table(entity).columns) <= set( + qualified.source_frame.table(entity).columns + ) | set(roles.COLUMNS) + + +def _rewrite_graph_case(tmp_path): + """A receiving version opened below the CREATE, carrying an incumbent leaf. + + A rewrite needs a version with a base to carry the incumbent from, so the + invented host opens one with an ordinary REWEIGHT that conserves mass and + changes no cell. That is the shape a country host has after its own + allocation, clone and geography stages. + """ + qualified = _qualified() + people = _receiving(qualified.rows, incumbent={20: False}).person + receiving = _frame(people) + create, source_kernel = _invented_source(receiving, people) + allocated = Weights( + receiving.weights_for("household").values.copy(), WeightKind.IMPORTANCE + ) + + class InventedAllocation(KernelBase): + ref = "invented.household_roles_allocate@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def implementation_hash(self): + return "b" * 64 # Test-only fixed producer for the invented version. + + def run(self, context): + return KernelResult(weights=allocated) + + allocate = Node( + ALLOCATED, + InventedAllocation.ref, + structural=StructuralDelta.REWEIGHT, + base=create.id, + weights=WeightTransition("household", "importance", "conserve"), + mass="conserve", + ) + after = _ordering_edge() + state = SimpleNamespace(calls=[]) + expected_seal = graph.household_role_projection_seal(qualified) + + def current(): + assert graph.household_role_projection_seal(qualified) == expected_seal + state.calls.append(True) + + nodes, kernels = graph.current_survey_household_roles_kernels( + qualified, + receiving, + receiving_version=allocate.id, + host_pins={}, + after=after, + host_edges=(), + require_current=current, + ) + compiled = compile_graph( + Graph( + "us", + (SourceRef(graph.SOURCE_NAME, "raw-bytes-v1"),), + (create, allocate, *nodes), + ) + ) + registry = KernelRegistry() + registry.register(source_kernel) + registry.register(InventedAllocation()) + for kernel in kernels: + registry.register(kernel) + path = tmp_path / "invented-source.txt" + path.write_bytes(b"invented household-role fixture only") + store = ContentStore(tmp_path / "store") + arguments = { + "sources": {graph.SOURCE_NAME: path}, + "store": store, + "kernels": registry, + } + cold = run_graph(compiled, **arguments) + observed = {} + by_node = {} + + def capture(node_id, population): + observed[population.version] = population + by_node[node_id] = population + + warm = run_graph( + compiled, **arguments, resume="require", _population_observer=capture + ) + return SimpleNamespace( + qualified=qualified, + receiving=receiving, + by_node=by_node, + nodes=nodes, + compiled=compiled, + cold=cold, + warm=warm, + store=store, + calls=state.calls, + observed=observed, + ) + + +def test_actual_graph_rewrite_preserves_incumbents_cold_and_required(tmp_path): + import pandas as pd + + case = _rewrite_graph_case(tmp_path) + assert len(case.compiled.order) == 4 + assert case.nodes[1].population == ALLOCATED + assert case.nodes[1].outputs[0].rewrite + assert case.nodes[1].outputs[0].dtype == "boolean" + assert case.cold.key == case.warm.key + assert all(receipt.hit for receipt in case.warm.nodes.values()) + assert len(case.calls) == 4 + output = case.warm.population(ALLOCATED) + column = output.person[roles.CANONICAL_COLUMN] + assert str(column.dtype) == "boolean" + original = output.person[roles.support_source_id_column("person")].to_numpy() + assert column[original == 20].tolist() == [False, False] # Incumbent preserved. + assert column[original == 10].tolist() == [True, True] # Known null filled. + assert column[original == 30].isna().tolist() == [True, True] + unowned = [c for c in case.receiving.person.columns if c != roles.CANONICAL_COLUMN] + pd.testing.assert_frame_equal( + output.person[unowned], case.receiving.person[unowned], check_exact=True + ) + assert list(output.person.columns) == list(case.receiving.person.columns) + binding = case.warm.node(graph.BIND_NODE) + document = json.loads(case.store.load_bytes(binding.opaque_artifacts["binding"])) + assert document["declared_rewrite"] is True + assert document["receiving_version"] == ALLOCATED + assert document["preserved_cells"] == 2 + assert document["filled_cells"] == 2 + assert document["unresolved_cells"] == 2 + assert set(document) == roles.PUBLIC_RECEIPT_KEYS + + +def test_rewrite_run_reconstructs_and_refuses_a_conflicting_incumbent(tmp_path): + case = _rewrite_graph_case(tmp_path) + node = case.compiled.graph.node(graph.BIND_NODE) + incoming = case.by_node[ALLOCATED] + receipt = graph.verify_materialized_survey_household_roles( + case.observed[ALLOCATED], incoming, qualified=case.qualified, node=node + ) + assert receipt["preserved_cells"] == 2 and receipt["declared_rewrite"] is True + conflicting = _qualified( + _table( + ( + (10, 2, "asec", 1, 1, 0), + (20, 3, "acs", 20, 1, 0), + (30, 5, "asec", 6, 0, 1), + ) + ) + ) + with pytest.raises(ValueError, match="CANONICAL_CONFLICT"): + graph.verify_materialized_survey_household_roles( + case.observed[ALLOCATED], incoming, qualified=conflicting, node=node + ) + + +@pytest.mark.parametrize("callback_number", [1, 2, 3, 4]) +@pytest.mark.parametrize( + "mutation", + [ + "callable", + "code", + "defaults", + "constant", + "shared_private", + "shared_public", + "graph_callable", + "graph_constant", + "callback_identity", + "callback_defaults", + "callback_closure", + ], +) +def test_first_and_final_owner_callbacks_cannot_change_live_implementation( + tmp_path, monkeypatch, callback_number, mutation +): + """First/final callbacks on each node cannot change executable behavior.""" + + def replacement(*args, **kwargs): + raise AssertionError("changed binder must not execute") + + restored_cells = [] + + def mutate(number, state): + if number != callback_number: + return + if mutation == "callable": + monkeypatch.setattr(roles, "_bind", replacement) + elif mutation == "code": + monkeypatch.setattr(roles._bind, "__code__", replacement.__code__) + elif mutation == "defaults": + monkeypatch.setattr(roles._bind, "__defaults__", (None,)) + elif mutation == "constant": + monkeypatch.setattr(roles, "ACS_OBSERVATION_YEAR", 2099) + elif mutation in {"shared_private", "shared_public"}: + name = ( + "_UNBOUND_DEMOGRAPHICS" + if mutation == "shared_private" + else "UNBOUND_DEMOGRAPHIC_COLUMNS" + ) + monkeypatch.setattr( + roles.demographic_contract, + name, + getattr(roles.demographic_contract, name)[1:], + ) + elif mutation == "graph_callable": + monkeypatch.setattr(graph, "_result", replacement) + elif mutation == "graph_constant": + monkeypatch.setattr(graph, "MAX_ARTIFACT_BYTES", 1) + elif mutation == "callback_identity": + state.kernels[(number - 1) // 2].require_current = lambda: None + elif mutation == "callback_defaults": + monkeypatch.setattr(state.current, "__defaults__", (None,)) + else: + freevars = dict( + zip( + state.current.__code__.co_freevars, + state.current.__closure__, + strict=True, + ) + ) + cell = freevars["expected_seal"] + restored_cells.append((cell, cell.cell_contents)) + cell.cell_contents = ("changed closure",) + + try: + with pytest.raises( + NodeRejectedError, match="IMPLEMENTATION_CHANGED|CALLBACK_CHANGED" + ): + _graph_case(tmp_path, owner_mutation=mutate) + finally: + for cell, value in restored_cells: + cell.cell_contents = value + + +def test_opaque_callable_owner_cannot_claim_an_executable_callback_seal(): + class Owner: + def __call__(self): + raise AssertionError("unsealed owner must not execute") + + qualified = _qualified() + receiving = _frame(_receiving(qualified.rows).person) + with pytest.raises(ValueError, match="HOST_CALLBACK"): + graph.current_survey_household_roles_kernels( + qualified, + receiving, + receiving_version=RECEIVING, + host_pins={}, + after=_ordering_edge(), + host_edges=(), + require_current=Owner(), + ) diff --git a/packages/microcosm-build/tests/test_us_current_survey_household_roles_source.py b/packages/microcosm-build/tests/test_us_current_survey_household_roles_source.py new file mode 100644 index 000000000..f5b6b1e02 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_household_roles_source.py @@ -0,0 +1,236 @@ +"""Actual retained source owners over the maintained tiny invented fixture. + +These cases require a separate bounded runtime proposal. They replace only +fixture pins before issuance and never substitute an issuer or checked accessor. +""" + +import json +import sys + +import pytest +from test_us_current_asec_demographics import _demographic_arguments + +from microcosm.build.us_runtime import current_survey_household_roles as roles + + +@pytest.mark.parametrize( + "defect,reason", + [ + ("native", "ASEC_NATIVE_JOIN"), + ("line", "ASEC_COORDINATE_DISAGREE"), + ("household", "ASEC_COORDINATE_DISAGREE"), + ], +) +def test_actual_asec_owner_rejects_detached_mismatched_join_coordinates( + tmp_path, monkeypatch, defect, reason +): + prepared = roles.source.prepare_authenticated_survey_population( + **_demographic_arguments(tmp_path, monkeypatch) + ) + entry = prepared._checked() + state = entry[2] + before = roles.source._frame_identity(state.frame) + origins = roles._origins(state.frame, json.loads(entry[1])) + keys = roles._asec_keys(origins) + observed, _ = roles._asec_observed(state, prepared) + expected = roles._asec_selected(origins, keys, observed) + changed = origins.copy(deep=True) + row = changed.index[changed.source.eq("asec")][0] + if defect == "native": + changed.loc[row, "native_person_id"] = ( + int(observed.array("person_id").max()) + 1 + ) + elif defect == "line": + old = int(changed.loc[row, "native_line_numeric_original"]) + changed.loc[row, "native_line_numeric_original"] = "2" if old == 1 else "1" + else: + old = int(changed.loc[row, "raw_native_household_id"]) + changed.loc[row, "raw_native_household_id"] = "2" if old == 1 else "1" + # Only a detached consumer coordinate is changed. The real preparation, + # owner buffers, member bytes and issuance accessors remain untouched. + with pytest.raises(ValueError, match=reason): + roles._asec_selected(changed, keys, observed) + assert roles._asec_selected(origins, keys, observed) == expected + observed.validate() + assert prepared._checked() is entry + assert roles.source._frame_identity(state.frame) == before + + +@pytest.mark.parametrize( + "defect,reason", + [ + ("household", "ACS_ROSTER_HOUSEHOLD_SCOPE"), + ("line", "ACS_ROSTER_COVERAGE"), + ], +) +def test_actual_acs_roster_rejects_detached_unpublished_retained_scope( + tmp_path, monkeypatch, defect, reason +): + prepared = roles.source.prepare_authenticated_survey_population( + **_demographic_arguments(tmp_path, monkeypatch) + ) + entry = prepared._checked() + state = entry[2] + before = roles.source._frame_identity(state.frame) + origins = roles._origins(state.frame, json.loads(entry[1])) + _, serials = roles._acs_keys(origins) + roster, owner = roles._acs_roster(state, serials) + expected = roles._acs_states(serials, roster) + changed = {serial: set(lines) for serial, lines in serials.items()} + if defect == "household": + invented = "2024HU9999999" + assert invented not in changed + changed[invented] = {1} + else: + serial = sorted(changed)[0] + absent = next( + line + for line in range(1, roles.MAX_HOUSEHOLD_MEMBERS + 1) + if (serial, line) not in roster + ) + changed[serial].add(absent) + # The retained scope is the adversarial input; the roster still comes from + # the real pinned catalogue and no issuer/classifier is substituted. + with pytest.raises(ValueError, match=reason): + roles._acs_states(changed, roster) + assert roles._acs_states(serials, roster) == expected + assert roles.source.acs_catalogue._lookup(state.catalogues[0]) is owner + assert prepared._checked() is entry + assert roles.source._frame_identity(state.frame) == before + + +def test_actual_sources_qualify_named_reference_roles_and_leave_gq_unbound( + tmp_path, monkeypatch +): + prepared = roles.source.prepare_authenticated_survey_population( + **_demographic_arguments(tmp_path, monkeypatch) + ) + before = roles.source._frame_identity(prepared.checked_view().frame) + qualified = roles.qualify_current_survey_household_roles(prepared) + table = qualified.rows + acs = table.loc[table[roles.SURVEY_COLUMN].eq("acs")] + asec = table.loc[table[roles.SURVEY_COLUMN].eq("asec")] + assert len(acs) == 5 and len(asec) == 4 + assert acs[roles.CODE_KNOWN_COLUMN].all() and asec[roles.CODE_KNOWN_COLUMN].all() + assert asec[roles.VALUE_KNOWN_COLUMN].all() + assert int(asec[roles.VALUE_COLUMN].sum()) == 2 + gq = acs.loc[acs[roles.CODE_COLUMN].isin((37, 38))] + housing = acs.loc[~acs.index.isin(gq.index)] + assert len(gq) == 2 and gq[roles.VALUE_COLUMN].isna().all() + assert not gq[roles.VALUE_KNOWN_COLUMN].any() + assert gq[roles.UNBOUND_REASON_COLUMN].eq(5).all() + assert housing[roles.VALUE_KNOWN_COLUMN].all() + assert int(housing[roles.VALUE_COLUMN].sum()) == 2 + assert not table[roles.ALLOCATION_KNOWN_COLUMN].any() + assert roles.source._frame_identity(qualified.source_frame) == before + assert roles.source._frame_identity(prepared.checked_view().frame) == before + fresh = roles.qualify_current_survey_household_roles(prepared) + assert roles.household_role_projection_seal( + fresh + ) == roles.household_role_projection_seal(qualified) + + +@pytest.mark.parametrize("borrow_number", [1, 2]) +@pytest.mark.parametrize( + "mutation", + [ + "callable", + "code", + "defaults", + "period", + "column", + "owner_code", + "shared_private", + "shared_public", + ], +) +def test_first_and_final_preparation_borrows_refuse_live_changes( + tmp_path, monkeypatch, borrow_number, mutation +): + prepared = roles.source.prepare_authenticated_survey_population( + **_demographic_arguments(tmp_path, monkeypatch) + ) + checked = type(prepared)._checked + checked_code = checked.__code__ + qualify_code = roles.qualify_current_survey_household_roles.__code__ + seen, fired = [], [] + + def replacement(*args, **kwargs): + raise AssertionError("mutated executable must not run") + + def profile(frame, event, arg): + if ( + event != "return" + or frame.f_code is not checked_code + or frame.f_back is None + or frame.f_back.f_code is not qualify_code + ): + return + seen.append(True) + if len(seen) != borrow_number: + return + fired.append(True) + if mutation == "callable": + monkeypatch.setattr(roles, "_project", replacement) + elif mutation == "code": + monkeypatch.setattr(roles._project, "__code__", replacement.__code__) + elif mutation == "defaults": + monkeypatch.setattr(roles._project, "__defaults__", (None,)) + elif mutation == "period": + monkeypatch.setattr(roles, "ACS_OBSERVATION_YEAR", 2099) + elif mutation == "column": + monkeypatch.setattr(roles, "VALUE_COLUMN", "unreviewed_role") + elif mutation in {"shared_private", "shared_public"}: + name = ( + "_UNBOUND_DEMOGRAPHICS" + if mutation == "shared_private" + else "UNBOUND_DEMOGRAPHIC_COLUMNS" + ) + monkeypatch.setattr( + roles.demographic_contract, + name, + getattr(roles.demographic_contract, name)[1:], + ) + else: + monkeypatch.setattr(checked, "__code__", replacement.__code__) + + previous = sys.getprofile() + sys.setprofile(profile) + try: + with pytest.raises( + ValueError, match="CURRENT_SURVEY_HOUSEHOLD_ROLES_IMPLEMENTATION_CHANGED" + ): + roles.qualify_current_survey_household_roles(prepared) + finally: + sys.setprofile(previous) + assert len(seen) == borrow_number and fired == [True] + + +def test_last_acs_owner_callback_cannot_change_a_column_alias(tmp_path, monkeypatch): + prepared = roles.source.prepare_authenticated_survey_population( + **_demographic_arguments(tmp_path, monkeypatch) + ) + lookup_code = roles.source.acs_catalogue._lookup.__code__ + qualify_code = roles.qualify_current_survey_household_roles.__code__ + fired = [] + + def profile(frame, event, arg): + if ( + event == "return" + and frame.f_code is lookup_code + and frame.f_back is not None + and frame.f_back.f_code is qualify_code + ): + fired.append(True) + monkeypatch.setattr(roles, "CODE_COLUMN", "unreviewed_source_code") + + previous = sys.getprofile() + sys.setprofile(profile) + try: + with pytest.raises( + ValueError, match="CURRENT_SURVEY_HOUSEHOLD_ROLES_IMPLEMENTATION_CHANGED" + ): + roles.qualify_current_survey_household_roles(prepared) + finally: + sys.setprofile(previous) + assert fired == [True] diff --git a/packages/microcosm-build/tests/test_us_current_survey_housing.py b/packages/microcosm-build/tests/test_us_current_survey_housing.py new file mode 100644 index 000000000..f3e78f0be --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_housing.py @@ -0,0 +1,521 @@ +"""Invented housing sources and household donors; no native data or engine.""" + +import hashlib +import io +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_survey_housing as housing +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + + +def add_housing_source_fields(arguments, patch): + """Add original interview observations before real fixture issuers run.""" + from microcosm.build.us_runtime import native_household_origin as origin + + path = arguments["source_dir"] / "asec" / "hhpub25.csv" + rows = pd.read_csv(path, dtype=str, keep_default_na=False) + rows["H_TENURE"] = "2" + rows["HPUBLIC"] = ["1" if i == 0 else "2" for i in range(len(rows))] + rows["HLORENT"] = ["0" if i == 0 else "2" for i in range(len(rows))] + rows["I_HPUBLI"] = "0" + rows["I_HLOREN"] = "0" + rows.to_csv(path, index=False) + payload = path.read_bytes() + patch.setattr( + origin, + "_ASEC_MEMBER_PINS", + tuple( + replace( + pin, + member_sha256=hashlib.sha256(payload).hexdigest(), + size_bytes=len(payload), + rows=len(rows), + ) + for pin in origin._ASEC_MEMBER_PINS + ), + ) + return arguments + + +def _invented(): + hids = np.array([10, 20, 30, 40, 50, 60], dtype=np.int64) + ids = np.array([11, 12, 13, 21, 31, 41, 51, 61], dtype=np.int64) + hh = np.array([10, 10, 10, 20, 30, 40, 50, 60], dtype=np.int64) + spm = np.array([100, 100, 101, 200, 300, 400, 500, 600], dtype=np.int64) + sources = ["asec"] * 6 + ["acs"] * 2 + person = pd.DataFrame( + { + "person_id": ids, + "person_household_id": hh, + "person_spm_unit_id": spm, + "person_tax_unit_id": spm, + "person_family_id": spm, + "person_marital_unit_id": spm, + } + ) + tables = { + "person": person, + "household": pd.DataFrame({"household_id": hids}), + **{ + e: pd.DataFrame({e + "_id": np.unique(spm)}) + for e in US_SCHEMA.group_entities + if e != "household" + }, + } + for entity, table in tables.items(): + entity_ids = table[entity + "_id"].to_numpy() + table[housing.provenance.support_source_id_column(entity)] = entity_ids + table[housing.provenance.spine_source_id_column(entity)] = entity_ids + 10000 + table[housing.provenance.support_clone_index_column(entity)] = np.zeros( + len(table), dtype="int64" + ) + if entity == "person": + channel = sources + elif entity == "household": + channel = ["asec"] * 4 + ["acs"] * 2 + else: + channel = ["asec"] * 5 + ["acs"] * 2 + table[housing.provenance.support_channel_column(entity)] = pd.array( + channel, dtype="string" + ) + frame = Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.array([100.0, 1.0, 3.0, 2.0, 5.0, 4.0]), WeightKind.DESIGN + ) + }, + pd.Series(["invented"] * len(person), dtype="string"), + ) + origins = pd.DataFrame( + { + "source": ["asec"] * 4 + ["acs"] * 2, + "source_year": [2024] * 6, + "survey_year": [2025] * 4 + [2024] * 2, + "raw_native_id": ["1", "2", "3", "4", "2024HU0000005", "2024GQ0000006"], + "selected_receiving_household_id": hids + 10000, + # Source preparation preserves exact rational design anchors. + "original_anchor": [[100, 1], [1, 1], [3, 1], [2, 1], [5, 1], [4, 1]], + }, + index=pd.Index(hids, name="household_id"), + ) + porigins = pd.DataFrame(index=pd.Index(ids, name="person_id")) + selected = { + k: {} for k in ("asec_person", "acs_person", "asec_household", "acs_household") + } + keys = {} + for index, pid in enumerate(ids): + source = sources[index] + hid = int(hh[index]) + line = index + 1 if hid == 10 else 1 + if source == "asec": + key = (hid // 10, str(pid).zfill(22), line) + selected["asec_person"][key] = { + "PERIDNUM": key[1], + "PH_SEQ": str(key[0]), + "A_LINENO": str(line), + "A_EXPRRP": "1" if line == 1 else "5", + "P_SEQ": str(line), + } + else: + serial = origins.loc[hid, "raw_native_id"] + key = (serial, "1", 1) + selected["acs_person"][key] = { + "SERIALNO": serial, + "SPORDER": "1", + "RELSHIPP": "20" if hid == 50 else "37", + } + keys[source, key] = pid + for hid in hids[:4]: + number = int(hid // 10) + selected["asec_household"][number] = { + "H_SEQ": str(number), + "H_TENURE": "1" if hid == 40 else "2", + "H_HHTYPE": "1", + "HRHTYPE": "1", + "H_LIVQRT": "1", + "HPUBLIC": {10: "1", 20: "2", 30: "0", 40: "0"}[int(hid)], + "HLORENT": "2" if hid == 20 else "0", + "I_HPUBLI": "0", + "I_HLOREN": "0", + } + for hid, kind, tenure in ((50, "1", "3"), (60, "2", "")): + serial = origins.loc[hid, "raw_native_id"] + selected["acs_household"][serial] = { + "SERIALNO": serial, + "TYPEHUGQ": kind, + "NP": "1", + "TEN": tenure, + } + pfeatures = pd.DataFrame( + { + housing.predictors.FEATURES[0]: [50, 10, 30, 40, 60, 35, 55, 70], + housing.predictors.FEATURES[1]: [100, 0, 50, 200, 300, 1000, 0, 0], + housing.predictors.FEATURES[2]: [0] * 8, + }, + index=porigins.index, + dtype="float64", + ) + return frame, origins, porigins, keys, selected, pfeatures + + +def _qualified(): + frame, origins, porigins, keys, selected, features = _invented() + native = housing._source_values(frame, origins, porigins, keys, selected) + return housing._qualified_values(frame, origins, native, features, {}) + + +@pytest.mark.parametrize("storage", ["python", "auto"]) +def test_source_strings_use_canonical_storage_for_scalar_updates(storage): + with pd.option_context("mode.string_storage", storage): + frame, origins, porigins, keys, selected, _ = _invented() + native = housing._source_values(frame, origins, porigins, keys, selected) + string_columns = native.select_dtypes(include="string") + assert not string_columns.empty + for dtype in string_columns.dtypes: + assert dtype == housing.population_ops.dtype_for_token("string") + assert dtype.storage == "python" + + +def test_source_classification_preserves_order_independent_exact_reference_ids(): + frame, origins, porigins, keys, selected, _ = _invented() + expected = housing._source_values(frame, origins, porigins, keys, selected) + # These identities cannot be represented exactly by float64. Reorder both + # source axes so batch classification must join by identity, not row order. + offset = 2**54 + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["person"]["person_id"] += offset + tables["person"] = tables["person"].iloc[::-1].reset_index(drop=True) + reordered = Frame( + tables, + frame.schema, + dict(frame._weights), + frame.strata.iloc[::-1].reset_index(drop=True), + ) + porigins.index += offset + porigins = porigins.iloc[::-1] + keys = {key: pid + offset for key, pid in reversed(list(keys.items()))} + actual = housing._source_values( + reordered, origins.iloc[::-1], porigins, keys, selected + ) + occupied = expected.housing_source_head_person_id.ge(0) + expected.loc[occupied, "housing_source_head_person_id"] += offset + pd.testing.assert_frame_equal(actual, expected.iloc[::-1]) + + +def _cloned(frame): + tables = {} + for entity in frame.entities: + parts = [] + for clone in (1, 0): + part = frame.table(entity).copy(deep=True) + part[entity + "_id"] += clone * 1000 + part[housing.provenance.support_clone_index_column(entity)] = clone + if entity == "person": + for group in US_SCHEMA.group_entities: + part["person_" + group + "_id"] += clone * 1000 + parts.append(part) + tables[entity] = ( + pd.concat(parts, ignore_index=True).iloc[::-1].reset_index(drop=True) + ) + if entity != "person": + tables[entity] = ( + tables[entity].sort_values(entity + "_id").reset_index(drop=True) + ) + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.ones(len(tables["household"])), WeightKind.IMPORTANCE + ) + }, + pd.Series(["invented"] * len(tables["person"]), dtype="string"), + ) + + +def test_known_unknown_and_explicit_model_exclusions_are_distinct(): + q = _qualified() + assert q.native.housing_observed_receipt.tolist()[:2] == [True, False] + assert q.native.housing_observed_receipt.iloc[2:].isna().all() + assert q.native.housing_observed_receipt__known.tolist() == [ + True, + True, + False, + False, + False, + False, + ] + assert q.native.loc[40, "housing_receipt__origin"] == "modeled_owner_exclusion" + assert ( + q.native.loc[60, "housing_receipt__origin"] + == "modeled_group_quarters_exclusion" + ) + assert housing.model_input.decode_recipient_matrix( + q.matrix + ).features.index.tolist() == [30, 50] + assert q.native.loc[60, "housing_source_head_person_id"] == -1 + + +def test_household_donors_preserve_design_mass_and_aggregate_members_once(): + q = _qualified() + assert q.donor_frame.table("household").household_id.tolist() == [10, 20] + assert q.keep.tolist() == [True, True, True, True, False, False, False, False] + assert q.donor_frame.weights_for("household").values.tolist() == [100, 1] + assert q.donor_frame.weights_for("household").kind is WeightKind.DESIGN + assert q.donor_columns[housing.TARGET].tolist() == [1.0, 0.0] + assert q.donor_columns.loc[10, housing.FEATURES[1]] == 3 + assert q.donor_columns.loc[10, housing.FEATURES[2]] == 150 + assert q.evidence["donor_households"] == 2 and q.evidence["donor_persons"] == 4 + + +def test_source_householder_role_never_uses_sequence_as_authority(): + frame, origins, porigins, keys, selected, _ = _invented() + selected["asec_person"][(1, str(11).zfill(22), 1)]["P_SEQ"] = "2" + selected["asec_person"][(1, str(12).zfill(22), 2)]["P_SEQ"] = "1" + native = housing._source_values(frame, origins, porigins, keys, selected) + assert native.loc[10, "housing_source_head_person_id"] == 11 + + +def test_origin_seal_preserves_and_checks_exact_rational_metadata(): + q = _qualified() + before = housing.seal(q) + assert q.origins.loc[10, "original_anchor"] == [100, 1] + q.origins.loc[10, "original_anchor"][1] = 3 + assert housing.seal(q) != before + q.origins.loc[10, "original_anchor"][1] = 1 + assert housing.seal(q) == before + changed = replace(q, origins=q.origins.rename_axis("wrong_identity")) + assert housing.seal(changed) != before + + +def test_source_range_index_materializes_exact_household_ids_for_the_model(): + frame, origins, porigins, keys, selected, features = _invented() + origins.index = pd.RangeIndex(10, 70, 10, name="household_id") + native = housing._source_values(frame, origins, porigins, keys, selected) + q = housing._qualified_values(frame, origins, native, features, {}) + matrix = housing.model_input.decode_recipient_matrix(q.matrix) + assert type(matrix.features.index) is pd.Index + assert matrix.features.index.dtype == np.dtype("int64") + assert matrix.features.index.tolist() == [30, 50] + assert isinstance(q.origins.index, pd.RangeIndex) + origins.index = origins.index.astype("float64") + with pytest.raises(ValueError, match="ORIGIN_HOUSEHOLD_INDEX"): + housing._qualified_values(frame, origins, native, features, {}) + + +def test_unnamed_asec_relationship_refuses_even_with_one_valid_head(): + frame, origins, porigins, keys, selected, _ = _invented() + selected["asec_person"][(1, str(12).zfill(22), 2)]["A_EXPRRP"] = "6" + with pytest.raises(ValueError, match="UNNAMED_RELATIONSHIP"): + housing._source_values(frame, origins, porigins, keys, selected) + + +@pytest.mark.parametrize( + "defect", + ("owner_positive", "unknown_tenure", "noninterview", "missing_head", "gq_head"), +) +def test_unresolved_universe_and_invalid_reference_roles_refuse(defect): + frame, origins, porigins, keys, selected, _ = _invented() + if defect == "owner_positive": + selected["asec_household"][4]["HPUBLIC"] = "1" + elif defect == "unknown_tenure": + selected["asec_household"][3]["H_TENURE"] = "0" + elif defect == "noninterview": + selected["asec_household"][3]["H_HHTYPE"] = "2" + elif defect == "missing_head": + selected["asec_person"][(1, str(11).zfill(22), 1)]["A_EXPRRP"] = "5" + else: + selected["acs_person"][("2024GQ0000006", "1", 1)]["RELSHIPP"] = "20" + with pytest.raises(ValueError, match="CURRENT_SURVEY_HOUSING_"): + housing._source_values(frame, origins, porigins, keys, selected) + + +def test_clone_attachment_shares_draws_and_routes_only_householder_spm(): + q = _qualified() + receiving = _cloned(q.source_frame) + parent_stamp = housing.source._frame_identity(receiving) + index = housing.model_input.decode_recipient_matrix(q.matrix).features.index + draw = pd.DataFrame({housing.TARGET: [1.0, 0.0]}, index=index) + result = housing.attach_columns(q, receiving, draw) + for hid in (10, 1010, 30, 1030): + assert result["household", "housing_receipt"].loc[hid] + for sid in (100, 1100, 300, 1300): + assert result["spm_unit", housing.SPM_OUTPUTS[0]].loc[sid] + for sid in (101, 1101, 600, 1600): + assert not result["spm_unit", housing.SPM_OUTPUTS[0]].loc[sid] + assert pd.isna(result["household", "housing_observed_receipt"].loc[30]) + assert not result["household", "housing_observed_receipt__known"].loc[1030] + assert housing.source._frame_identity(receiving) == parent_stamp + assert not {"housing_assistance", "spm_unit_housing_subsidy"} & { + c for _, c in result + } + declared = housing.output_columns(q, receiving) + assert {(o.entity, o.column) for o in declared} == set(result) + + +@pytest.mark.parametrize( + "defect", ("fractional", "axis", "clone", "native", "collision") +) +def test_attachment_refuses_invalid_draw_or_clone_identity(defect): + q = _qualified() + receiving = _cloned(q.source_frame) + index = housing.model_input.decode_recipient_matrix(q.matrix).features.index + draw = pd.DataFrame({housing.TARGET: [1.0, 0.0]}, index=index) + if defect == "fractional": + draw.iloc[0, 0] = 0.5 + elif defect == "axis": + draw = draw.iloc[::-1] + elif defect in ("clone", "native"): + col = ( + housing.provenance.support_clone_index_column("household") + if defect == "clone" + else housing.provenance.spine_source_id_column("household") + ) + receiving.table("household").loc[0, col] = 9 + else: + receiving.table("spm_unit")[housing.SPM_OUTPUTS[0]] = False + with pytest.raises(ValueError, match="CURRENT_SURVEY_HOUSING_"): + housing.attach_columns(q, receiving, draw) + + +def test_no_recipient_requires_no_donor_model(): + frame, origins, porigins, keys, selected, features = _invented() + selected["asec_household"][3].update(HPUBLIC="2", HLORENT="2") + selected["acs_household"]["2024HU0000005"]["TEN"] = "1" + native = housing._source_values(frame, origins, porigins, keys, selected) + q = housing._qualified_values(frame, origins, native, features, {}) + assert q.matrix is None and q.donor_frame is None + assert housing.attach_columns(q, _cloned(frame), None) + + +def test_qualifier_live_seal_catches_mapper_and_constant_changes(monkeypatch): + before = housing.live() + monkeypatch.setattr(housing.participation, "route_participation", lambda *_: None) + assert housing.live() != before + monkeypatch.undo() + before = housing.live() + monkeypatch.setattr(housing, "OWNER_ASSUMPTION", ("A5", "changed")) + assert housing.live() != before + + +@pytest.mark.parametrize("mutation", ("mapper", "constant")) +def test_owner_callback_cannot_change_live_mapper_or_constants(monkeypatch, mutation): + class ChangedBorrower: + def _checked(self): + if mutation == "mapper": + monkeypatch.setattr( + housing.participation, "observed_participation", lambda *_: None + ) + else: + monkeypatch.setattr(housing, "OWNER_ASSUMPTION", ("A5", "changed")) + # No source state is supplied: the entry fence must reject the + # changed callback before consuming any alleged owner result. + return None + + with pytest.raises(ValueError, match="CODE_CHANGED_DURING_OWNER"): + housing._original_columns(ChangedBorrower()) + + +def test_attachment_checks_live_configuration_after_routing_callback(monkeypatch): + q = _qualified() + receiving = _cloned(q.source_frame) + index = housing.model_input.decode_recipient_matrix(q.matrix).features.index + draw = pd.DataFrame({housing.TARGET: [1.0, 0.0]}, index=index) + route = housing.participation.route_participation + + def changed_route(*args): + result = route(*args) + monkeypatch.setattr(housing, "OWNER_ASSUMPTION", ("A5", "changed")) + return result + + monkeypatch.setattr(housing.participation, "route_participation", changed_route) + with pytest.raises(ValueError, match="ATTACH_CODE_OR_VALUES_CHANGED"): + housing.attach_columns(q, receiving, draw) + + +def test_partial_observed_head_column_remains_unchanged(): + q = _qualified() + receiving = _cloned(q.source_frame) + receiving.person["is_household_head"] = pd.array( + [pd.NA] * len(receiving.person), dtype="boolean" + ) + gq = receiving.person.person_household_id.isin((60, 1060)) + receiving.person.loc[gq, "is_household_head"] = False + before = receiving.person.is_household_head.copy() + index = housing.model_input.decode_recipient_matrix(q.matrix).features.index + housing.attach_columns( + q, receiving, pd.DataFrame({housing.TARGET: [1.0, 0.0]}, index=index) + ) + pd.testing.assert_series_equal(receiving.person.is_household_head, before) + + +def test_bounded_literal_scan_refuses_duplicate_and_exhausts_unselected_rows(): + payload = b"H_SEQ,HPUBLIC\n1,1\n2,0\n" + rows, count = housing._scan( + io.BytesIO(payload), + ("H_SEQ", "HPUBLIC"), + key=housing._hh_key, + wanted={1}, + maximum=2, + ) + assert count == 2 and rows[1]["HPUBLIC"] == "1" + with pytest.raises(ValueError, match="LITERAL_DUPLICATE"): + housing._scan( + io.BytesIO(payload + b"01,2\n"), + ("H_SEQ", "HPUBLIC"), + key=housing._hh_key, + wanted={1}, + maximum=3, + ) + + +def test_actual_original_member_qualifier_has_spm_membership_and_named_heads( + tmp_path, monkeypatch +): + from test_us_current_asec_demographics import _demographic_arguments + + arguments = add_housing_source_fields( + _demographic_arguments(tmp_path, monkeypatch, zero=False), monkeypatch + ) + prepared = housing.source.prepare_authenticated_survey_population(**arguments) + frame, origins, porigins, keys, selected, evidence = housing._original_columns( + prepared + ) + assert "spm_unit" in frame.entities and "person_spm_unit_id" in frame.person + assert "is_household_head" not in frame.person + assert housing.source.LEGACY_HEAD_COLUMN in frame.person + assert ( + frame.person.loc[ + frame.person.person_support_channel.eq("asec"), + housing.source.LEGACY_HEAD_COLUMN, + ] + .isna() + .all() + ) + native = housing._source_values(frame, origins, porigins, keys, selected) + asec = origins.source.eq("asec") + assert ( + native.loc[asec, "housing_observed_receipt"].tolist() + == native.loc[asec, "housing_source_HPUBLIC"].eq("1").tolist() + ) + assert sorted(native.loc[asec, "housing_observed_receipt"].tolist()) == [ + False, + True, + ] + assert native.loc[asec, "housing_observed_receipt__known"].all() + assert ( + native.loc[ + native.housing_participation_universe.eq("occupied_housing_unit"), + "housing_source_head_person_id", + ] + .ge(0) + .all() + ) + assert evidence["asec_household_member_sha256"] + prepared.checked_view() diff --git a/packages/microcosm-build/tests/test_us_current_survey_immigration_source.py b/packages/microcosm-build/tests/test_us_current_survey_immigration_source.py new file mode 100644 index 000000000..42aa9140e --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_immigration_source.py @@ -0,0 +1,364 @@ +"""Genuine source issuers on invented bytes, never a rules engine or real data.""" + +import hashlib +import io +import json +import shutil +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_survey_immigration_source as owner + + +def source_arguments(tmp_path, monkeypatch): + import test_us_child_property_income_source_owner as fixture + import test_us_survey_population_preparation as preparation_fixture + + from microcosm.build.us_runtime import asec_person_income_source as restoration + + original_person = preparation_fixture._person + + def person(*args, **kwargs): + row = original_person(*args, **kwargs) + row.update(CIT="1", POBP="001", YOEP="") + return row + + monkeypatch.setattr(preparation_fixture, "_person", person) + full, partial, donor = fixture.source_arguments(tmp_path, monkeypatch) + folder = full["source_dir"] / "asec" + pins, paths = [], {} + for year, member, archive, *_ in owner.original.asec._MEMBER_PINS: + path = folder / member + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in owner.ASEC_VALUE_COLUMNS: + if name not in raw: + raw[name] = "0" + raw["PRCITSHP"] = "1" + raw["A_LFSR"] = " 7 " + raw.loc[raw.A_AGE.map(int).lt(16), "A_LFSR"] = "0" + raw["SPM_CAPHOUSESUB"] = "0.00" + raw.to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + shutil.copyfile(path, partial["source_dir"] / "asec" / member) + for module in (owner.original.asec, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + restored = tmp_path / "immigration-restored-money" + restoration.restore_asec_person_income_source( + folder / "parent.h5", + folder / "household-attachment.h5", + member_paths=paths, + output_dir=restored, + ) + for arguments in (full, partial): + shutil.copyfile( + restored / restoration.CHECKPOINT_FILENAME, + arguments["source_dir"] / "asec" / "person-income-attachment.h5", + ) + return full, partial, donor + + +@pytest.fixture(scope="module") +def actual(tmp_path_factory): + with pytest.MonkeyPatch.context() as patch: + full_args, partial_args, donor = source_arguments( + tmp_path_factory.mktemp("immigration-source"), patch + ) + full = owner.source.prepare_authenticated_survey_population(**full_args) + partial = owner.source.prepare_authenticated_survey_population(**partial_args) + full_seal = owner.source._frame_identity(full._checked()[2].frame) + partial_seal = owner.source._frame_identity(partial._checked()[2].frame) + yield SimpleNamespace( + full=full, + partial=partial, + donor=donor, + values=owner.qualify_current_survey_immigration(full), + smaller=owner.qualify_current_survey_immigration(partial), + ) + assert owner.source._frame_identity(full._checked()[2].frame) == full_seal + assert owner.source._frame_identity(partial._checked()[2].frame) == partial_seal + + +def test_complete_current_asec_evidence_survives_support_selection(actual): + left, right = actual.values, actual.smaller + left.validate() + right.validate() + pd.testing.assert_frame_equal(left.asec_full_raw, right.asec_full_raw) + assert len(right.asec_selected_raw) < len(right.asec_full_raw) + assert str(actual.donor - 100).zfill(22) in right.asec_full_raw.PERIDNUM.tolist() + # The deliberately unselected donor is age 15, leaving one adult. + assert right.asec_full_raw.A_LFSR.tolist().count(" 7 ") == 1 + assert not {"WSAL_VAL", "SEMP_VAL", "person_weight"} & set(left.asec_full_raw) + evidence = json.loads(left.receipt) + assert evidence["asec_income_year"] == 2024 + assert evidence["asec_observation_year"] == 2025 + assert evidence["acs_observation_year"] == 2024 + assert evidence["status_assignment_performed"] is False + assert evidence["national_stock_alignment_qualified"] is False + assert evidence["rules_head"] == "7ee36ac1bea125218912996b1030b77a208db963" + + +def test_owning_literal_blank_and_nonowning_missing_remain_distinct(actual): + value = actual.values + assert value.person_evidence.index.equals(value.origins.index) + acs = value.origins.source.eq("acs") + asec = ~acs + assert value.person_evidence.loc[acs, owner.PREFIX + "YOEP"].eq("").all() + assert value.person_evidence.loc[asec, owner.PREFIX + "YOEP"].isna().all() + assert value.person_evidence.loc[acs, owner.PREFIX + "A_LFSR"].isna().all() + assert "immigration_status_str" not in value.person_evidence + pd.testing.assert_frame_equal( + actual.smaller.person_evidence, + value.person_evidence.loc[actual.smaller.origins.index], + ) + + +def receiving_people(value): + import test_us_current_survey_hours_source as hours_fixture + + return hours_fixture.receiving_people(value) + + +def test_clone_evidence_is_exact_and_does_not_assign_status(actual): + people = receiving_people(actual.values) + before = people.copy(deep=True) + columns = owner.borrow_cloned_immigration_evidence(actual.values, people) + for (_, name), values in columns.items(): + source_ids = people[ + owner.attachment.provenance.support_source_id_column("person") + ] + expected = ( + actual.values.person_evidence[name] + .reindex(source_ids) + .reset_index(drop=True) + ) + pd.testing.assert_series_equal( + values.reset_index(drop=True), expected, check_names=False + ) + pd.testing.assert_frame_equal(people, before) + assert all(name.startswith(owner.PREFIX) for _, name in columns) + + +@pytest.mark.parametrize( + "defect", ["source", "native", "clone", "channel", "collision"] +) +def test_clone_evidence_refuses_bad_ancestry_and_overwrite(actual, defect): + people = receiving_people(actual.values) + p = owner.attachment.provenance + if defect == "collision": + people[owner.PREFIX + "A_LFSR"] = "7" + elif defect == "channel": + people.loc[0, p.support_channel_column("person")] = "foreign" + else: + column = { + "source": p.support_source_id_column("person"), + "native": p.spine_source_id_column("person"), + "clone": p.support_clone_index_column("person"), + }[defect] + people.loc[0, column] = 99999 + with pytest.raises(ValueError): + owner.borrow_cloned_immigration_evidence(actual.values, people) + + +@pytest.mark.parametrize( + "field,column", + [ + ("origins", "source"), + ("asec_full_raw", "A_LFSR"), + ("asec_selected_raw", "SPM_CAPHOUSESUB"), + ("acs_raw", "YOEP"), + ("person_evidence", "immigration_source_A_LFSR"), + ], +) +def test_changed_projection_refuses(actual, field, column): + table = getattr(actual.values, field) + pid, previous = table.index[0], table.iloc[0][column] + try: + table.at[pid, column] = "changed" + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + actual.values.validate() + finally: + table.at[pid, column] = previous + actual.values.validate() + + +@pytest.mark.parametrize( + "field", + ["origins", "asec_full_raw", "asec_selected_raw", "acs_raw", "person_evidence"], +) +def test_equal_detached_table_is_not_retained_identity(actual, field): + previous = getattr(actual.values, field) + try: + object.__setattr__(actual.values, field, previous.copy(deep=True)) + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + actual.values.validate() + finally: + object.__setattr__(actual.values, field, previous) + + +def test_copy_or_forged_callback_cannot_issue_source_owner(actual): + with pytest.raises(ValueError, match="PROJECTION_OBJECT_CHANGED"): + replace(actual.values).validate() + with pytest.raises(ValueError, match="RETAINED_OWNER_REQUIRED"): + replace(actual.values, _revalidate=lambda _: None).validate() + + +def test_unissued_preparation_refuses(actual): + fake = object.__new__(type(actual.full)) + object.__setattr__(fake, "payload", actual.full.payload) + with pytest.raises(ValueError): + owner.qualify_current_survey_immigration(fake) + + +def test_readset_matches_the_reviewed_successor_interface_without_income_proxies(): + from microcosm.build.us_runtime import immigration + + assert set(owner.ASEC_VALUE_COLUMNS) == ( + set(immigration.US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS) + - {"WSAL_VAL", "SEMP_VAL"} + ) | {"A_LFSR"} + + +@pytest.mark.parametrize("which", ["receiving", "borrowed"]) +def test_final_owner_io_cannot_change_transport_output(actual, monkeypatch, which): + people = receiving_people(actual.values) + validation = owner.QualifiedSurveyImmigrationSources.validate + attach = owner.attachment.attach_columns + observed = SimpleNamespace(calls=0, columns={}) + + def capture(*args): + columns = attach(*args) + observed.columns.update(columns) + return columns + + def validate(value): + validation(value) + observed.calls += 1 + if observed.calls == 2: + if which == "receiving": + people.loc[0, "person_id"] += 1 + else: + observed.columns["person", owner.PREFIX + "A_LFSR"].iloc[0] = "changed" + + with monkeypatch.context() as patch: + patch.setattr(owner.attachment, "attach_columns", capture) + patch.setattr(owner.QualifiedSurveyImmigrationSources, "validate", validate) + patch.setattr(owner, "_LIVE", owner._live()) + with pytest.raises( + ValueError, match="RECEIVING_CHANGED|BORROWED_COLUMNS_CHANGED" + ): + owner.borrow_cloned_immigration_evidence(actual.values, people) + actual.values.validate() + + +def test_final_io_cannot_mutate_a_projection(actual, monkeypatch): + value = actual.values + table, pid = value.asec_full_raw, value.asec_full_raw.index[0] + previous, read_bytes, triggered = table.at[pid, "A_LFSR"], Path.read_bytes, [] + + def mutate(path): + result = read_bytes(path) + if not triggered: + triggered.append(True) + table.at[pid, "A_LFSR"] = "4" + return result + + try: + with monkeypatch.context() as patch: + patch.setattr(Path, "read_bytes", mutate) + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + value.validate() + assert triggered + finally: + table.at[pid, "A_LFSR"] = previous + value.validate() + + +@pytest.mark.parametrize("helper", ["_series_digest", "_index_identity"]) +@pytest.mark.parametrize("warm_epoch", [False, True]) +def test_preparation_rejects_replaced_physical_helpers_even_when_digest_is_masked( + actual, monkeypatch, helper, warm_epoch +): + from contextlib import nullcontext + + value = actual.values + table = value.asec_full_raw + expected = owner._table_seal(table) + original_index = table.index.copy() + original_series = table.A_LFSR.copy(deep=True) + series_digest = owner.physical._series_digest + index_identity = owner.physical._index_identity + # The authority is transitive: preparation -> ASEC native -> parent_owner. + assert owner.physical in owner.source._modules() + assert (owner.physical.__name__, helper) in owner.source._LIVE + + def hide_series(digest, series): + if series.name == "A_LFSR" and series.index.equals(original_index): + series = original_series + return series_digest(digest, series) + + def hide_index(index): + if index.name == "source_person_id": + return index_identity(original_index) + return index_identity(index) + + epoch = owner.source.verification_epoch() if warm_epoch else nullcontext() + with epoch: + if warm_epoch: + actual.full._checked() + try: + with monkeypatch.context() as patch: + if helper == "_series_digest": + table.loc[table.index[0], "A_LFSR"] = "changed" + patch.setattr(owner.physical, helper, hide_series) + else: + table.index = table.index + 100000 + patch.setattr(owner.physical, helper, hide_index) + # The attack actually hides the projection change from this + # table digest. The retained owner's live-function seal must + # still reject it, including when a borrow was already warm. + assert owner._table_seal(table) == expected + with pytest.raises(ValueError, match="FINAL_AUTHORITY_CHANGED"): + value.validate() + finally: + table.index = original_index + table["A_LFSR"] = original_series + value.validate() + + +@pytest.mark.parametrize("bad", ["missing_header", "duplicate_key", "short_row"]) +def test_literal_scanner_refuses_incomplete_or_duplicate_source(bad): + row = dict.fromkeys(owner.ASEC_COLUMNS, "0") + row.update(PERIDNUM="0000000000000000000001", PH_SEQ="7", A_LINENO="1") + columns = owner.ASEC_COLUMNS + if bad == "missing_header": + columns = tuple(c for c in columns if c != "A_LFSR") + values = [row[c] for c in columns] + if bad == "short_row": + values.pop() + text = ",".join(columns) + "\n" + ",".join(values) + "\n" + if bad == "duplicate_key": + text += ",".join(values) + "\n" + with pytest.raises(ValueError): + owner._scan( + io.BytesIO(text.encode()), + survey="asec", + wanted=set(), + selected={}, + complete={}, + maximum=2, + ) diff --git a/packages/microcosm-build/tests/test_us_current_survey_immigration_transfer.py b/packages/microcosm-build/tests/test_us_current_survey_immigration_transfer.py new file mode 100644 index 000000000..38e9862b8 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_immigration_transfer.py @@ -0,0 +1,564 @@ +"""Original allocation authority and paired transfer on invented sources.""" + +import hashlib +import json +import os +import shutil +import time +from dataclasses import replace +from fractions import Fraction +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_survey_immigration_transfer as owner + + +def _phase(name): + if os.environ.get("MICROCOSM_INVENTED_PHASES") == "1": + print( + "IMMIGRATION_PHASE " + + json.dumps( + { + "phase": name, + "cpu_s": time.process_time(), + "wall_s": time.monotonic(), + } + ), + flush=True, + ) + + +def test_fit_implementation_uses_canonical_class_and_refuses_rebound_cache(monkeypatch): + owner._require_fit_implementation() + with monkeypatch.context() as patch: + patch.setattr(owner.transfer, "QRF", object()) + with pytest.raises(ValueError, match="FIT_IMPLEMENTATION_CHANGED"): + owner._require_fit_implementation() + + +def test_temporary_lineage_text_preserves_exact_draw_keys_and_serialization(): + from microcosm.build.serialization_dtypes import canonicalize_table_string_dtypes + + original = pd.DataFrame( + { + "source_year": [2024, 2024], + "source_household_id": pd.Series( + [2**100 + 1, "2024HU0000001"], dtype=object + ), + "source_person_id": pd.Series( + ["0001234567890123456789", "001"], dtype=object + ), + } + ) + projected = original.copy(deep=True) + for name in ("source_household_id", "source_person_id"): + projected[name] = owner._lineage_text(original[name]) + assert projected.source_household_id.tolist() == [str(2**100 + 1), "2024HU0000001"] + assert projected.source_person_id.tolist() == original.source_person_id.tolist() + np.testing.assert_array_equal( + owner.rules._stable_person_draws(original, seed=42, salt="lineage-contract"), + owner.rules._stable_person_draws(projected, seed=42, salt="lineage-contract"), + ) + canonical = canonicalize_table_string_dtypes( + projected, boundary="invented lineage projection", table_name="person" + ) + # The serializer may normalize the missing-value string dtype convention; + # it must accept the declared string columns and preserve every key value. + pd.testing.assert_frame_equal(canonical, projected, check_dtype=False) + assert original.source_household_id.iloc[0] == 2**100 + 1 + assert type(original.source_household_id.iloc[0]) is int + raw = {"PH_SEQ": "01234", "PERIDNUM": "0" * 19 + "123", "A_LINENO": "01"} + retained = dict(raw) + native_key = owner.assignment_owner.donor_owner.literals.original._key(raw, "asec") + assert native_key == (1234, retained["PERIDNUM"], 1) + assert owner._lineage_text([native_key[0]])[0] == "1234" + assert owner._lineage_text([native_key[1]])[0] == retained["PERIDNUM"] + assert raw == retained + assert ( + owner.assignment_owner.donor_owner.literals.original._key(raw, "asec") + == native_key + ) + + +@pytest.mark.parametrize("value", [1.0, True, pd.NA, ""]) +def test_temporary_lineage_text_refuses_ambiguous_scalars(value): + with pytest.raises(ValueError, match="LINEAGE_SCALAR"): + owner._lineage_text([value]) + + +def test_unissued_parent_refuses_before_sources_or_fit(): + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN"): + owner.transfer_current_survey_immigration( + object(), object(), object(), seed=42, n_estimators=2 + ) + + +@pytest.mark.parametrize("seed", [-1, True, 2**64, 1.0]) +def test_invalid_seed_refuses_before_sources_or_fit(seed): + with pytest.raises(ValueError, match="SEED"): + owner.transfer_current_survey_immigration( + object(), object(), object(), seed=seed, n_estimators=2 + ) + + +@pytest.mark.parametrize("trees", [0, True, -1, 1.5]) +def test_invalid_forest_refuses_before_sources_or_fit(trees): + with pytest.raises(ValueError, match="TREES"): + owner.transfer_current_survey_immigration( + object(), object(), object(), seed=42, n_estimators=trees + ) + + +def test_copied_owner_is_not_issued(): + value = owner.CurrentSurveyImmigrationTransfer(None, None, b"{}") + with pytest.raises(ValueError, match="ISSUED_OWNER_REQUIRED"): + replace(value).validate() + + +def source_arguments(root, patch): + """Compose literal sources before all real graph/source issuers run.""" + import test_us_graph_atomic_completion_host as completion + import test_us_survey_population_preparation as preparation + + from microcosm.build.us_runtime import asec_person_income_source as restoration + from microcosm.build.us_runtime import current_asec_demographics as demographics + from microcosm.build.us_runtime import current_survey_person_status_source as status + + original_person = preparation._person + + def person(*args, **kwargs): + row = original_person(*args, **kwargs) + row.update( + CIT="1", POBP="001", YOEP="", SEX="1" if int(row["SPORDER"]) == 1 else "2" + ) + if row["SERIALNO"] == "2024HU0000001" and int(row["SPORDER"]) == 1: + row.update(CIT="5", POBP="303", YOEP="1980") + return row + + patch.setattr(preparation, "_person", person) + call, _ = completion._source_arguments(root, patch) + call["fraction"] = Fraction(1) + request = call["source_dir"] / "selection-request.json" + document = json.loads(request.read_bytes()) + document["fraction"] = [1, 1] + request.write_bytes( + json.dumps(document, sort_keys=True, separators=(",", ":")).encode() + ) + folder = call["source_dir"] / "asec" + paths, pins = {}, [] + for ( + year, + member, + archive, + *_, + ) in owner.assignment_owner.donor_owner.literals.original.asec._MEMBER_PINS: + path = folder / member + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in owner.assignment_owner.donor_owner.literals.ASEC_VALUE_COLUMNS: + if name not in raw: + raw[name] = "0" + raw["PRCITSHP"] = "1" + raw["PENATVTY"] = "057" + raw["A_MARITL"] = "7" + raw["A_LFSR"] = raw.A_AGE.map(lambda age: "7" if int(age) >= 16 else "0") + raw["PEAFEVER"] = raw.A_AGE.map(lambda age: "2" if int(age) >= 17 else "-1") + raw["SPM_CAPHOUSESUB"] = "0" + for name in ("MCARE", "CAID", "IHSFLG", "CHAMPVA", "MIL"): + raw[name] = "2" + if year == 2024: + adult = raw.A_AGE.eq("55") + assert adult.sum() == 1 + raw.loc[adult, ["PRCITSHP", "PENATVTY", "PEINUSYR"]] = ["5", "303", "1"] + raw.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(raw), + len(data), + ) + ) + paths[year] = path + for module in ( + owner.assignment_owner.donor_owner.literals.original.asec, + restoration, + demographics.demographic, + status.student, + ): + patch.setattr(module, "_MEMBER_PINS", tuple(pins)) + restored = root / "immigration-restored" + restoration.restore_asec_person_income_source( + folder / "parent.h5", + folder / "household-attachment.h5", + member_paths=paths, + output_dir=restored, + ) + shutil.copyfile( + restored / restoration.CHECKPOINT_FILENAME, + folder / "person-income-attachment.h5", + ) + return call + + +def zero_fixture_controls(patch): + """Fixture-only stage bytes; original parser/issuers/guards stay active. + + Five invented recipients cannot represent national humanitarian stocks. Set + only those fixture targets to zero, retain citations and undocumented + controls, and intercept every resource read consistently through validation. + These bytes never change the maintained resource or production defaults. + """ + read = Path.read_text + + def replace_targets(value): + if isinstance(value, dict): + return { + k: 0 if k == "target" else replace_targets(v) for k, v in value.items() + } + return value + + def fixture_text(path, *args, **kwargs): + text = read(path, *args, **kwargs) + if path.name == "source_stages.json": + doc = json.loads(text) + + def visit(value): + if isinstance(value, dict): + for key, child in value.items(): + if key == "humanitarian_status_stocks": + value[key] = replace_targets(child) + else: + visit(child) + elif isinstance(value, list): + for child in value: + visit(child) + + visit(doc) + return json.dumps(doc) + return text + + patch.setattr(Path, "read_text", fixture_text) + + +@pytest.fixture(scope="module") +def actual(tmp_path_factory): + with pytest.MonkeyPatch.context() as patch: + root = tmp_path_factory.mktemp("original-immigration") + _phase("sources.before") + call = source_arguments(root, patch) + _phase("sources.after") + _phase("full51.before") + run = owner.host.run_atomic_survey_financial(**call) + _phase("full51.after") + original_stamp = owner.allocation._population_stamp( + run.prefix.allocated_population + ) + zero_fixture_controls(patch) + _phase("asec_donor.before") + donor = owner.assignment_owner.donor_owner.borrow_full_asec_immigration_donor( + run.prefix.preparation + ) + _phase("asec_donor.after") + _phase("asec_assignment.before") + assigned = owner.assignment_owner.assign_full_asec_immigration(donor, seed=42) + _phase("asec_assignment.after") + _phase("acs_projection.before") + acs = owner.acs_owner.borrow_current_acs_immigration_projection( + run.prefix.preparation + ) + _phase("acs_projection.after") + bank_root = root / "paired-draw-bank" + _phase("original_transfer.before") + result = owner.transfer_current_survey_immigration( + run, assigned, acs, seed=42, n_estimators=2, bank_root=bank_root + ) + _phase("original_transfer.after") + yield SimpleNamespace( + root=root, + run=run, + donor=donor, + assigned=assigned, + acs=acs, + result=result, + call=call, + original_stamp=original_stamp, + bank_root=bank_root, + ) + _phase("postchecks.before") + assert ( + owner.allocation._population_stamp(run.prefix.allocated_population) + == original_stamp + ) + _phase("postchecks.after") + + +def test_genuine_original_transfer_preserves_weights_sources_and_asec(actual): + from microcosm.frame import WeightKind + + result, original = actual.result, actual.run.prefix.allocated_population.frame + _phase("positive_validate.before") + result.validate() + _phase("positive_validate.after") + receipt = json.loads(result.receipt) + assert actual.donor.frame.weights_for("household").kind is WeightKind.DESIGN + assert result.frame.weights_for("household").kind is WeightKind.IMPORTANCE + assert result.frame.weights_for("household") is original.weights_for("household") + assert result.frame.metadata == original.metadata + assert result.frame.mass_log == original.mass_log + for entity in original.entities: + table = result.frame.table(entity) + if entity == "person": + table = table.drop(columns=list(owner.OUTPUTS)) + pd.testing.assert_frame_equal(table, original.table(entity)) + assert set(result.pairs.columns) == set(owner.OUTPUTS) + assert result.pairs.notna().all().all() + origins = owner.acs_owner._RETAINED[id(actual.acs)][2].qualified.origins + assert receipt["asec_immutable_rows"] == int(origins.source.eq("asec").sum()) == 2 + assert receipt["acs_imputed_rows"] == int(origins.source.eq("acs").sum()) == 3 + # The full donor includes rows outside this receiving native spine. + assert len(actual.donor.frame.person) == 4 + assert receipt["national_stock_alignment_qualified"] is False + assert receipt["release_eligible"] is False + assert all(r["reconciliation"] is not None for r in receipt["imputed_inputs"]) + for pid, origin in origins.loc[origins.source.eq("asec")].iterrows(): + np.testing.assert_array_equal( + result.pairs.loc[pid].to_numpy(), + actual.assigned.pairs.loc[ + origin.native_person_id, list(owner.OUTPUTS) + ].to_numpy(), + ) + + +def test_exact_row_years_masks_predictors(actual): + origins = owner.acs_owner._RETAINED[id(actual.acs)][2].qualified.origins + donor, recipient, mutable = owner._project_inputs( + actual.run.prefix.allocated_population.frame, + actual.assigned, + actual.acs, + origins, + ) + assert donor.person[owner.YEAR].eq(2025).all() + np.testing.assert_array_equal( + recipient.person[owner.YEAR], np.where(mutable, 2024, 2025) + ) + assert recipient.person.loc[mutable, "A_LINENO"].isna().all() + assert recipient.person.loc[mutable, "PRCITSHP"].isna().all() + assert recipient.person.loc[~mutable, "CIT"].isna().all() + for name in owner.OUTPUTS: + np.testing.assert_array_equal(recipient.person[name].isna(), mutable) + surface = owner.transfer._transfer_feature_surface( + donor, + recipient, + entity="person", + targets=owner.OUTPUTS, + observation_year_column=owner.YEAR, + ) + assert surface.required == ( + *owner.transfer.ACS_PERSON_TRANSFER_PREDICTORS, + *owner.transfer._IMMIGRATION_REQUIRED_FEATURES, + ) + assert set(surface.optional) <= {owner.transfer._HEAD_FEATURE} + + +def test_bank_replay(actual): + _phase("warm_transfer.before") + repeated = owner.transfer_current_survey_immigration( + actual.run, + actual.assigned, + actual.acs, + seed=42, + n_estimators=2, + bank_root=actual.bank_root, + ) + _phase("warm_transfer.after") + pd.testing.assert_frame_equal(repeated.pairs, actual.result.pairs) + assert ( + json.loads(repeated.receipt)["bank_identity_sha256"] + == json.loads(actual.result.receipt)["bank_identity_sha256"] + ) + + +@pytest.mark.parametrize("which", ["run", "assigned", "acs", "result"]) +def test_copied_authority_refuses(actual, which): + if which == "result": + with pytest.raises(ValueError, match="ISSUED_OWNER_REQUIRED"): + replace(actual.result).validate() + else: + inputs = [actual.run, actual.assigned, actual.acs] + inputs[("run", "assigned", "acs").index(which)] = replace( + getattr(actual, which) + ) + with pytest.raises( + ValueError, + match="UNISSUED_FINANCIAL_RUN|ISSUED_OWNER_REQUIRED|RETAINED_VIEW", + ): + owner.transfer_current_survey_immigration(*inputs, seed=42, n_estimators=2) + + +@pytest.mark.parametrize("field", ["frame", "pairs", "receipt"]) +def test_equal_detached_result_values_refuse(actual, field): + value = actual.result + old = getattr(value, field) + replacement = ( + owner._frame_with_person(old, old.person.copy(deep=True)) + if field == "frame" + else old.copy(deep=True) + if field == "pairs" + else b"{}" + ) + try: + object.__setattr__(value, field, replacement) + with pytest.raises(ValueError, match="RETAINED_CHANGED"): + value.validate() + finally: + object.__setattr__(value, field, old) + + +@pytest.mark.parametrize( + "surface", + [ + "cell", + "metadata", + "weight", + "kind", + "anchor", + "owners", + "version", + "ledger", + ], +) +def test_original_population_authority_is_not_replaceable(actual, surface): + # Detached pure reconstruction controls never alter the issued parent. + # They prove allocation equality checks, not issuance of these copies. + population = owner.allocation._copy_population( + actual.run.prefix.allocated_population + ) + view = actual.run.prefix.preparation.checked_view() + restore = [] + + def set_attr(obj, name, value): + previous = getattr(obj, name) + restore.append(lambda: object.__setattr__(obj, name, previous)) + object.__setattr__(obj, name, value) + + if surface == "cell": + table = population.frame.person + old = table.at[table.index[0], "person_household_id"] + restore.append( + lambda: table.__setitem__( + "person_household_id", + table.person_household_id.where(table.index != table.index[0], old), + ) + ) + table.at[table.index[0], "person_household_id"] += 1 + elif surface == "metadata": + set_attr(population.frame, "_metadata", {"foreign": True}) + elif surface == "weight": + weights = population.frame.weights_for("household") + changed = weights.values.copy() + changed[0] += 1 + changed[1] -= 1 + set_attr(weights, "values", changed) + elif surface == "kind": + set_attr( + population.frame.weights_for("household"), "kind", owner.WeightKind.DESIGN + ) + elif surface == "anchor": + changed = dict(population.design_weights) + changed["household"] = changed["household"].copy() + 1 + set_attr(population, "design_weights", changed) + elif surface == "owners": + set_attr(population, "owners", {}) + elif surface == "version": + set_attr(population, "version", "foreign") + elif surface == "ledger": + set_attr(population, "mass_ledger", ()) + try: + with pytest.raises(ValueError): + owner.allocation._raw_allocation(view, population) + finally: + for operation in reversed(restore): + operation() + + +def test_real_nonzero_controls_refuse_invented_stock_shortfall(actual): + origins = owner.acs_owner._RETAINED[id(actual.acs)][2].qualified.origins + donor, recipient, mutable = owner._project_inputs( + actual.run.prefix.allocated_population.frame, + actual.assigned, + actual.acs, + origins, + ) + controls = owner._ISSUED[id(actual.result)][2].controls + draw = next(d for d in controls.humanitarian if d.origin == "afghanistan") + nonzero = replace(controls, humanitarian=(replace(draw, target=73566),)) + person = recipient.person.copy(deep=True) + for name in owner.OUTPUTS: + person[name] = actual.result.pairs[name].to_numpy() + with pytest.raises(ValueError, match="shortfall|candidate|available"): + owner.rules.reconcile_us_immigration_humanitarian_transfer( + person, + weights=recipient.resolve_weights("person").values, + mutable_rows=mutable, + seed=42, + controls=nonzero, + observation_year_column=owner.YEAR, + ) + + +@pytest.mark.parametrize( + "module,name", + [ + (owner.allocation, "_raw_allocation"), + (owner.allocation, "_population_stamp"), + (owner.host, "_pure_run"), + (owner.source, "_pure_final"), + (owner.allocation.graph, "_check_population_state"), + (owner.allocation.replay, "same_replayed_frame"), + ], +) +def test_rebound_authority_helpers_refuse_before_invocation( + actual, monkeypatch, module, name +): + def changed(*args, **kwargs): + raise AssertionError("Rebound authority must never be called") + + with monkeypatch.context() as patch: + patch.setattr(module, name, changed) + with pytest.raises(ValueError, match="IMPLEMENTATION_CHANGED"): + actual.result.validate() + + +def test_final_control_io_mutation_permanently_revokes_parent_last(actual, monkeypatch): + # Last test: this poisons the one genuine host. Restore only borrowed bytes, + # never reset the permanent revocation flag or claim the owner is usable. + population = actual.run.prefix.allocated_population + original = population.owners + reader = Path.read_text + changed = [] + + def late(path, *args, **kwargs): + text = reader(path, *args, **kwargs) + if path.name == "source_stages.json" and not changed: + changed.append(True) + object.__setattr__(population, "owners", {}) + return text + + try: + with monkeypatch.context() as patch: + patch.setattr(Path, "read_text", late) + with pytest.raises(ValueError): + actual.result.validate() + finally: + object.__setattr__(population, "owners", original) + assert changed + assert owner.host._run_entry(actual.run)[2].completion_boundary.revoked + with pytest.raises(ValueError, match="REVOKED"): + actual.run.checked_view() diff --git a/packages/microcosm-build/tests/test_us_current_survey_person_status.py b/packages/microcosm-build/tests/test_us_current_survey_person_status.py new file mode 100644 index 000000000..d33cd956d --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_person_status.py @@ -0,0 +1,226 @@ +"""Published item universes and periods; all observations are invented literals.""" + +import pytest + +from microcosm.build.us_runtime import current_survey_person_status as status + + +def asec(age=20, person_type="2", **changes): + row = {c: "0" for c in status.ASEC_COLUMNS} + row.update( + PERIDNUM="0000000000000000000001", + PH_SEQ="00007", + A_LINENO="1", + A_AGE=str(age), + PRPERTYP=person_type, + PRDISFLG="2", + A_ENRLW="2", + ) + for item in status.ITEMS: + row[item.asec] = "2" if person_type == "2" else "-1" + if not 16 <= age <= 54 or person_type in ("1", "3"): + row.update(A_ENRLW="0", A_FTPT="0", A_HSCOL="0") + if person_type in ("1", "3"): + row["PRDISFLG"] = "-1" + row.update(changes) + return row + + +def acs(age=20, **changes): + row = {c: "0" for c in status.ACS_COLUMNS} + row.update( + SERIALNO="2024HU0000001", + SPORDER="1", + AGEP=str(age), + DIS="2", + SCH="1" if age >= 3 else "", + SCHG="", + ) + for item in status.ITEMS: + row[item.acs] = "2" if age >= item.acs_min_age else "" + row.update(changes) + return row + + +@pytest.mark.parametrize("age,count", [(4, 2), (5, 5), (14, 5), (15, 6)]) +def test_acs_battery_uses_item_universe_not_six_mandatory_answers(age, count): + result = status.recode_person_status(acs(age), survey="acs") + assert result["survey_any_applicable_difficulty"] is False + assert result["survey_any_applicable_difficulty__complete"] + assert len(result["survey_any_applicable_difficulty__applicable_fields"]) == count + if age < 15: + assert result["survey_independent_living_difficulty"] is None + assert ( + result["survey_independent_living_difficulty__status"] == "outside_universe" + ) + + +@pytest.mark.parametrize( + "age,person_type,known", + [(14, "2", True), (40, "1", False), (40, "3", False), (40, "0", False)], +) +def test_asec_battery_uses_published_person_type_not_substituted_age( + age, person_type, known +): + result = status.recode_person_status(asec(age, person_type), survey="asec") + assert result["survey_any_applicable_difficulty__known"] is known + assert result["survey_status_original_age"] == age + + +@pytest.mark.parametrize("survey", ["acs", "asec"]) +def test_a_partial_valid_yes_is_known_but_incomplete_and_never_statutory_blindness( + survey, +): + row = acs(DEYE="1", DEAR="") if survey == "acs" else asec(PEDISEYE="1", PEDISEAR="") + result = status.recode_person_status(row, survey=survey) + assert result["survey_vision_difficulty"] is True + assert result["survey_any_applicable_difficulty"] is True + assert not result["survey_any_applicable_difficulty__complete"] + assert result["survey_any_applicable_difficulty__coherent"] + assert result["survey_publisher_disability_recode__agrees_with_battery"] is False + assert ( + not {"is_blind", "is_disabled", "is_full_time_college_student"} & result.keys() + ) + assert result["survey_status_canonical_eligibility_assigned"] is False + + +@pytest.mark.parametrize("survey", ["acs", "asec"]) +def test_missing_applicable_no_does_not_become_a_negative_battery(survey): + row = acs(DEYE="") if survey == "acs" else asec(PEDISEYE="") + result = status.recode_person_status(row, survey=survey) + assert result["survey_any_applicable_difficulty"] is None + assert not result["survey_any_applicable_difficulty__complete"] + + +@pytest.mark.parametrize("token", ["1", "2", "b", " "]) +def test_acs_outside_universe_codes_do_not_manufacture_summary(token): + result = status.recode_person_status(acs(4, DEYE="1", DOUT=token), survey="acs") + assert result["survey_vision_difficulty"] is True + assert result["survey_independent_living_difficulty"] is None + assert result["survey_any_applicable_difficulty"] is None + assert not result["survey_any_applicable_difficulty__coherent"] + + +@pytest.mark.parametrize( + "token,expected", + [ + ("-4", "unlabelled_code"), + ("0", "unlabelled_code"), + ("", "missing"), + ("1.0", "malformed"), + (" 1", "malformed"), + ], +) +def test_printed_range_and_malformed_dressing_answers_are_not_named_no(token, expected): + result = status.recode_person_status(asec(PEDISDRS=token), survey="asec") + assert result["person_status_source_PEDISDRS__literal_status"] == expected + assert result["survey_self_care_difficulty"] is None + assert result["survey_any_applicable_difficulty"] is None + + +@pytest.mark.parametrize( + "token,meaning", + [ + ("00", "value_no_change"), + ("1", "blank_no_change"), + ("21", "blank_to_longitudinal_value"), + ("40", "value_to_allocated_value"), + ("50", "value_to_blank"), + ("-1", "not_allocated"), + ("51", "allocation_or_edit_unknown"), + ], +) +def test_asec_edit_transition_is_preserved_without_nonzero_allocated_collapse( + token, meaning +): + result = status.recode_person_status( + asec(PEDISEYE="1", PXDISEYE=token), survey="asec" + ) + assert result["person_status_source_PXDISEYE"] == token + assert result["person_status_source_PXDISEYE__meaning"] == meaning + assert result["survey_vision_difficulty"] is True + + +@pytest.mark.parametrize("token", ["1", "2", "3", "", "9"]) +def test_unlabelled_student_allocation_does_not_invent_no_or_reported_answer(token): + result = status.recode_person_status( + asec(A_ENRLW="1", A_FTPT="1", A_HSCOL="2", AXHSCOL=token), survey="asec" + ) + assert ( + result["person_status_source_AXHSCOL__meaning"] == "allocation_or_edit_unknown" + ) + assert result["survey_full_time_college_student_last_week"] is True + assert result["survey_student_reference_period"] == status.ASEC_STUDENT_PERIOD + assert not result["survey_student_annual_five_month_status_validated"] + + +@pytest.mark.parametrize( + "enrolled,workload,level,expected", + [ + ("0", "0", "0", None), + ("2", "0", "0", False), + ("1", "1", "2", True), + ("1", "1", "1", False), + ("1", "2", "2", False), + ("2", "1", "2", None), + ("1", "0", "2", None), + ("1", "1", "", None), + ], +) +def test_asec_student_combination_describes_only_last_week( + enrolled, workload, level, expected +): + result = status.recode_person_status( + asec(A_ENRLW=enrolled, A_FTPT=workload, A_HSCOL=level), survey="asec" + ) + assert result["survey_full_time_college_student_last_week"] is expected + assert result["survey_college_attended_last_3_months"] is None + + +@pytest.mark.parametrize( + "school,level,expected", + [ + ("1", "", False), + ("2", "15", True), + ("3", "16", True), + ("2", "14", False), + ("1", "15", None), + ("", "15", None), + ("2", "bb", None), + ], +) +def test_acs_college_attendance_has_unknown_full_time_workload(school, level, expected): + result = status.recode_person_status( + acs(SCH=school, SCHG=level, FSCHP="1", FSCHGP="0"), survey="acs" + ) + assert result["survey_college_attended_last_3_months"] is expected + assert result["survey_full_time_college_student_last_week"] is None + assert not result["survey_student_full_time_measured"] + assert result["survey_school_level_separate_allocation_measured"] + assert "SCHL" not in status.ACS_COLUMNS and "FSCHGP" in status.ACS_COLUMNS + assert result["person_status_source_FSCHP__meaning"] == "allocated" + assert result["person_status_source_FSCHGP__meaning"] == "not_allocated" + + +@pytest.mark.parametrize("token", [None, True, 1, 1.0, "0" * 65]) +def test_nonliteral_or_unbounded_source_values_refuse(token): + with pytest.raises(ValueError, match="LITERAL_TYPE_OR_BOUND"): + status.recode_person_status(asec(PEDISEYE=token), survey="asec") + + +@pytest.mark.parametrize("token", ["01", "02", "-1"]) +def test_acs_item_width_agrees_with_literal_code_knownness(token): + result = status.recode_person_status(acs(DEYE=token), survey="acs") + assert result["person_status_source_DEYE__code"] is None + assert result["survey_vision_difficulty"] is None + assert not result["survey_vision_difficulty__known"] + + +@pytest.mark.parametrize("token", ["", "-1", "9"]) +def test_unreadable_student_literal_is_unresolved_even_outside_age_universe(token): + result = status.recode_person_status(asec(age=55, A_HSCOL=token), survey="asec") + assert result["survey_full_time_college_student_last_week"] is None + assert ( + result["survey_full_time_college_student_last_week__status"] + == "unresolved_school_literal" + ) diff --git a/packages/microcosm-build/tests/test_us_current_survey_person_status_graph.py b/packages/microcosm-build/tests/test_us_current_survey_person_status_graph.py new file mode 100644 index 000000000..245a9876a --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_person_status_graph.py @@ -0,0 +1,580 @@ +"""Descriptive schema controls and actual four-node replay on tiny real owners. + +Detached fixtures are explicitly unissued and used only for pure declarations, +recodes and bindings. The separate actual_graph fixture prepares invented +members through the real issuer before any kernel is admitted or executed. +""" + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_survey_person_status import acs, asec + +from microcosm.build.us_runtime import graph_current_survey_person_status as graph +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ArtifactInput, ArtifactType, ArtifactValue, NumericScope +from microcosm.graph import population as populations +from microcosm.graph.keys import opaque_artifact_key + + +def _frame(people): + people = people.copy() + tables = {"person": people} + for entity in US_SCHEMA.group_entities: + people[US_SCHEMA.membership_column(entity)] = people.person_id.to_numpy() + tables[entity] = pd.DataFrame( + {US_SCHEMA.entity_id_column(entity): np.sort(people.person_id.to_numpy())} + ) + return Frame( + tables, + US_SCHEMA, + {"household": Weights(np.ones(len(people)), WeightKind.DESIGN)}, + ) + + +def detached(arms=("asec", "acs"), changes=None): + """No owner callback: these values cannot pass actual kernel admission.""" + rows = [ + asec(PERIDNUM=str(10**21 + i)) if arm == "asec" else acs(SPORDER=str(i + 1)) + for i, arm in enumerate(arms) + ] + if changes: + for row, arm in zip(rows, arms, strict=True): + row.update(changes.get(arm, {})) + index = pd.Index( + [2**53 + 1 + 2 * i for i in range(len(rows))], name="person_id", dtype="int64" + ) + origins = pd.DataFrame( + {"source": arms, "native_person_id": index.to_numpy() + 100}, index=index + ) + frame = _frame( + pd.DataFrame( + { + "person_id": index.to_numpy(), + "A_AGE": [ + int(r["A_AGE" if a == "asec" else "AGEP"]) + for a, r in zip(arms, rows, strict=True) + ], + } + ) + ) + selected, keys = {"asec": {}, "acs": {}}, {} + for arm, row, pid in zip(arms, rows, index, strict=True): + key = graph.source.original._key(row, arm) + selected[arm][key] = row + keys[arm, key] = int(pid) + raw, observed = graph.source._combine(frame, origins, keys, selected) + return graph.source.QualifiedSurveyPersonStatus( + frame, origins, raw, observed, b'{"invented_unissued":true}' + ) + + +def receiving(qualified): + pairs = [(int(pid), clone) for pid in qualified.origins.index for clone in (0, 1)][ + ::-1 + ] + people = pd.DataFrame( + { + "person_id": np.arange(1, len(pairs) + 1, dtype="int64"), + graph.provenance.support_source_id_column("person"): np.array( + [p for p, _ in pairs], dtype="int64" + ), + graph.provenance.support_clone_index_column("person"): np.array( + [c for _, c in pairs], dtype="int64" + ), + graph.provenance.spine_source_id_column("person"): np.array( + [qualified.origins.at[p, "native_person_id"] for p, _ in pairs], + dtype="int64", + ), + graph.provenance.support_channel_column("person"): pd.array( + [qualified.origins.at[p, "source"] for p, _ in pairs], + dtype=graph.source.STRING, + ), + "unrelated_amount": np.arange(len(pairs), dtype="float64"), + } + ) + return _frame(people) + + +def declarations(qualified, frame=None): + frame = receiving(qualified) if frame is None else frame + edge = ArtifactInput( + "ordering", "invented.after", "evidence", ArtifactType("invented.ordering", 1) + ) + pins = { + edge.name: { + "producer_key": "1" * 64, + "artifact_key": "2" * 64, + "payload_sha256": hashlib.sha256(b"invented ordering").hexdigest(), + } + } + nodes = graph.current_survey_person_status_nodes( + qualified, + frame, + receiving_version="invented.postclone", + after=edge, + host_edges=(edge,), + host_pins=pins, + ) + return nodes, edge, pins + + +@pytest.mark.parametrize("arms", [("acs",), ("asec",), ("asec", "acs")]) +def test_fixed_schema_including_all_null_opposite_survey_codes(arms): + qualified = detached(arms) + raw, observed = graph.status_tables(qualified) + assert tuple(raw) == tuple(n for n, _ in graph.LITERAL_TOKENS) + assert tuple(observed) == tuple(n for n, _ in graph.RECODE_TOKENS) + for table, schema in ((raw, graph.LITERAL_TOKENS), (observed, graph.RECODE_TOKENS)): + assert tuple( + populations.token_for_dtype(table[n].dtype) for n, _ in schema + ) == tuple(t for _, t in schema) + nodes, _, _ = declarations(qualified) + assert tuple(n.id for n in nodes) == graph.NODE_IDS + assert tuple(o.column for o in nodes[-1].outputs) == graph.BIND_COLUMNS + assert not { + "is_blind", + "is_disabled", + "is_full_time_college_student", + "is_household_head", + "is_female", + "prior_wages", + } & set(graph.BIND_COLUMNS) + assert not any(c.startswith("person_status_source_") for c in graph.BIND_COLUMNS) + missing = "PEDISEYE" if arms == ("acs",) else "DEYE" if arms == ("asec",) else None + if missing: + assert observed["person_status_source_" + missing + "__code"].isna().all() + + +@pytest.mark.parametrize( + "changes,expected,known", + [ + ({"DEYE": "1", "DEAR": ""}, True, True), + ({"DEYE": ""}, None, False), + ({"DEYE": "b"}, None, False), + ({"DEYE": "2"}, False, True), + ], +) +def test_exact_partial_and_unknown_semantics_survive_binding(changes, expected, known): + qualified = detached(("acs",), {"acs": changes}) + frame = receiving(qualified) + nodes, _, _ = declarations(qualified, frame) + result = graph._result(qualified, nodes[-1], frame.person) + value = result.columns["person", "survey_vision_difficulty"] + assert value.isna().all() if expected is None else value.eq(expected).all() + assert result.columns["person", "survey_vision_difficulty__known"].eq(known).all() + assert ( + result.columns["person", "survey_full_time_college_student_last_week"] + .isna() + .all() + ) + assert ( + result.columns["person", "survey_student_reference_period"] + .eq(graph.status.ACS_STUDENT_PERIOD) + .all() + ) + + +def test_large_integer_private_artifact_and_both_exact_clone_identities(): + qualified = detached() + before = graph.person_status_seal(qualified) + frame = receiving(qualified) + nodes, _, _ = declarations(qualified, frame) + bound = graph._result(qualified, nodes[-1], frame.person) + payload = json.loads(graph.projection_payload(qualified)) + assert payload["origins"]["index"] == [str(v) for v in qualified.origins.index] + assert payload["origins"]["rows"][0][1] == {"integer_literal": str(2**53 + 101)} + assert payload["raw"]["rows"][0][graph.status.RAW_COLUMNS.index("PERIDNUM")] == str( + 10**21 + ) + originals = frame.person[graph.provenance.support_source_id_column("person")] + assert ( + bound.columns["person", "survey_status_source"].tolist() + == qualified.origins.source.reindex(originals).tolist() + ) + assert graph.person_status_seal(qualified) == before + public = json.dumps(dict(bound.receipt)) + assert str(10**21) not in public and "native_person_id" not in public + columns_doc = json.loads(graph._columns_payload(graph.status_tables(qualified)[0])) + assert set(columns_doc) == {"protocol", "rows", "columns", "table_sha256"} + assert isinstance(columns_doc["rows"], int) + + +@pytest.mark.parametrize( + "defect", + ["float_original", "float_clone", "native", "arm", "pair", "missing", "collision"], +) +def test_binder_refuses_inexact_foreign_missing_and_duplicate_joins(defect): + qualified = detached() + frame = receiving(qualified) + nodes, _, _ = declarations(qualified, frame) + people = frame.person.copy() + original, clone, native, channel = graph._identities() + if defect == "float_original": + people[original] = people[original].astype(float) + elif defect == "float_clone": + people[clone] = people[clone].astype(float) + elif defect == "native": + people.loc[0, native] += 1 + elif defect == "arm": + people.loc[0, channel] = "foreign" + elif defect == "pair": + people.loc[0, clone] = people.loc[1, clone] + elif defect == "missing": + people = people.iloc[:-1].copy() + else: + people["survey_vision_difficulty"] = pd.array( + [False] * len(people), dtype="boolean" + ) + with pytest.raises(ValueError): + graph._result(qualified, nodes[-1], people) + + +@pytest.mark.parametrize("column", ["survey_vision_difficulty", "unrelated_amount"]) +def test_complete_materialization_refuses_status_or_untouched_cell_drift(column): + qualified = detached() + frame = receiving(qualified) + nodes, _, _ = declarations(qualified, frame) + incoming = populations.Population.from_frame(frame, nodes[-1].population) + result = graph._result(qualified, nodes[-1], frame.person) + output = populations.patch(incoming, nodes[-1], result) + graph.verify_materialized_person_status( + output, incoming, qualified=qualified, node=nodes[-1] + ) + output.frame.person.loc[0, column] = True if column.startswith("survey_") else 999.0 + with pytest.raises(ValueError): + graph.verify_materialized_person_status( + output, incoming, qualified=qualified, node=nodes[-1] + ) + + +@pytest.mark.parametrize( + "defect", ["raw", "recode", "source_arm", "host_pin", "artifact"] +) +def test_recode_inputs_and_exact_typed_artifacts_are_checked(defect): + qualified = detached() + nodes, edge, pins = declarations(qualified) + projected = graph._projection_frame(qualified) + literal = graph._result(qualified, nodes[1], projected.person) + incoming = populations.patch( + populations.Population.from_frame(projected, nodes[0].id), nodes[1], literal + ) + if defect == "raw": + incoming.frame.person.loc[0, "person_status_source_PEDISEYE"] = "1" + with pytest.raises(ValueError, match="RECODE_SOURCE_SLICE"): + graph._result(qualified, nodes[2], incoming.frame.person) + elif defect == "source_arm": + incoming.frame.person.loc[0, "person_status_source_arm"] = "acs" + with pytest.raises(ValueError, match="RECODE_SOURCE_ARM"): + graph._result(qualified, nodes[2], incoming.frame.person) + elif defect == "recode": + qualified.observations.iloc[ + 0, qualified.observations.columns.get_loc("survey_vision_difficulty") + ] = True + with pytest.raises(ValueError, match="RECODE_QUALIFIER_DISAGREEMENT"): + graph.status_tables(qualified) + else: + pin = pins[edge.name] + value = ArtifactValue( + b"invented ordering", + edge.type, + pin["artifact_key"], + pin["producer_key"], + NumericScope(), + ) + if defect == "host_pin": + value = replace(value, producer_key="3" * 64) + else: + value = replace(value, payload=b"changed") + with pytest.raises(ValueError, match="HOST_EDGE_PIN"): + graph._check_artifacts(nodes[0], {edge.name: value}, qualified) + + +def test_pure_declarations_do_not_issue_retained_authority(): + qualified = detached() + frame = receiving(qualified) + _, edge, pins = declarations(qualified, frame) + with pytest.raises(ValueError, match="RETAINED_OWNER_REQUIRED"): + graph.person_status_kernels( + qualified, + frame, + receiving_version="invented.postclone", + after=edge, + host_edges=(edge,), + host_pins=pins, + require_current=lambda: None, + ) + + +@pytest.fixture(scope="module") +def actual_graph(tmp_path_factory): + from test_us_current_survey_person_status_source import _source_arguments + from test_us_graph_atomic_survey_population import _support_payload + + from microcosm.build.us_runtime import graph_atomic_survey_population as atomic + from microcosm.graph import compile_graph, run_graph + from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys + + root = tmp_path_factory.mktemp("person-status-actual-graph") + with pytest.MonkeyPatch.context() as patch: + arguments = _source_arguments(root, patch) + payload, source_ids = _support_payload() + support = root / "invented-block-support.npz" + support.write_bytes(payload) + config = atomic.reconstruction.AtomicSurveyReconstruction( + support_path=str(support), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + prefix = atomic.run_atomic_survey_population( + **arguments, + store_root=root / "store", + geography_config=config, + return_values=True, + ) + qualified = graph.qualify_current_survey_person_status(prefix.preparation) + # The actual host supplies the same four real typed ancestor pins. + from microcosm.build.us_runtime import ( + graph_current_survey_predictors as financial, + ) + + edges = ( + *financial.host.current_survey_host_edges(), + financial._geography_edge(), + ) + pins = {} + for edge in edges: + record = prefix.manifest.node(edge.producer) + pins[edge.name] = { + "producer_key": record.key, + "artifact_key": record.opaque_artifacts[edge.artifact], + "payload_sha256": graph._sha( + prefix.store.load_bytes(record.opaque_artifacts[edge.artifact]) + ), + } + seal = graph.person_status_seal(qualified) + state = SimpleNamespace(calls=0, mutate=None) + + def current(): + assert graph.person_status_seal(qualified) == seal + state.calls += 1 + if state.mutate is not None: + state.mutate(state.calls) + + nodes, kernels = graph.person_status_kernels( + qualified, + prefix.clone_population.frame, + receiving_version=prefix.clone_population.version, + after=edges[-1], + host_edges=edges, + host_pins=pins, + require_current=current, + ) + compiled = compile_graph( + replace(prefix.compiled.graph, nodes=(*prefix.compiled.graph.nodes, *nodes)) + ) + for kernel in kernels: + prefix.kernels.register(kernel) + _, source_keys = _source_paths_and_keys(compiled, prefix.sources, prefix.store) + keys, _ = _all_node_keys(compiled, prefix.kernels, source_keys) + outputs = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + sources=prefix.sources, + store=prefix.store, + kernels=prefix.kernels, + resume=resume, + _population_observer=lambda n, p, target=observed: target.__setitem__( + n, p + ), + ) + qualified.validate() + assert graph.person_status_seal(qualified) == seal + outputs.append((manifest, observed)) + yield SimpleNamespace( + prefix=prefix, + qualified=qualified, + nodes=nodes, + kernels=kernels, + compiled=compiled, + keys=keys, + outputs=outputs, + state=state, + ) + qualified.validate() + + +def test_actual_thirteen_node_cold_required_replay_and_private_source_schema( + actual_graph, +): + case = actual_graph + payload = graph.projection_payload(case.qualified) + assert 0 < len(payload) <= 16 * 1024 * len(case.qualified.origins) + assert len(payload) <= graph.MAX_ARTIFACT_BYTES + assert len(case.compiled.order) == 13 + assert case.outputs[0][0].key == case.outputs[1][0].key + assert all(n.hit for n in case.outputs[1][0].nodes.values()) + for manifest, observed in case.outputs: + assert set(observed) == set(case.compiled.order) + graph.verify_materialized_person_status( + observed[graph.BIND_NODE], + case.prefix.clone_population, + qualified=case.qualified, + node=case.nodes[-1], + ) + current = {} + for node in case.nodes: + artifacts = {} + for edge in node.artifact_inputs: + record = manifest.node(edge.producer) + artifacts[edge.name] = ArtifactValue( + case.prefix.store.load_bytes( + record.opaque_artifacts[edge.artifact] + ), + edge.type, + opaque_artifact_key(case.keys[edge.producer], edge.artifact), + case.keys[edge.producer], + NumericScope(), + ) + incoming = ( + None + if node.id == graph.SOURCE_NODE + else case.prefix.clone_population + if node.id == graph.BIND_NODE + else current[graph.SOURCE_NODE] + ) + expected, result = graph.expected_person_status_population( + incoming, qualified=case.qualified, node=node, artifacts=artifacts + ) + graph.replay.same_replayed_population(expected, observed[node.id]) + current[expected.version] = expected + for name, payload in result.artifacts.items(): + assert ( + case.prefix.store.load_bytes( + manifest.node(node.id).opaque_artifacts[name] + ) + == payload + ) + assert ( + case.state.calls == 8 + ) # Two pure owner boundaries per executed fragment node. + + +@pytest.mark.parametrize("boundary", [1, 2]) +@pytest.mark.parametrize( + "mutation", + [ + "raw", + "observations", + "receipt", + "callback", + "owner", + "code", + "defaults", + "closure", + "constant", + "student", + ], +) +def test_actual_kernel_first_and_final_owner_mutation_refuses( + actual_graph, boundary, mutation +): + from microcosm.graph.executor import _project_context + + case = actual_graph + qualified = case.qualified + kernel = case.kernels[0] + node = case.nodes[0] + manifest = case.outputs[0][0] + artifacts = {} + for edge in node.artifact_inputs: + record = manifest.node(edge.producer) + artifacts[edge.name] = ArtifactValue( + case.prefix.store.load_bytes(record.opaque_artifacts[edge.artifact]), + edge.type, + record.opaque_artifacts[edge.artifact], + record.key, + NumericScope(), + ) + context = _project_context( + node, + None, + key=case.keys[node.id], + sources=case.prefix.sources, + tolerances={}, + numerics={}, + artifacts=artifacts, + ) + pid = qualified.raw.index[0] + raw, observed, receipt, callback = ( + qualified.raw.copy(deep=True), + qualified.observations.copy(deep=True), + qualified.receipt, + kernel.require_current, + ) + prep_payload = case.prefix.preparation.payload + source_closure = dict( + zip( + qualified._revalidate.__code__.co_freevars, + (c.cell_contents for c in qualified._revalidate.__closure__), + strict=True, + ) + ) + controls = source_closure["controls"] + controls_body = controls._body + callback_code, callback_defaults = callback.__code__, callback.__defaults__ + closure_cell = dict( + zip(callback.__code__.co_freevars, callback.__closure__, strict=True) + )["seal"] + closure_value, protocol = closure_cell.cell_contents, graph.status.PROTOCOL + start = case.state.calls + + def mutate(count): + if count != start + boundary: + return + if mutation == "raw": + qualified.raw.loc[pid, "DEYE"] = "changed" + elif mutation == "observations": + qualified.observations.loc[pid, "survey_vision_difficulty"] = True + elif mutation == "receipt": + object.__setattr__(qualified, "receipt", b"{}") + elif mutation == "callback": + kernel.require_current = lambda: None + elif mutation == "code": + callback.__code__ = callback.__code__.replace(co_name="changed_callback") + elif mutation == "defaults": + callback.__defaults__ = ("changed",) + elif mutation == "closure": + closure_cell.cell_contents = ("changed",) + elif mutation == "constant": + graph.status.PROTOCOL = "changed" + elif mutation == "student": + object.__setattr__(controls, "_body", controls_body + b"changed") + else: + object.__setattr__(case.prefix.preparation, "payload", b"{}") + + case.state.mutate = mutate + try: + with pytest.raises((ValueError, AssertionError)): + kernel.run(context) + finally: + for column in raw: + qualified.raw[column] = raw[column].copy(deep=True) + for column in observed: + qualified.observations[column] = observed[column].copy(deep=True) + object.__setattr__(qualified, "receipt", receipt) + kernel.require_current = callback + callback.__code__, callback.__defaults__ = callback_code, callback_defaults + closure_cell.cell_contents, graph.status.PROTOCOL = closure_value, protocol + object.__setattr__(case.prefix.preparation, "payload", prep_payload) + object.__setattr__(controls, "_body", controls_body) + case.state.mutate = None + qualified.validate() diff --git a/packages/microcosm-build/tests/test_us_current_survey_person_status_source.py b/packages/microcosm-build/tests/test_us_current_survey_person_status_source.py new file mode 100644 index 000000000..9befe54c8 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_person_status_source.py @@ -0,0 +1,335 @@ +"""Actual source qualification over small invented and privately pinned members.""" + +import copy +import csv +import hashlib +import io +import json +import shutil +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_survey_person_status import acs, asec + +from microcosm.build.us_runtime import current_survey_person_status as status +from microcosm.build.us_runtime import current_survey_person_status_source as owner + + +def _source_arguments(tmp_path, monkeypatch, *, incumbent_conflict=False): + import test_us_survey_population_preparation as preparation_fixture + from test_us_current_asec_demographics import _demographic_arguments + + from microcosm.build.us_runtime import asec_person_income_source as restoration + from microcosm.build.us_runtime import current_asec_demographics as demographic + + original_person = preparation_fixture._person + original_asec = preparation_fixture.asec_fixture + + def person(*args, **kwargs): + row = original_person(*args, **kwargs) + observations = acs(int(row["AGEP"])) + row.update( + { + c: v + for c, v in observations.items() + if c not in ("SERIALNO", "SPORDER", "AGEP") + } + ) + return row + + def asec_fixture(*args, **kwargs): + # Correct the invented parent before its real checkpoint/owner issuance. + # The shared fixture has ages 55/14, so source enrollment is NIU, not 2. + kwargs["person_changes"] = { + "A_ENRLW": np.zeros(6, dtype=np.int64), + "A_FTPT": np.zeros(6, dtype=np.int64), + **{ + item.asec: np.array([2, -1] * 3, dtype=np.int64) + for item in status.ITEMS + }, + } + if incumbent_conflict: + kwargs["person_changes"]["PEDISEYE"] = np.array([1, -1] * 3, dtype=np.int64) + return original_asec(*args, **kwargs) + + monkeypatch.setattr(preparation_fixture, "_person", person) + monkeypatch.setattr(preparation_fixture, "asec_fixture", asec_fixture) + arguments = _demographic_arguments(tmp_path, monkeypatch, zero=False) + source_dir = arguments["source_dir"] / "asec" + members, pins = {}, [] + for year, member, archive, *_ in owner.original.asec._MEMBER_PINS: + path = source_dir / member + table = pd.read_csv(path, dtype=str, keep_default_na=False) + # ASEC fixture already has the precise PRPERTYP roster and original age. + for position, row in table.iterrows(): + observations = asec(int(row.A_AGE), row.PRPERTYP) + for c, value in observations.items(): + if c not in ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "PRPERTYP"): + table.loc[position, c] = value + table.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(table), + len(data), + ) + ) + members[year] = path + for module in ( + owner.original.asec, + restoration, + demographic.demographic, + owner.student, + ): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "status-restored-money" + restoration.restore_asec_person_income_source( + source_dir / "parent.h5", + source_dir / "household-attachment.h5", + member_paths=members, + output_dir=output, + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, + source_dir / "person-income-attachment.h5", + ) + return arguments + + +def test_actual_qualifier_retains_complete_sources_and_closes_mutation_seals( + tmp_path, monkeypatch +): + arguments = _source_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + original = prepared.checked_view().frame + before = owner.source._frame_identity(original) + result = owner.qualify_current_survey_person_status(prepared) + result.validate() + receipt = json.loads(result.receipt) + assert result.source_frame is original + assert result.origins.index.tolist() == original.person.person_id.tolist() + assert result.raw.index.equals(result.observations.index) + assert receipt["student_controls_income_cohorts"] == [2022, 2023, 2024] + assert receipt["student_controls_selected_income_cohort"] == 2024 + assert ( + receipt["asec_observation_year"] == 2025 + and receipt["acs_observation_year"] == 2024 + ) + assert not receipt["canonical_eligibility_assigned"] + assert not receipt["annual_five_month_student_status_validated"] + assert not receipt["source_admission_issued"] and not receipt["release_eligible"] + assert receipt["named_observations"] == list(status.OBSERVATIONS) + assert receipt["raw_columns"] == list(status.RAW_COLUMNS) + assert receipt["observation_columns"] == list(result.observations) + assert receipt["difficulty_items"] == [dict(vars(item)) for item in status.ITEMS] + assert receipt["allocation_code_meanings"]["acs"] == [ + {"code": 0, "meaning": "not_allocated"}, + {"code": 1, "meaning": "allocated"}, + ] + assert not {"is_blind", "is_disabled", "is_full_time_college_student"} & set( + result.observations + ) + assert owner.source._frame_identity(original) == before + asec_mask = result.origins.source.eq("asec") + assert set(receipt["known_incumbent_status_compared_rows"]) == set( + owner._INCUMBENT_STATUS_FIELDS + ) + assert set(receipt["known_incumbent_status_compared_rows"].values()) == { + int(asec_mask.sum()) + } + assert result.raw.loc[asec_mask, "PERIDNUM"].str.len().eq(22).all() + assert result.raw.loc[~asec_mask, "PERIDNUM"].isna().all() + assert ( + result.observations.loc[ + ~asec_mask, "survey_full_time_college_student_last_week" + ] + .isna() + .all() + ) + assert ( + result.observations.loc[asec_mask, "survey_full_time_college_student_last_week"] + .isna() + .all() + ) + with pytest.raises(ValueError, match="PROJECTION_OBJECT_CHANGED"): + replace(result).validate() + original_callback = result._revalidate + object.__setattr__(result, "_revalidate", lambda _: None) + with pytest.raises(ValueError, match="RETAINED_OWNER_REQUIRED"): + result.validate() + object.__setattr__(result, "_revalidate", original_callback) + result.validate() + with pytest.raises(ValueError): + owner.qualify_current_survey_person_status(copy.copy(prepared)) + # No source reconstruction or normalization can hide a mutated projection. + pid = result.raw.index[0] + previous = result.raw.at[pid, "DEYE"] + result.raw.at[pid, "DEYE"] = "1" + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + result.validate() + result.raw.at[pid, "DEYE"] = previous + result.validate() + old_receipt = result.receipt + object.__setattr__(result, "receipt", b"{}") + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + result.validate() + object.__setattr__(result, "receipt", old_receipt) + # Mutating the retained issuer object must fail, even when detached rows agree. + closure = dict( + zip( + result._revalidate.__code__.co_freevars, + (c.cell_contents for c in result._revalidate.__closure__), + strict=True, + ) + ) + controls = closure["controls"] + body = controls._body + object.__setattr__(controls, "_body", body + b"x") + with pytest.raises(ValueError): + result.validate() + object.__setattr__(controls, "_body", body) + result.validate() + assert owner.source._frame_identity(original) == before + original_read = Path.read_bytes + triggered = [] + + def mutate_after_source_io(path): + data = original_read(path) + if not triggered: + triggered.append(str(path)) + result.raw.at[pid, "DEYE"] = "1" + return data + + with monkeypatch.context() as patch: + patch.setattr(Path, "read_bytes", mutate_after_source_io) + with pytest.raises(ValueError, match="PROJECTION_CHANGED"): + result.validate() + assert len(triggered) == 1 + result.raw.at[pid, "DEYE"] = previous + result.validate() + + +def test_actual_qualifier_refuses_known_incumbent_disagreement(tmp_path, monkeypatch): + arguments = _source_arguments(tmp_path, monkeypatch, incumbent_conflict=True) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + with pytest.raises(ValueError, match="INCUMBENT_STATUS_LITERAL_CONFLICT"): + owner.qualify_current_survey_person_status(prepared) + + +@pytest.mark.parametrize( + "column", + ["A_HSCOL", "PEDISDRS", "PEDISEAR", "PEDISEYE", "PEDISOUT", "PEDISPHY", "PEDISREM"], +) +def test_incumbent_literal_comparison_covers_each_retained_field(column): + row = asec() + carried = pd.DataFrame( + {name: [int(row[name])] for name in owner._INCUMBENT_STATUS_FIELDS} + ) + assert set(owner._incumbent_status(carried, 0, row)) == set( + owner._INCUMBENT_STATUS_FIELDS + ) + carried.loc[0, column] = 1 + with pytest.raises(ValueError, match="INCUMBENT_STATUS_LITERAL_CONFLICT"): + owner._incumbent_status(carried, 0, row) + + +def test_incumbent_comparison_does_not_authorize_missing_niu_or_unnamed_answers(): + row = asec(person_type="1", PEDISDRS="-4") + carried = pd.DataFrame( + {name: [int(row[name])] for name in owner._INCUMBENT_STATUS_FIELDS} + ) + carried["A_HSCOL"] = pd.array([None], dtype="Int64") + assert "A_HSCOL" not in owner._incumbent_status(carried, 0, row) + recoded = status.recode_person_status(row, survey="asec") + assert all(recoded[item.observation] is None for item in status.ITEMS) + assert recoded["person_status_source_PEDISDRS__literal_status"] == "unlabelled_code" + + +@pytest.mark.parametrize( + "column", ["person_status_source_SCHG__code", "person_status_source_FSCHGP__code"] +) +def test_all_null_published_codes_retain_integer_dtype(column): + index = pd.Index([1], name="person_id", dtype="int64") + table = owner._table([{column: None}], index) + assert table[column].dtype == pd.Int64Dtype() + assert table[column].isna().all() + + +def _stream(rows, columns): + text = io.StringIO(newline="") + writer = csv.writer(text) + writer.writerow(columns) + writer.writerows([[r[c] for c in columns] for r in rows]) + return io.BytesIO(text.getvalue().encode()) + + +def test_scanner_joins_exact_native_keys_without_float_coercion(): + first = asec(PERIDNUM="9999999999999999999999") + second = asec(PERIDNUM="9999999999999999999998", A_LINENO="2") + keys = {owner.original._key(r, "asec") for r in (first, second)} + selected = {} + assert ( + owner._scan( + _stream([second, first], status.ASEC_COLUMNS), + survey="asec", + wanted=keys, + selected=selected, + maximum=2, + ) + == 2 + ) + assert selected[owner.original._key(first, "asec")]["PERIDNUM"] == first["PERIDNUM"] + with pytest.raises(ValueError, match="DUPLICATE_SELECTED_SOURCE_KEY"): + owner._scan( + _stream([first, first], status.ASEC_COLUMNS), + survey="asec", + wanted=keys, + selected={}, + maximum=2, + ) + + +@pytest.mark.parametrize("defect", ["missing", "duplicate", "key", "age"]) +def test_source_join_refuses_missing_duplicate_mismatched_keys_and_original_age(defect): + row = acs(age=85) + key = owner.original._key(row, "acs") + index = pd.Index([2**53 + 1], name="person_id", dtype="int64") + origins = pd.DataFrame({"source": ["acs"], "native_person_id": [3]}, index=index) + frame = SimpleNamespace( + person=pd.DataFrame({"person_id": index.to_numpy(), "A_AGE": [85], "age": [90]}) + ) + keys, selected = {("acs", key): int(index[0])}, {"acs": {key: row}, "asec": {}} + if defect == "missing": + selected["acs"] = {} + elif defect == "duplicate": + selected["acs"][("2024HU0000001", "2", 2)] = row.copy() + elif defect == "key": + row["SERIALNO"] = "2024HU0000002" + else: + row["AGEP"] = "90" # Mapped age never replaces the original-source age. + with pytest.raises(ValueError, match="SURVEY_PERSON_STATUS_"): + owner._combine(frame, origins, keys, selected) + + +def test_source_join_preserves_original_age_and_large_stacked_id(): + row = acs(age=85) + key = owner.original._key(row, "acs") + index = pd.Index([2**53 + 1], name="person_id", dtype="int64") + origins = pd.DataFrame({"source": ["acs"], "native_person_id": [3]}, index=index) + frame = SimpleNamespace( + person=pd.DataFrame({"person_id": index.to_numpy(), "A_AGE": [85], "age": [90]}) + ) + raw, values = owner._combine( + frame, origins, {("acs", key): int(index[0])}, {"acs": {key: row}, "asec": {}} + ) + assert values.index[0] == 2**53 + 1 + assert raw.AGEP.iloc[0] == "85" + assert values.survey_status_original_age.iloc[0] == 85 diff --git a/packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py b/packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py new file mode 100644 index 000000000..8957fd85f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py @@ -0,0 +1,295 @@ +"""Opted-in financial conditioning over actual issuers and invented sources.""" + +import json +import sys +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments + +from microcosm.build.us_runtime import current_survey_predictors as values +from microcosm.build.us_runtime import graph_current_survey_predictors as graph +from microcosm.build.us_runtime import graph_survey_population as survey +from microcosm.fit import qrf_target +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import WeightKind +from microcosm.graph import compile_graph, run_graph + + +@pytest.fixture(scope="module") +def known_source(tmp_path_factory): + root = tmp_path_factory.mktemp("financial-demographics") + with pytest.MonkeyPatch.context() as patch: + arguments = _demographic_arguments(root, patch, zero=False) + live = survey.run_authenticated_survey_population( + **arguments, store_root=root / "store", clones=True, return_values=True + ) + yield live + values.source.verify_survey_population_preparation(live.preparation) + + +def _qualify(live, **kwargs): + return values.qualify_current_survey_predictors( + live.preparation, + live.allocated_population, + live.clone_population, + **kwargs, + ) + + +def test_demographic_features_fit_attach_and_require_replay(known_source): + live = known_source + original = live.clone_population + legacy = _qualify(live) + qualified = _qualify(live, demographic_conditioning=True) + matrix = graph.model_input.decode_recipient_matrix(qualified.matrix) + assert tuple(matrix.features) == values.DEMOGRAPHIC_FEATURES + assert tuple(graph.model_input.decode_recipient_matrix(legacy.matrix).features) == ( + values.FEATURES + ) + assert len(values.DEMOGRAPHIC_FEATURES) == 5 + assert not any("prior" in name for name in values.DEMOGRAPHIC_FEATURES) + pd.testing.assert_frame_equal(qualified.native_money, legacy.native_money) + details = qualified.evidence["model_judgments"]["demographic_conditioning"] + assert details["asec_sex_and_state_observation_year"] == 2025 + assert details["acs_sex_and_state_observation_year"] == 2024 + assert details["state_is_income_year_residence_claim"] is False + # The maintained fixture reverses ASEC source CSV rows before issuance. + # These independently expected native IDs therefore test the keyed join. + origins = qualified.origins + asec_ids = origins.loc[origins.source.eq("asec")].sort_values("native_person_id") + assert asec_ids.native_person_id.tolist() == [105, 106, 107, 108] + donor = qualified.donor_columns.loc[asec_ids.index] + np.testing.assert_array_equal(donor.survey_predictor_is_female, [0, 1, 0, 1]) + np.testing.assert_array_equal(donor.survey_predictor_state_fips, [6, 6, 36, 36]) + assert matrix.features.survey_predictor_is_female.eq(0).all() + assert matrix.features.survey_predictor_state_fips.eq(6).all() + assert qualified.donor_frame.resolve_weights("person").kind is WeightKind.DESIGN + np.testing.assert_array_equal( + qualified.donor_frame.resolve_weights("person").values, + legacy.donor_frame.resolve_weights("person").values, + ) + pins = {} + for edge in graph.host.current_survey_host_edges(): + receipt = live.manifest.node(edge.producer) + key = receipt.opaque_artifacts[edge.artifact] + pins[edge.name] = { + "producer_key": receipt.key, + "artifact_key": key, + "payload_sha256": graph.codec.sha(live.store.load_bytes(key)), + } + nodes = graph.current_survey_predictor_nodes( + qualified, original.frame, host_pins=pins, n_estimators=2 + ) + compiled = compile_graph( + replace(live.compiled.graph, nodes=(*live.compiled.graph.nodes, *nodes)) + ) + for cls in ( + graph.CurrentSurveyPredictorProjectionKernel, + graph.CurrentSurveyPredictorDonorFilterKernel, + graph.CurrentSurveyPredictorDonorColumnsKernel, + graph.CurrentSurveyPredictorAttachKernel, + ): + live.kernels.register( + cls( + live.preparation, + live.allocated_population, + original, + host_pins=pins, + n_estimators=2, + demographic_conditioning=True, + ) + ) + live.kernels.register(LegacyQRFTrainKernel()) + live.kernels.register(LegacyQRFApplyMatrixKernel()) + artifacts = [ + ( + graph.PROJECTION_NODE, + "projection", + graph.PROJECTION_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ( + graph.PROJECTION_NODE, + "matrix", + graph.model_input.RECIPIENT_MATRIX_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ] + for i in range(len(values.TARGETS)): + artifacts.extend( + ( + ( + f"{graph.FIT_PREFIX}.{i:03d}", + "model", + qrf_target.LEGACY_QRF_TARGET_TYPE, + LegacyQRFTrainKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "raw_draw", + graph.codec.RAW_TARGET_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "apply_state", + graph.MATRIX_APPLY_STATE_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ) + ) + results = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume=resume, + _population_observer=lambda name, population, observed=observed: ( + observed.__setitem__(name, population) + ), + ) + loaded = {} + for node_id, name, type_, cls in artifacts: + receipt = manifest.node(node_id) + payload = live.store.load_bytes(receipt.opaque_artifacts[name]) + survey._final_artifact( + manifest, + live.store, + node_id=node_id, + name=name, + type_=type_, + payload=payload, + capabilities=cls.capabilities, + ) + loaded[node_id, name] = payload + for i, target in enumerate(values.TARGETS): + payload = loaded[f"{graph.FIT_PREFIX}.{i:03d}", "model"] + model = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=graph.codec.sha(payload) + ) + assert model.target == target + assert model._target_model.columns == ( + *values.DEMOGRAPHIC_FEATURES, + *values.TARGETS[:i], + ) + final = observed[graph.ATTACH_NODE] + verification = { + "population": final, + "projection": loaded[graph.PROJECTION_NODE, "projection"], + "matrix": loaded[graph.PROJECTION_NODE, "matrix"], + "matrix_producer_key": manifest.node(graph.PROJECTION_NODE).key, + "raw_draws": tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ), + "apply_states": tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ), + "host_pins": pins, + "n_estimators": 2, + "demographic_conditioning": True, + } + result = graph.verify_materialized_current_survey_predictors( + live.preparation, live.allocated_population, original, **verification + ) + assert result["all_output_cells_available"] + assert not result["release_eligible"] + with pytest.raises(ValueError, match="MATERIALIZED_SOURCE"): + graph.verify_materialized_current_survey_predictors( + live.preparation, + live.allocated_population, + original, + **{**verification, "demographic_conditioning": False}, + ) + for entity in original.frame.entities: + before, after = original.frame.table(entity), final.frame.table(entity) + retained = [ + c for c in before if entity != "person" or c not in values.OUTPUTS + ] + pd.testing.assert_frame_equal( + before[retained], after[retained], check_exact=True + ) + assert final.version == original.version + assert final.weight_kind == original.weight_kind + assert final.mass_ledger == original.mass_ledger + for entity in original.design_weights: + np.testing.assert_array_equal( + final.design_weights[entity], original.design_weights[entity] + ) + np.testing.assert_array_equal( + final.frame.weights_for(entity).values, + original.frame.weights_for(entity).values, + ) + expected_owners = dict(original.owners) + expected_owners.update( + {("person", c): graph.ATTACH_NODE for c in values.OUTPUTS} + ) + assert dict(final.owners) == expected_owners + if resume == "require": + assert all(record.hit for record in manifest.nodes.values()) + results.append(final) + graph.replay.same_replayed_population(*results) + + +def test_unresolved_source_demographics_refuse_opt_in_and_preserve_default( + tmp_path, monkeypatch +): + arguments = _demographic_arguments(tmp_path, monkeypatch, unknown=True) + live = survey.run_authenticated_survey_population( + **arguments, store_root=tmp_path / "store", clones=True, return_values=True + ) + original = values.source._frame_identity(live.preparation.frame) + default = _qualify(live) + assert not default.demographic_conditioning + with pytest.raises( + ValueError, match="CURRENT_SURVEY_PREDICTOR_DEMOGRAPHIC_UNKNOWN" + ): + _qualify(live, demographic_conditioning=True) + assert values.source._frame_identity(live.preparation.frame) == original + assert _qualify(live).projection == default.projection + source_values = ( + values.observed_geography.demographics.qualify_current_asec_demographics( + live.preparation + ) + ) + assert source_values.person.is_female.isna().any() + assert source_values.household.state_fips.isna().any() + assert json.loads(source_values.receipt)["sex_unknown_persons"] == 1 + + +def test_last_money_owner_borrow_cannot_change_detached_demographic_values( + known_source, +): + live = known_source + captured, changed = [], [] + parent = values.source.asec_native._ISSUED[ + id(live.preparation._checked()[2].native[1]) + ][2].parent + ready_code = parent.ready.__code__ + + def profile(frame, event, arg): + if event != "return": + return + if frame.f_code is values._demographic_features.__code__ and arg is not None: + captured.append(arg[2]) + elif frame.f_code is ready_code and captured and not changed: + table = captured[-1].person + table.loc[table.index[0], "is_female"] = not bool(table.iloc[0].is_female) + changed.append(True) + + previous = sys.getprofile() + sys.setprofile(profile) + try: + with pytest.raises(ValueError, match="DEMOGRAPHIC_PROJECTION_CHANGED"): + _qualify(live, demographic_conditioning=True) + finally: + sys.setprofile(previous) + assert changed == [True] + # Only a detached projection changed; the live source can still qualify. + assert _qualify(live, demographic_conditioning=True).demographic_conditioning diff --git a/packages/microcosm-build/tests/test_us_current_survey_predictors.py b/packages/microcosm-build/tests/test_us_current_survey_predictors.py new file mode 100644 index 000000000..77ec4df96 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_predictors.py @@ -0,0 +1,589 @@ +"""Actual source owners and graph replay on pinned invented survey inputs. + +No production source issuer/readiness call is replaced. The maintained fixture +constructs its own real bytes and test pins before normal authentication. The two fixtures exercise both zero and nonconstant positive-regime forests. Neither +fixture establishes held-out fit quality. +""" + +import copy +import json +import pickle +from dataclasses import replace +from types import MappingProxyType, SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_survey_puf_transfer import _copy_population +from test_us_graph_survey_population import authenticated_arguments + +from microcosm.build.us_runtime import cps_carried_current as leaves +from microcosm.build.us_runtime import current_survey_predictors as values +from microcosm.build.us_runtime import graph_current_survey_predictors as graph +from microcosm.build.us_runtime import graph_survey_population as survey +from microcosm.build.us_runtime import puf_support as support +from microcosm.build.us_runtime import survey_population_preparation as preparation +from microcosm.fit import qrf, qrf_target +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import Frame, WeightKind +from microcosm.frame.bundle import _freeze_metadata +from microcosm.graph import compile_graph, run_graph +from microcosm.graph.keys import opaque_artifact_key + + +@pytest.mark.parametrize("operation", ["copy", "deepcopy", "pickle"]) +def test_metadata_copy_preserves_producer_identity(monkeypatch, operation): + metadata = _freeze_metadata({"origin": {"stages": ["survey", "clone"]}}) + # Exercise a cold class even if a preceding test already copied metadata. + # Python 3.14 annotation closures capture this mutable class namespace. + monkeypatch.delattr(type(metadata), "__slotnames__", raising=False) + namespace = dict(vars(type(metadata))) + producer = preparation._live() + + if operation == "pickle": + restored = pickle.loads(pickle.dumps(metadata)) + else: + restored = getattr(copy, operation)(metadata) + + assert restored == metadata + assert restored is not metadata + assert restored["origin"]["stages"] == ("survey", "clone") + with pytest.raises(TypeError): + restored["origin"]["stages"] = () + assert dict(vars(type(metadata))) == namespace + assert preparation._live() == producer + + +@pytest.mark.parametrize("method", ["__getitem__", "__reduce__"]) +def test_metadata_method_replacement_still_invalidates_producer(monkeypatch, method): + metadata = _freeze_metadata({"origin": "survey"}) + producer = preparation._live() + monkeypatch.setattr(type(metadata), method, lambda *args: None) + assert preparation._live() != producer + with pytest.raises( + preparation.SurveyPopulationPreparationError, match="PRODUCER_CHANGED" + ): + preparation._producer() + + +def test_survey_source_ids_preserve_zero_without_relaxing_physical_or_unique_axis(): + original = pd.Series([0, 2, 5], dtype="int64") + result = values._ids(original) + np.testing.assert_array_equal(result, np.array([0, 2, 5], dtype="int64")) + result[0] = 9 + assert original.iloc[0] == 0 + for bad in ( + pd.Series([-1, 0, 2], dtype="int64"), + pd.Series([0, 0, 2], dtype="int64"), + ): + with pytest.raises(ValueError, match="^CURRENT_SURVEY_PREDICTOR_ID_AXIS$"): + values._ids(bad) + for bad in ( + pd.Series([0.0, 1.0]), + pd.Series([False, True]), + pd.Series([0, 1], dtype="Int64"), + ): + with pytest.raises(ValueError, match="^CURRENT_SURVEY_PREDICTOR_ID_DTYPE$"): + values._ids(bad) + + +def _money(): + return { + "WSAL_VAL": np.array([0.0, 50000.0, 100000.0]), + "SEMP_VAL": np.array([-400.0, 0.0, 500.0]), + "INT_VAL": np.array([0.0, 120.0, 300.0]), + "DIV_VAL": np.array([40.0, 0.0, 75.0]), + "CAP_VAL": np.array([-700.0, 0.0, 1100.0]), + } + + +def test_shared_current_money_split_preserves_signs_and_partition_totals(): + source = _money() + result = leaves.derive_cps_current_predictor_leaves(source) + assert set(result) == set(leaves.CPS_CURRENT_PREDICTOR_PERSON_LEAVES) + # The legacy full leaf owner unpacks this mapping. Preserve its original + # insertion order so consumers constructing a DataFrame see the same schema. + assert tuple(result) == ( + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "long_term_capital_gains_before_response", + "short_term_capital_gains", + ) + np.testing.assert_array_equal( + result["self_employment_income_before_lsr"], source["SEMP_VAL"] + ) + np.testing.assert_allclose( + result["qualified_dividend_income"] + result["non_qualified_dividend_income"], + source["DIV_VAL"], + rtol=1e-15, + atol=0, + ) + np.testing.assert_allclose( + result["short_term_capital_gains"] + + result["long_term_capital_gains_before_response"], + source["CAP_VAL"], + rtol=1e-15, + atol=0, + ) + np.testing.assert_array_equal( + result["taxable_interest_income"], + source["INT_VAL"] * leaves.TAXABLE_INTEREST_FRACTION, + ) + result["employment_income_before_lsr"][0] = 10.0 + assert source["WSAL_VAL"][0] == 0.0 + + +@pytest.mark.parametrize("reverse", (False, True)) +def test_financial_attachment_conserves_native_and_drawn_interest(reverse): + # Descriptive, invented inputs exercise the numerical attachment only; + # constructing this dataclass does not authenticate a survey source. + ids = pd.Index([11, 12, 13], name="person_id") + money = pd.DataFrame(0.0, index=ids, columns=values.MONEY_FIELDS) + money["INT_VAL"] = [120.031, 0.0, np.nan] + origins = pd.DataFrame( + {"native_person_id": [101, 102, 201], "source": ["asec", "asec", "acs"]}, + index=ids, + ) + features = pd.DataFrame(0.0, index=ids[-1:], columns=values.FEATURES) + qualified = values.QualifiedSurveyPredictors( + projection=b"", + matrix=graph.model_input.encode_recipient_matrix( + features, entity="person", entity_ids=ids[-1:].to_numpy(dtype=" 0 + assert type(receipt) is MappingProxyType + with pytest.raises(TypeError): + receipt["minimum_age"] = 0 + assert receipt["minimum_age"] == 15 + probe.person.loc[acs_index, "age"] = 15 + probe.person.loc[acs_index, "AGEP"] = 15 + with pytest.raises(ValueError, match="ACS_ELIGIBLE_EARNINGS_UNKNOWN"): + values._acs_earnings(probe) + matrix = graph.model_input.decode_recipient_matrix(qualified.matrix) + invented_draw = pd.DataFrame( + { + target: np.arange(1.0, 1.0 + len(matrix.entity_ids)) * (i + 1.0) + for i, target in enumerate(values.TARGETS) + }, + index=matrix.features.index, + ) + original_join = values.complete_predictor_columns( + qualified, clone.frame, invented_draw + ) + reversed_join = values.complete_predictor_columns( + replace( + qualified, + native_money=qualified.native_money.iloc[::-1], + origins=qualified.origins.iloc[::-1], + ), + clone.frame, + invented_draw, + ) + for name in original_join: + pd.testing.assert_series_equal(original_join[name], reversed_join[name]) + completed_interest = qualified.native_money["INT_VAL"].fillna( + invented_draw[values.TARGETS[0]] + ) + expected_interest = completed_interest.reindex( + clone.frame.person[values.provenance.support_source_id_column("person")] + ).to_numpy() + np.testing.assert_allclose( + original_join["person", "taxable_interest_income"] + + original_join["person", "tax_exempt_interest_income"], + expected_interest, + rtol=1e-15, + atol=0, + ) + assert allocated.frame.resolve_weights("person").kind is WeightKind.IMPORTANCE + if nonconstant: + origins = qualified.origins + ids = origins.index[origins.source.eq("asec")] + # Join the current-money projection by native identity, never row position. + ordered = origins.loc[ids].sort_values("native_person_id").index + assert origins.loc[ordered, "native_person_id"].tolist() == [105, 106, 107, 108] + for field, expected in observations.items(): + np.testing.assert_array_equal( + qualified.native_money.loc[ordered, field], expected + ) + for target in values.TARGETS: + assert qualified.donor_columns[target].nunique() == 3 + weights = qualified.donor_frame.resolve_weights("person") + assert weights.kind is WeightKind.DESIGN and (weights.values > 0).all() + np.testing.assert_array_equal(weights.values, [2552.12, 2552.12, 100.0, 100.0]) + else: + assert qualified.donor_columns.loc[:, list(values.TARGETS)].eq(0).all().all() + pins = {} + for edge in graph.host.current_survey_host_edges(): + receipt = live.manifest.node(edge.producer) + key = receipt.opaque_artifacts[edge.artifact] + pins[edge.name] = { + "producer_key": receipt.key, + "artifact_key": key, + "payload_sha256": graph.codec.sha(live.store.load_bytes(key)), + } + nodes = graph.current_survey_predictor_nodes( + qualified, clone.frame, host_pins=pins, n_estimators=2 + ) + assert nodes[1].base == survey.CREATE_NODE + assert nodes[0].population == graph.DONOR_NODE + assert tuple(edge.name for edge in nodes[1].artifact_inputs) == ("preparation",) + assert all("state" not in c and "female" not in c for c in values.FEATURES) + compiled = compile_graph( + replace(live.compiled.graph, nodes=(*live.compiled.graph.nodes, *nodes)) + ) + # Actual compiler dependencies retain allocation evidence without an + # ordinary node in CREATE depending on its own downstream allocation. + assert graph.PROJECTION_NODE not in compiled.predecessors[survey.ALLOCATION_NODE] + assert graph.DONOR_NODE in compiled.predecessors[graph.PROJECTION_NODE] + assert survey.ALLOCATION_NODE in compiled.predecessors[graph.PROJECTION_NODE] + assert graph.PROJECTION_NODE not in compiled.predecessors[graph.DONOR_NODE] + for cls in ( + graph.CurrentSurveyPredictorProjectionKernel, + graph.CurrentSurveyPredictorDonorFilterKernel, + graph.CurrentSurveyPredictorDonorColumnsKernel, + graph.CurrentSurveyPredictorAttachKernel, + ): + live.kernels.register( + cls(preparation, allocated, clone, host_pins=pins, n_estimators=2) + ) + live.kernels.register(LegacyQRFTrainKernel()) + live.kernels.register(LegacyQRFApplyMatrixKernel()) + artifacts = [ + ( + graph.PROJECTION_NODE, + "projection", + graph.PROJECTION_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ( + graph.PROJECTION_NODE, + "matrix", + graph.model_input.RECIPIENT_MATRIX_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ] + for i in range(3): + artifacts.extend( + [ + ( + f"{graph.FIT_PREFIX}.{i:03d}", + "model", + qrf_target.LEGACY_QRF_TARGET_TYPE, + LegacyQRFTrainKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "raw_draw", + graph.codec.RAW_TARGET_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "apply_state", + graph.MATRIX_APPLY_STATE_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ] + ) + results = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume=resume, + _population_observer=lambda name, population, observed=observed: ( + observed.__setitem__(name, population) + ), + ) + donor_weights = observed[graph.DONOR_NODE].frame.resolve_weights("person") + assert donor_weights.kind is WeightKind.DESIGN + np.testing.assert_array_equal( + donor_weights.values, + qualified.donor_frame.resolve_weights("person").values, + ) + loaded = {} + for node_id, name, type_, cls in artifacts: + receipt = manifest.node(node_id) + key = opaque_artifact_key(receipt.key, name) + assert receipt.opaque_artifacts[name] == key + payload = live.store.load_bytes(key) + survey._final_artifact( + manifest, + live.store, + node_id=node_id, + name=name, + type_=type_, + payload=payload, + capabilities=cls.capabilities, + ) + loaded[node_id, name] = payload + for i, target in enumerate(values.TARGETS): + fit_id = f"{graph.FIT_PREFIX}.{i:03d}" + payload = loaded[fit_id, "model"] + # Actual typed producer/store verification above precedes pickle loading. + artifact = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=graph.codec.sha(payload) + ) + assert artifact.target == target + assert artifact._target_model.columns == ( + *values.FEATURES, + *values.TARGETS[:i], + ) + expected_regime = ( + qrf.Regime.ZERO_INFLATED_POSITIVE + if nonconstant + else qrf.Regime.DEGENERATE_ZERO + ) + assert artifact.regime == expected_regime + if nonconstant: + assert artifact._target_model.gate is not None + assert artifact._target_model.positive is not None + if resume == "require": + assert all(record.hit for record in manifest.nodes.values()) + raw = tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ) + states = tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ) + matrix_key = manifest.node(graph.PROJECTION_NODE).key + actual = observed[graph.ATTACH_NODE] + verification = dict( + population=actual, + projection=loaded[graph.PROJECTION_NODE, "projection"], + matrix=loaded[graph.PROJECTION_NODE, "matrix"], + matrix_producer_key=matrix_key, + raw_draws=raw, + apply_states=states, + host_pins=pins, + n_estimators=2, + ) + result = graph.verify_materialized_current_survey_predictors( + preparation, allocated, clone, **verification + ) + assert ( + result["all_output_cells_available"] is True + and result["release_eligible"] is False + ) + assert not actual.frame.person.loc[:, list(values.OUTPUTS)].isna().any().any() + actual_person = actual.frame.person + for _, rows in actual_person.groupby( + values.provenance.support_source_id_column("person") + ): + assert len(rows) == 2 + np.testing.assert_array_equal( + rows.loc[:, list(values.OUTPUTS)].iloc[0], + rows.loc[:, list(values.OUTPUTS)].iloc[1], + ) + for raw_name in ("WAGP", "SEMP", "ADJINC"): + pd.testing.assert_series_equal( + actual_person[raw_name], clone.frame.person[raw_name] + ) + puf_mask = support.puf_tax_detail_clone_mask( + actual.frame.table("tax_unit"), entity="tax_unit" + ) + features, _ = support._strict_recipient_predictor_surface( + actual.frame, + puf_mask, + support.PUF_TAX_DETAIL_DEFAULT_PREDICTORS, + person_outputs=(), + ) + assert features.shape[1] == 8 and np.isfinite(features.to_numpy()).all() + changed = _copy_population(actual) + changed.frame.person.loc[changed.frame.person.index[0], values.OUTPUTS[2]] += ( + 1.0 + ) + with pytest.raises(ValueError): + graph.verify_materialized_current_survey_predictors( + preparation, allocated, clone, **{**verification, "population": changed} + ) + with pytest.raises(ValueError, match="DRAW_MATRIX_BINDING"): + graph.read_current_survey_draws(qualified.matrix, "f" * 64, raw, states) + results.append(actual) + graph.replay.same_replayed_population(*results) + + +def test_predictor_projection_is_the_segmented_stream_of_the_owners_transport( + tmp_path, monkeypatch +): + """The projection carries one origins row per stacked person. + + That is a whole-roster document -- 85,702,912 bytes of origins rows at full + source, measured through this encoder, against the 64 MiB one accumulation + holds -- so it is issued as the owner's segmented stream. Its bytes are the + single-accumulation canonical encoding, unchanged at every segment size + down to the longest token, and the owner's total admits the full-source + document without any allocation here. + """ + live = survey.run_authenticated_survey_population( + **authenticated_arguments(tmp_path, monkeypatch), + clones=True, + return_values=True, + ) + qualified = values.qualify_current_survey_predictors( + live.preparation, live.allocated_population, live.clone_population + ) + document = json.loads(qualified.projection) + whole = survey._bounded_json(document, 64 * 1024**2) + assert qualified.projection == whole + encoder = json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ) + longest = max(len(piece.encode("utf-8")) for piece in encoder.iterencode(document)) + for segment in (longest, len(whole) // 3, len(whole)): + assert ( + survey._segmented_json( + document, segment=segment, maximum=preparation.MAX_ROSTER_BYTES + ) + == whole + ) + assert preparation.MAX_ROSTER_BYTES == 64 * preparation.MAX_PAYLOAD_BYTES + assert 85_702_912 > preparation.MAX_PAYLOAD_BYTES + assert 4 * 85_702_912 <= preparation.MAX_ROSTER_BYTES diff --git a/packages/microcosm-build/tests/test_us_current_survey_primary_family.py b/packages/microcosm-build/tests/test_us_current_survey_primary_family.py new file mode 100644 index 000000000..91324ea39 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_primary_family.py @@ -0,0 +1,250 @@ +"""Invented literal households; no population, model, weights, or source claims.""" + +import pytest + +from microcosm.build.us_runtime import current_survey_primary_family as family + + +def person(survey, line, role, age=30, marital=None, spouse=0, household="7"): + if survey == "asec": + return dict( + PERIDNUM=str(10**30 + line), + PH_SEQ=household, + A_LINENO=str(line), + A_EXPRRP=str(role), + A_AGE=str(age), + A_MARITL=str(7 if marital is None else marital), + A_SPOUSE=str(spouse), + ) + return dict( + SERIALNO=household, + SPORDER=str(line), + RELSHIPP=str(role), + AGEP=str(age), + MAR=str(5 if marital is None else marital), + ) + + +def classify(survey, people, *, head=1, married=True, household=None): + raw = ( + dict( + H_SEQ="00007", + H_HHTYPE="1", + H_LIVQRT="1", + HRHTYPE="1" if married else "3", + H_NUMPER=str(len(people)), + ) + if survey == "asec" + else dict(SERIALNO="7", TYPEHUGQ="1", NP=str(len(people))) + ) + if household: + raw.update(household) + return family.classify_primary_household( + survey, raw, people, qualified_head_line=head + ) + + +def couple(survey): + return [ + person(survey, 1, 1 if survey == "asec" else 20, marital=1, spouse=2), + person(survey, 2, 4 if survey == "asec" else 21, marital=1, spouse=1), + ] + + +@pytest.mark.parametrize("survey", ["asec", "acs"]) +@pytest.mark.parametrize( + "age,count,u6", + [(0, 1, 1), (5, 1, 1), (6, 1, 0), (17, 1, 0), (18, 0, 0), (30, 0, 0)], +) +def test_own_child_age_edges_and_one_primary_family(survey, age, count, u6): + people = couple(survey) + [person(survey, 3, 5 if survey == "asec" else 25, age)] + result = classify(survey, people) + assert result[family.PRIMARY] == 1 + assert result[family.CHILD_COUNT] == result[family.CHILD_U18] == count + assert result[family.CHILD_U6] == u6 + assert result[family.HOUSEHOLD_POPULATION] == 3 + assert ( + sum(result[name] for name in family.SIZE_BINS) + == result[family.HOUSEHOLD_COUNT] + == 1 + ) + + +@pytest.mark.parametrize( + "role,expected", [(25, 1), (26, 1), (27, 1), (30, 0), (32, 0), (35, 0), (36, 0)] +) +def test_acs_biological_adopted_step_and_unrelated_children(role, expected): + result = classify("acs", couple("acs") + [person("acs", 3, role, 5)]) + assert result[family.CHILD_COUNT] == expected + + +@pytest.mark.parametrize( + "survey,marital", + [ + ("asec", 1), + ("asec", 4), + ("asec", 5), + ("asec", 6), + ("acs", 1), + ("acs", 2), + ("acs", 3), + ("acs", 4), + ], +) +def test_ever_married_child_is_not_an_own_child_under18(survey, marital): + result = classify( + survey, + couple(survey) + + [person(survey, 3, 5 if survey == "asec" else 25, 17, marital)], + ) + assert result[family.CHILD_COUNT] == 0 + assert result["own_children_status"] == "qualified" + + +def test_unmarried_householder_with_married_subfamily_is_not_primary_couple(): + people = [ + person("asec", 1, 1, 70), + person("asec", 2, 5, 35, 1, 3), + person("asec", 3, 10, 35, 1, 2), + person("asec", 4, 7, 4), + ] + result = classify("asec", people, married=False) + assert result[family.PRIMARY] == result[family.CHILD_COUNT] == 0 + assert result[family.SPOUSE_PAIRS] == 1 + + +def test_two_resident_couples_are_one_primary_family_and_two_pairs(): + people = couple("asec") + [ + person("asec", 3, 5, 30, 1, 4), + person("asec", 4, 10, 30, 1, 3), + person("asec", 5, 7, 4), + ] + result = classify("asec", people) + assert result[family.PRIMARY] == 1 and result[family.SPOUSE_PAIRS] == 2 + assert result[family.CHILD_COUNT] == 0 + + +def test_same_sex_acs_spouse_and_unavailable_secondary_pair_count(): + people = couple("acs") + people[1]["RELSHIPP"] = "23" + result = classify("acs", people) + assert result[family.PRIMARY] == 1 and result[family.SPOUSE_PAIRS] is None + assert result["spouse_pairs_status"] == "unavailable_acs_secondary_spouse_pointer" + + +@pytest.mark.parametrize("pointer", ["", "16", "1", "0", "2.0"]) +def test_bad_primary_spouse_pointer_never_infers_marriage(pointer): + people = couple("asec") + people[0]["A_SPOUSE"] = pointer + result = classify("asec", people) + assert result[family.PRIMARY] is None and result[family.SPOUSE_PAIRS] is None + + +def test_bad_secondary_pointer_preserves_independently_qualified_primary(): + result = classify("asec", couple("asec") + [person("asec", 3, 10, 30, 1, 16)]) + assert result[family.PRIMARY] == 1 and result[family.SPOUSE_PAIRS] is None + + +@pytest.mark.parametrize("head", [None, 2, 12]) +def test_actual_qualified_head_cannot_be_replaced_by_position_or_spouse(head): + result = classify("acs", couple("acs"), head=head) + assert result[family.PRIMARY] is None + assert result["primary_family_status"] == "unavailable_qualified_householder" + + +@pytest.mark.parametrize( + "survey,override,expected", + [ + ("acs", {"TYPEHUGQ": "2"}, "excluded_group_quarters"), + ("acs", {"TYPEHUGQ": "3"}, "excluded_group_quarters"), + ("acs", {"TYPEHUGQ": ""}, "unavailable_housing_universe"), + ("asec", {"HRHTYPE": "9", "H_LIVQRT": "11"}, "excluded_nonhousing_unit"), + ("asec", {"HRHTYPE": "10", "H_LIVQRT": "12"}, "excluded_nonhousing_unit"), + ("asec", {"HRHTYPE": "9"}, "unavailable_household_type_conflict"), + ("asec", {"H_LIVQRT": "11"}, "unavailable_household_type_conflict"), + ("asec", {"H_HHTYPE": ""}, "unavailable_housing_universe"), + ("asec", {"H_HHTYPE": "2"}, "excluded_noninterview"), + ], +) +def test_gq_noninterview_and_ambiguous_household_universe(survey, override, expected): + people = couple(survey) + if survey == "acs" and override.get("TYPEHUGQ") in ("2", "3"): + people = [person("acs", 1, int(override["TYPEHUGQ"]) + 35)] + result = classify(survey, people, household=override) + assert result["household_status"] == expected + assert result[family.HOUSEHOLD_COUNT] == ( + 0 if expected.startswith("excluded") else None + ) + assert result[family.PRIMARY] == result[family.HOUSEHOLD_COUNT] + + +@pytest.mark.parametrize("size", [1, 2, 3, 4, 5, 6, 7, 12, 16]) +def test_size_bins_partition_complete_housing_unit_households(size): + people = [person("asec", i, 1 if i == 1 else 10) for i in range(1, size + 1)] + result = classify("asec", people, married=False) + assert result[family.HOUSEHOLD_POPULATION] == size + assert [result[column] for column in family.SIZE_BINS] == [ + int(i == min(size, 7)) for i in range(1, 8) + ] + + +@pytest.mark.parametrize("field,value", [("AGEP", ""), ("MAR", ""), ("MAR", "NA")]) +def test_unknown_own_child_age_or_marital_evidence_is_not_zero(field, value): + child = person("acs", 3, 26, 5) + child[field] = value + result = classify("acs", couple("acs") + [child]) + assert result[family.PRIMARY] == 1 and result[family.CHILD_COUNT] is None + + +def test_no_unrelated_child_age_or_tax_role_can_manufacture_own_child(): + child = person("acs", 3, 36, 4) + child.update(is_tax_unit_dependent=True, PEPAR1="1", person_spm_unit_id="7") + result = classify("acs", couple("acs") + [child]) + assert result[family.CHILD_COUNT] == 0 + + +def test_arbitrary_size_ids_exact_line_join_and_order_invariance(): + people = couple("asec") + raw_id = str(2**130 + 17) + for row in people: + row["PH_SEQ"] = raw_id + first = classify("asec", people, household={"H_SEQ": raw_id}) + assert first == classify( + "asec", list(reversed(people)), household={"H_SEQ": raw_id} + ) + assert first[family.PRIMARY] == 1 + people[1]["PH_SEQ"] = str(int(raw_id) + 1) + with pytest.raises(ValueError, match="HOUSEHOLD_JOIN"): + classify("asec", people, household={"H_SEQ": raw_id}) + + +def test_incomplete_roster_and_unnamed_relationship_preserve_unknown(): + result = classify("asec", couple("asec"), household={"H_NUMPER": "3"}) + assert ( + result[family.HOUSEHOLD_COUNT] == 1 + and result[family.HOUSEHOLD_POPULATION] is None + ) + assert result[family.PRIMARY] is None + people = couple("asec") + [person("asec", 3, 6, 5)] + assert classify("asec", people)[family.PRIMARY] is None + + +def test_acs_relationship_universe_and_spouse_partner_conflicts(): + result = classify("acs", couple("acs"), household={"TYPEHUGQ": "2"}) + assert result[family.HOUSEHOLD_COUNT] is None + result = classify("acs", [person("acs", 1, 37)]) + assert result[family.HOUSEHOLD_COUNT] is None + result = classify("acs", couple("acs") + [person("acs", 3, 22)]) + assert result[family.PRIMARY] is None + + +def test_asec_reference_without_relatives_cannot_have_a_primary_spouse(): + people = couple("asec") + people[0]["A_EXPRRP"] = "2" + result = classify("asec", people) + assert result[family.PRIMARY] is None + assert result[family.SPOUSE_PAIRS] == 1 + assert ( + result["primary_family_status"] + == "unavailable_householder_relationship_conflict" + ) diff --git a/packages/microcosm-build/tests/test_us_current_survey_primary_family_source.py b/packages/microcosm-build/tests/test_us_current_survey_primary_family_source.py new file mode 100644 index 000000000..6f48c3860 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_primary_family_source.py @@ -0,0 +1,239 @@ +"""Genuine source issuers over small, wholly invented original survey archives.""" + +import hashlib +import io +import json +import shutil +import sys +from dataclasses import replace + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_survey_primary_family_source as owner + + +def primary_family_source_arguments(tmp_path, patch): + import test_us_survey_population_preparation as fixture + from test_us_acs_person_coverage_authentication import build_fixture + from test_us_current_asec_demographics import _demographic_arguments + + from microcosm.build.us_runtime import asec_demographic_source as demographics + from microcosm.build.us_runtime import asec_person_income_source as restoration + from microcosm.build.us_runtime import native_household_origin as origin + + build = build_fixture + + def archives(*args, **kwargs): + # Add an actual resident spouse before archive pins or owners exist. + h = pd.read_csv( + io.BytesIO(kwargs["household_members"][0].data), + dtype=str, + keep_default_na=False, + ) + p = pd.read_csv( + io.BytesIO(kwargs["person_members"][0].data), + dtype=str, + keep_default_na=False, + ) + p["MAR"] = "5" + hh = "2024HU0000001" + head = p.SERIALNO.eq(hh) & p.SPORDER.eq("1") + p.loc[head, "MAR"] = "1" + child = p.SERIALNO.eq(hh) & p.SPORDER.eq("2") + p.loc[child, "AGEP"] = "5" + spouse = p.loc[head].copy(deep=True) + spouse["SPORDER"], spouse["RELSHIPP"], spouse["AGEP"] = "3", "23", "32" + p = pd.concat([p, spouse], ignore_index=True) + h.loc[h.SERIALNO.eq(hh), "NP"] = "3" + kwargs["household_members"] = ( + fixture.Member("psam_husa.csv", fixture._csv(h.to_dict("records"))), + ) + kwargs["person_members"] = ( + fixture.Member( + "psam_pusa.csv", fixture._csv(p.iloc[::-1].to_dict("records")) + ), + ) + return build(*args, **kwargs) + + patch.setattr(fixture, "build_fixture", archives) + arguments = _demographic_arguments(tmp_path, patch) + folder = arguments["source_dir"] / "asec" + pins, paths = [], {} + for year, member, archive, *_ in owner.reader.source_reader.asec._MEMBER_PINS: + path = folder / member + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + raw["A_MARITL"], raw["A_SPOUSE"] = "7", "0" + raw.to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.reader.source_reader.asec, restoration, demographics): + patch.setattr(module, "_MEMBER_PINS", tuple(pins)) + restored = tmp_path / "primary-restored-income" + restoration.restore_asec_person_income_source( + folder / "parent.h5", + folder / "household-attachment.h5", + member_paths=paths, + output_dir=restored, + ) + shutil.copyfile( + restored / restoration.CHECKPOINT_FILENAME, + folder / "person-income-attachment.h5", + ) + path = folder / "hhpub25.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + raw["HRHTYPE"] = "3" + raw.to_csv(path, index=False) + payload = path.read_bytes() + patch.setattr( + origin, + "_ASEC_MEMBER_PINS", + tuple( + replace( + pin, + member_sha256=hashlib.sha256(payload).hexdigest(), + size_bytes=len(payload), + ) + for pin in origin._ASEC_MEMBER_PINS + ), + ) + return arguments + + +@pytest.fixture(scope="module") +def actual(tmp_path_factory): + with pytest.MonkeyPatch.context() as patch: + arguments = primary_family_source_arguments( + tmp_path_factory.mktemp("primary-family"), patch + ) + preparation = owner.source.prepare_authenticated_survey_population(**arguments) + original = owner.source._frame_identity(preparation._checked()[2].frame) + yield preparation, arguments + assert owner.source._frame_identity(preparation._checked()[2].frame) == original + + +def test_original_archives_qualify_primary_children_and_exclude_gq(actual): + preparation, _ = actual + result = owner.qualify_current_survey_primary_family(preparation) + family = owner.classifier + table = result.household + married = table.loc[table[family.PRIMARY].eq(1)] + assert len(married) == 1 + assert married[family.CHILD_COUNT].tolist() == [1] + assert married[family.CHILD_U6].tolist() == [1] + assert table[family.HOUSEHOLD_COUNT].sum() == 4 + assert table[family.HOUSEHOLD_POPULATION].sum() == 8 + assert table["household_status"].eq("excluded_group_quarters").sum() == 2 + assert table[family.HOUSEHOLD_COUNT].equals( + table[list(family.SIZE_BINS)].sum(axis=1).astype("Int64") + ) + asec = result.origins.source.eq("asec") + assert table.loc[asec, family.SPOUSE_PAIRS].tolist() == [0, 0] + assert table.loc[~asec, family.SPOUSE_PAIRS].isna().all() + receipt = json.loads(result.receipt) + assert receipt["readsets"] == [list(columns) for columns in family.READSETS] + assert receipt["projection_sha256"] == owner.roles._sha(result.projection) + assert not any( + receipt[name] + for name in ( + "source_admission_issued", + "population_admission_issued", + "graph_attachment_qualified", + "release_eligible", + ) + ) + assert ( + owner.qualify_current_survey_primary_family(preparation).receipt + == result.receipt + ) + result.household.iloc[0, result.household.columns.get_loc(family.PRIMARY)] = 99 + fresh = owner.qualify_current_survey_primary_family(preparation) + assert not fresh.household[family.PRIMARY].eq(99).any() + + +def test_copied_preparation_cannot_supply_source_authority(actual): + import copy + + preparation, _ = actual + with pytest.raises(ValueError): + owner.qualify_current_survey_primary_family(copy.copy(preparation)) + + +@pytest.mark.parametrize("mutation", ["metric", "status", "raw", "function"]) +def test_final_owner_callback_cannot_change_projection_or_literal_evidence( + actual, monkeypatch, mutation +): + preparation, _ = actual + qualify_code = owner.qualify_current_survey_primary_family.__code__ + checked_code = type(preparation)._checked.__code__ + fired = [] + + def profile(frame, event, arg): + if ( + event != "return" + or frame.f_code is not checked_code + or frame.f_back is None + or frame.f_back.f_code is not qualify_code + ): + return + local = frame.f_back.f_locals + if "result" not in local: + return + fired.append(True) + if mutation in ("metric", "status"): + column = ( + owner.classifier.PRIMARY if mutation == "metric" else "household_status" + ) + local["result"].household.iloc[ + 0, local["result"].household.columns.get_loc(column) + ] = 99 if mutation == "metric" else "fabricated" + elif mutation == "raw": + next(iter(local["selected"]["asec_person"].values()))["A_SPOUSE"] = "16" + else: + monkeypatch.setattr( + owner.classifier, "METRICS", owner.classifier.METRICS[:-1] + ) + + previous = sys.getprofile() + sys.setprofile(profile) + try: + if mutation == "raw": + # Detached literal mutation after its freeze cannot change output. + result = owner.qualify_current_survey_primary_family(preparation) + assert result.household[owner.classifier.SPOUSE_PAIRS].dropna().eq(0).all() + else: + with pytest.raises( + ValueError, match="IMPLEMENTATION_CHANGED|FINAL_PROJECTION" + ): + owner.qualify_current_survey_primary_family(preparation) + finally: + sys.setprofile(previous) + assert fired == [True] + + +def test_original_member_change_refuses_before_classification(tmp_path, monkeypatch): + arguments = primary_family_source_arguments(tmp_path, monkeypatch) + preparation = owner.source.prepare_authenticated_survey_population(**arguments) + path = arguments["source_dir"] / "asec" / "pppub25.csv" + payload = path.read_bytes() + path.write_bytes(payload.replace(b"A_SPOUSE", b"A_SPOUSX", 1)) + with pytest.raises(ValueError): + owner.qualify_current_survey_primary_family(preparation) + + +def test_readset_extension_requires_original_join_fields(actual): + preparation, _ = actual + readsets = list(owner.classifier.READSETS) + readsets[1] = tuple(name for name in readsets[1] if name != "PH_SEQ") + with pytest.raises(ValueError, match="ORIGINAL_READSETS"): + owner.reader._original_columns(preparation, readsets=tuple(readsets)) diff --git a/packages/microcosm-build/tests/test_us_current_survey_property_graph.py b/packages/microcosm-build/tests/test_us_current_survey_property_graph.py new file mode 100644 index 000000000..3bba5a9bd --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_property_graph.py @@ -0,0 +1,566 @@ +"""Invented source owners and the actual property fragment; no native/engine.""" + +import json +from dataclasses import FrozenInstanceError, replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_property_income_sources import actual # noqa: F401 + +from microcosm.build.us_runtime import graph_current_survey_property as graph +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import WeightKind +from microcosm.graph import compile_graph, run_graph +from microcosm.graph import population as population_ops + +financial = graph.financial +shared = graph.shared +OPTIONS = dict(scales=(1.0, 1.0, 1.0, 1.0), atol=1e-10, rtol=1e-12, n_estimators=2) + + +def test_joint_fit_exclusion_preserves_independent_asec_components(actual): # noqa: F811 + """Actual invented source qualification and pure attachment, without fitting. + + Native fixture person107 reports interest/dividends but survivor source8 + excludes the person from the joint fit. Those independent observations + remain usable on both clones. ACS draw inputs below are invented numbers, + not fitted or authenticated model artifacts; they only exercise the + separate recipient assignment in the pure attachment function. + """ + from microcosm.build.us_runtime import current_property_income_sources as sources + from microcosm.build.us_runtime import graph_property_tax_leaves as tax + + live, qualified, _ = actual + basis = qualified.donor_basis + excluded = basis.person.index[basis.person.native_person_id.eq(107)] + assert len(excluded) == 1 + origin = excluded[0] + assert basis.exclusions.loc[origin, "survivor_possible_property"] + assert not basis.person.loc[origin, "joint_component_fit_eligible"] + assert origin not in qualified.donor_columns.index + assert origin not in qualified.recipient_columns.index + assert basis.provenance.loc[origin, "interest.TRDINT_VAL_amount_known"] + assert basis.provenance.loc[origin, "dividend.DIV_VAL_amount_known"] + assert basis.provenance.loc[origin, "dividend.SUR_SC2_code"] == 8 + source_seal = sources.property_income_sources_seal(qualified) + frame = live.clone_population.frame + frame_seal = sources._frame_seal(frame) + + draws = pd.DataFrame( + np.tile([11.0, 7.0, 13.0, -2.0], (len(qualified.recipient_columns), 1)), + index=qualified.recipient_columns.index, + columns=graph.PROPERTY_DRAW_COLUMNS, + ) + reconciled = graph._reconciliation_result( + qualified, graph.PropertyIncomeOptions(**OPTIONS), draws + ) + legacy = pd.DataFrame(0.0, index=draws.index, columns=shared.TARGETS) + columns = graph.complete_property_columns( + qualified, frame, draws, reconciled, legacy + ) + people = frame.person.copy(deep=True) + for (entity, name), values in columns.items(): + assert entity == "person" + np.testing.assert_array_equal(values.index, people.person_id) + people[name] = values.array.copy() + split = tax.split_property_tax_leaves(people) + source_ids = people[shared.provenance.support_source_id_column("person")] + paired = people.loc[source_ids.eq(origin)] + assert len(paired) == 2 + assert set(paired[shared.provenance.support_clone_index_column("person")]) == { + 0, + 1, + } + for component, known_column, primary, complement in ( + ( + graph.PROPERTY_COMPONENTS[0], + "interest.TRDINT_VAL_amount", + *tax.TAX_LEAF_COLUMNS[:2], + ), + ( + graph.PROPERTY_COMPONENTS[2], + "dividend.DIV_VAL_amount", + *tax.TAX_LEAF_COLUMNS[2:], + ), + ): + amount = np.float64(basis.provenance.loc[origin, known_column]) + assert amount == 500.0 + np.testing.assert_array_equal( + paired[component].to_numpy().view("uint64"), + np.repeat(amount, 2).view("uint64"), + ) + parts = split.loc[paired.person_id, [primary, complement]] + assert np.isfinite(parts.to_numpy()).all() + np.testing.assert_array_equal(parts.sum(axis=1), np.repeat(amount, 2)) + assert paired[list(graph.PROPERTY_DRAW_COLUMNS)].isna().all().all() + # Every original ASEC component, including unknowns, survives unfiltered. + for original_id, row in basis.person.iterrows(): + pair = people.loc[source_ids.eq(original_id)] + for component in graph.PROPERTY_COMPONENTS: + np.testing.assert_array_equal(pair[component], np.repeat(row[component], 2)) + assert sources.property_income_sources_seal(qualified) == source_seal + assert sources._frame_seal(frame) == frame_seal + + +def test_explicit_frozen_options_and_complete_owned_roster(): + options = graph.PropertyIncomeOptions(**OPTIONS) + assert graph.codec.decode_json(options.to_bytes()) == graph.codec.decode_json( + graph.codec.encode_json(OPTIONS) + ) + with pytest.raises(FrozenInstanceError): + options.atol = 1 + assert ( + len(graph.owned_columns()) + == len({o.column for o in graph.owned_columns()}) + == 24 + ) + assert not set(shared.OUTPUTS) & {o.column for o in graph.owned_columns()} + with pytest.raises(TypeError): + graph.PropertyIncomeOptions() + + +@pytest.mark.parametrize( + "key,value", + [ + ("scales", (1.0, 1.0, 1.0)), + ("scales", (1.0, 1.0, 1.0, 0.0)), + ("scales", (1.0, 1.0, 1.0, float("inf"))), + ("atol", -1.0), + ("rtol", float("nan")), + ("atol", 1), + ("n_estimators", 0), + ("n_estimators", True), + ], +) +def test_invalid_options_refuse(key, value): + with pytest.raises((ValueError, TypeError)): + graph.PropertyIncomeOptions(**{**OPTIONS, key: value}) + + +def _pins(live): + result = {} + for edge in graph.host.current_survey_host_edges(): + record = live.manifest.node(edge.producer) + key = record.opaque_artifacts[edge.artifact] + result[edge.name] = { + "producer_key": record.key, + "artifact_key": key, + "payload_sha256": graph.codec.sha(live.store.load_bytes(key)), + } + return result + + +@pytest.fixture(scope="module") +def complete(actual, tmp_path_factory): # noqa: F811 + live, qualified, _ = actual + pins = _pins(live) + options = graph.PropertyIncomeOptions(**OPTIONS) + legacy_nodes = financial.current_survey_predictor_nodes( + qualified.shared_predictors, + live.clone_population.frame, + host_pins=pins, + n_estimators=2, + ) + nodes = graph.current_survey_property_nodes( + qualified, live.clone_population.frame, host_pins=pins, options=options + ) + compiled = compile_graph( + replace( + live.compiled.graph, + nodes=(*live.compiled.graph.nodes, *legacy_nodes, *nodes), + ) + ) + for cls in ( + financial.CurrentSurveyPredictorProjectionKernel, + financial.CurrentSurveyPredictorDonorFilterKernel, + financial.CurrentSurveyPredictorDonorColumnsKernel, + financial.CurrentSurveyPredictorAttachKernel, + ): + live.kernels.register( + cls( + live.preparation, + live.allocated_population, + live.clone_population, + host_pins=pins, + n_estimators=2, + ) + ) + live.kernels.register(LegacyQRFTrainKernel()) + live.kernels.register(LegacyQRFApplyMatrixKernel()) + graph.register_property_kernels( + live.kernels, + live.preparation, + live.allocated_population, + live.clone_population, + host_pins=pins, + options=options, + ) + observed = {} + manifest = run_graph( + compiled, + sources=live.sources, + store=live.store, + kernels=live.kernels, + _population_observer=lambda node_id, p: observed.__setitem__(node_id, p), + ) + loaded = { + (node_id, name): live.store.load_bytes(key) + for node_id, record in manifest.nodes.items() + for name, key in record.opaque_artifacts.items() + } + results = graph.reconstruct_property_results( + qualified, + live.clone_population.frame, + host_pins=pins, + options=options, + artifacts=loaded, + legacy_matrix_producer_key=manifest.node(financial.PROJECTION_NODE).key, + ) + again_observed = {} + again = run_graph( + compiled, + sources=live.sources, + store=live.store, + kernels=live.kernels, + resume="require", + _population_observer=lambda node_id, p: again_observed.__setitem__(node_id, p), + ) + assert again.key == manifest.key and all(n.hit for n in again.nodes.values()) + metadata = { + "scope": "invented full source qualification plus actual legacy10/property16 graph; no native/country engine", + "nodes": len(compiled.order), + "property_nodes": len(nodes), + "cold_property_hits": {n.id: manifest.node(n.id).hit for n in nodes}, + "required_hits": len(again.nodes), + "source_people": len(qualified.origins), + "clone_people": len(live.clone_population.frame.person), + "donor_people": len(qualified.donor_columns), + "recipient_people": len(qualified.recipient_columns), + } + ( + tmp_path_factory.mktemp("property-fragment-metadata") / "acceptance.json" + ).write_text(json.dumps(metadata, indent=2)) + return ( + live, + qualified, + pins, + options, + compiled, + manifest, + observed, + loaded, + results, + again_observed, + ) + + +def test_sixteen_actual_nodes_replay_and_reconstructed_populations(complete): + ( + live, + qualified, + pins, + options, + compiled, + manifest, + observed, + loaded, + results, + again, + ) = complete + assert len(results) == 8 + assert sum(n.id.startswith("survey_property.") for n in compiled.graph.nodes) == 16 + assert financial.ATTACH_NODE in compiled.predecessors[graph.ATTACH_NODE] + assert set(shared.OUTPUTS).issubset( + next( + s.columns + for s in compiled.graph.node(graph.ATTACH_NODE).inputs + if s.entity == "person" + ) + ) + expected = {} + for node_id in compiled.order: + node = compiled.graph.node(node_id) + if node_id in results: + parent = ( + expected[node.base] + if node.structural is graph.StructuralDelta.FILTER + else expected.get(compiled.versions[node_id]) + ) + if parent is None: + # The first property operation in the pre-existing donor or + # clone version receives the actual independently checked prefix. + parent = ( + observed[financial.ATTACH_NODE] + if node_id == graph.ATTACH_NODE + else observed[financial.DONOR_NODE] + ) + result = results[node_id] + if node.structural is graph.StructuralDelta.FILTER: + mask = result.keep.reindex(parent.frame.person.person_id).to_numpy( + dtype=bool + ) + result = replace(result, frame=parent.frame.select(mask), keep=None) + current = population_ops.patch(parent, node, result) + graph.replay.same_replayed_population(current, observed[node_id]) + expected[compiled.versions[node_id]] = current + elif node_id.startswith(graph.PREFIX + "."): + assert node_id in { + f"{graph.PREFIX}.{kind}.{i:03d}" + for kind in ("fit", "apply") + for i in range(4) + } + else: + expected[compiled.versions[node_id]] = observed[node_id] + graph.replay.same_replayed_population(observed[node_id], again[node_id]) + final = observed[graph.ATTACH_NODE] + receipt = graph.verify_materialized_property_income( + live.preparation, + live.allocated_population, + live.clone_population, + legacy_population=observed[financial.ATTACH_NODE], + population=final, + host_pins=pins, + options=options, + artifacts=loaded, + legacy_matrix_producer_key=manifest.node(financial.PROJECTION_NODE).key, + ) + assert ( + receipt["tax_split_rebased"] is False + and receipt["capital_gains"] == graph.CAP_LIMITATION + ) + + +def test_exact_clone_pairs_unknowns_and_asec_qualified_values(complete): + live, qualified, _, _, _, _, observed, *_ = complete + final = observed[graph.ATTACH_NODE].frame + legacy = observed[financial.ATTACH_NODE].frame + for entity in legacy.entities: + pd.testing.assert_frame_equal( + final.table(entity)[list(legacy.table(entity))], legacy.table(entity) + ) + assert final.metadata == legacy.metadata and final.mass_log == legacy.mass_log + for entity in legacy.weighted_entities: + assert final.weights_for(entity).kind is legacy.weights_for(entity).kind + np.testing.assert_array_equal( + final.weights_for(entity).values, legacy.weights_for(entity).values + ) + ids = final.person[shared.provenance.support_source_id_column("person")].to_numpy() + columns = [o.column for o in graph.owned_columns()] + for origin in qualified.origins.index: + pair = final.person.loc[ids == origin, columns].reset_index(drop=True) + pd.testing.assert_series_equal(pair.iloc[0], pair.iloc[1], check_names=False) + basis = qualified.donor_basis.person + for origin in basis.index: + pair = final.person.loc[ids == origin] + for name in (*graph.PROPERTY_COMPONENTS, graph.PROPERTY_REPORTED_TOTAL): + np.testing.assert_array_equal( + pair[name].to_numpy(), np.repeat(basis.loc[origin, name], 2) + ) + assert pair[list(graph.PROPERTY_DRAW_COLUMNS)].isna().all().all() + excluded = qualified.recipient_diagnostics.index[ + ~qualified.recipient_diagnostics.eligible_recipient + ] + mask = np.isin(ids, excluded) + assert mask.sum() > 0 + assert ( + final.person.loc[ + mask, [*graph.PROPERTY_COMPONENTS, graph.PROPERTY_REPORTED_TOTAL] + ] + .isna() + .all() + .all() + ) + assert ( + not final.person.loc[ + mask, ["property_anchor_known", "property_components_known"] + ] + .to_numpy() + .any() + ) + assert ( + observed[graph.RECIPIENT_NODE].frame.person.person_id.to_numpy() + == qualified.recipient_columns.index.to_numpy() + ).all() + assert ( + observed[graph.DONOR_NODE].frame.resolve_weights("person").kind + is WeightKind.DESIGN + ) + assert ( + observed[graph.DONOR_NODE].frame.person.person_id.to_numpy() + == qualified.donor_columns.index.to_numpy() + ).all() + + +@pytest.mark.parametrize( + "which", ["raw", "state", "summary", "reconciliation", "projection"] +) +def test_tampered_graph_artifacts_refuse(complete, which): + live, q, pins, options, _, manifest, _, artifacts, *_ = complete + copied = dict(artifacts) + key = { + "raw": (graph.PREFIX + ".apply.001", "raw_draw"), + "state": (graph.PREFIX + ".apply.001", "apply_state"), + "summary": (graph.PREFIX + ".draws", "summary"), + "reconciliation": (graph.PREFIX + ".reconcile", "summary"), + "projection": (graph.PROJECTION_NODE, "projection"), + }[which] + if which == "raw": + raw = graph.codec.read_raw_target( + copied[key], + target=graph.PROPERTY_COMPONENTS[1], + index=q.recipient_frame.person.index, + ).copy() + raw[0] += 1 + copied[key] = graph.codec.encode_raw_target( + raw, + target=graph.PROPERTY_COMPONENTS[1], + index=q.recipient_frame.person.index, + ) + else: + data = graph.codec.decode_json(copied[key]) + data["unexpected"] = True + copied[key] = graph.codec.encode_json(data) + with pytest.raises((ValueError, KeyError)): + graph.reconstruct_property_results( + q, + live.clone_population.frame, + host_pins=pins, + options=options, + artifacts=copied, + legacy_matrix_producer_key=manifest.node(financial.PROJECTION_NODE).key, + ) + + +def test_scales_change_only_reconciliation_and_attachment_declarations(complete): + live, q, pins, options, *_ = complete + old = graph.current_survey_property_nodes( + q, live.clone_population.frame, host_pins=pins, options=options + ) + new = graph.current_survey_property_nodes( + q, + live.clone_population.frame, + host_pins=pins, + options=replace(options, scales=(2.0, 1.0, 1.0, 1.0)), + ) + assert [a.id for a, b in zip(old, new, strict=True) if a != b] == [ + graph.PREFIX + ".reconcile", + graph.ATTACH_NODE, + ] + + +def test_origin_identity_refusal_and_order_independence(complete): + live, q, pins, options, _, manifest, observed, artifacts, *_ = complete + ids, axis = graph._clone_lookup(q, live.clone_population.frame) + reordered = replace(q, origins=q.origins.iloc[::-1]) + assert np.array_equal( + graph._clone_lookup(reordered, live.clone_population.frame)[0], ids + ) + bad = q.origins.copy(deep=True) + bad.iloc[0, bad.columns.get_loc("native_person_id")] += 1 + with pytest.raises(ValueError, match="CLONE_ORIGIN_IDENTITY"): + graph._clone_lookup(replace(q, origins=bad), live.clone_population.frame) + assert axis.dtype == np.dtype("int64") + + +@pytest.mark.parametrize("target", ["donor_frame", "recipient_frame"]) +def test_empty_branch_refuses_without_model_defaults(complete, target): + live, q, pins, options, *_ = complete + with pytest.raises(ValueError, match="EMPTY_MODEL_BRANCH"): + graph.current_survey_property_nodes( + replace(q, **{target: None}), + live.clone_population.frame, + host_pins=pins, + options=options, + ) + + +def test_declaration_without_source_execution(monkeypatch): + from microcosm.frame import US_SCHEMA, Frame, Weights + + tables = { + e: pd.DataFrame( + { + US_SCHEMA.entity_id_column(e): np.array([1], dtype="int64"), + "fixture_" + e: np.array([1.0]), + } + ) + for e in US_SCHEMA.entities + } + for e in US_SCHEMA.group_entities: + tables["person"][US_SCHEMA.membership_column(e)] = np.array([1], dtype="int64") + tables["person"]["employment_income"] = 5.0 + frame = Frame( + tables, US_SCHEMA, {"household": Weights(np.array([1.0]), WeightKind.DESIGN)} + ) + features = (*shared.FEATURES, graph.PROPERTY_REPORTED_TOTAL) + q = SimpleNamespace( + source_frame=frame, + donor_frame=frame, + recipient_frame=frame, + donor_columns=pd.DataFrame(columns=(*features, *graph.PROPERTY_COMPONENTS)), + recipient_columns=pd.DataFrame(columns=features), + shared_predictors=SimpleNamespace( + donor_frame=frame, + demographic_conditioning=False, + geography_config_payload=None, + ), + ) + # Isolate declaration shape only. No fixture object claims source authority; + # the full graph fixture below uses the actual source qualifiers unchanged. + monkeypatch.setattr(graph, "_params", lambda *args: {"protocol": graph.PROTOCOL}) + nodes = graph.current_survey_property_nodes( + q, frame, host_pins={}, options=graph.PropertyIncomeOptions(**OPTIONS) + ) + assert len(nodes) == 16 + assert set(nodes[-1].params) == {"protocol", *OPTIONS} + + context_tables = {} + for entity in frame.entities: + structural = [US_SCHEMA.entity_id_column(entity)] + if entity == "person": + structural.extend( + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + ) + context_tables[entity] = frame.table(entity)[ + structural + [c for c in frame.table(entity) if c not in structural] + ].copy() + for name in shared.OUTPUTS: + context_tables["person"][name] = 0.0 + context = graph.KernelContext( + node=nodes[-1], + tables=context_tables, + weights={e: frame.resolve_weights(e) for e in frame.entities}, + strata=frame.strata, + params=nodes[-1].params, + rng=None, + ) + graph._clone_context(context, frame) + assert set(shared.OUTPUTS).issubset( + next(x.columns for x in nodes[-1].inputs if x.entity == "person") + ) + + +@pytest.mark.parametrize("field", ["property_reported_total", shared.OUTPUTS[0]]) +def test_final_verifier_refuses_changed_owned_or_retained_values(complete, field): + from test_us_current_survey_puf_transfer import _copy_population + + live, _, pins, options, _, manifest, observed, loaded, *_ = complete + changed = _copy_population(observed[graph.ATTACH_NODE]) + table = changed.frame.person + row = table.index[np.flatnonzero(np.isfinite(table[field].to_numpy()))[0]] + table.loc[row, field] += 1 + with pytest.raises((ValueError, AssertionError)): + graph.verify_materialized_property_income( + live.preparation, + live.allocated_population, + live.clone_population, + legacy_population=observed[financial.ATTACH_NODE], + population=changed, + host_pins=pins, + options=options, + artifacts=loaded, + legacy_matrix_producer_key=manifest.node(financial.PROJECTION_NODE).key, + ) diff --git a/packages/microcosm-build/tests/test_us_current_survey_puf_host.py b/packages/microcosm-build/tests/test_us_current_survey_puf_host.py new file mode 100644 index 000000000..50454a91d --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_puf_host.py @@ -0,0 +1,444 @@ +"""Invented host-only bridge; ordinary pytest, no donor/fit/engine/target access. + +The actual source test deliberately retains normal readiness and inventory gates. +Its proposal has not been executed. Fixture private file pins follow the existing +invented source helpers; no source/issuer function or money.ready is replaced. +""" + +import copy +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_survey_population import authenticated_arguments +from test_us_puf_detail_transfer import host as legacy_host +from test_us_survey_population_preparation import fixture as source_fixture + +from microcosm.build.us_runtime import asec_current_money as money +from microcosm.build.us_runtime import graph_combined_clone as clone +from microcosm.build.us_runtime import graph_puf_diagnostic_consumer as bridge +from microcosm.build.us_runtime import graph_survey_population as survey +from microcosm.build.us_runtime import puf_detail_transfer as detail +from microcosm.build.us_runtime import puf_diagnostic_consumer as consumer +from microcosm.build.us_runtime import support_provenance as provenance +from microcosm.build.us_runtime import survey_population_preparation as owner +from microcosm.build.us_runtime import survey_population_replay as replay +from microcosm.build.us_runtime.graph_sources import frame_column_declarations +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit.model_input import decode_recipient_matrix +from microcosm.frame import Frame, Weights +from microcosm.graph import NodeRejected, compile_graph, executor, run_graph +from microcosm.graph.keys import opaque_artifact_key + + +def _copy_frame(frame): + return Frame( + {e: frame.table(e).copy(deep=True) for e in frame.entities}, + frame.schema, + {e: frame.weights_for(e) for e in frame.weighted_entities}, + frame.strata.copy(deep=True), + metadata=frame.metadata, + mass_log=frame.mass_log, + ) + + +def _legacy_clone(): + return clone.clone_us_frame_for_puf_support(legacy_host()) + + +def test_existing_matrix_default_and_explicit_projection_have_identical_bytes(): + frame = _legacy_clone() + old, old_mask = detail.recipient_matrix(frame) + people = frame.person + values = [ + row["asec_reported_wage_income_2024_price"] + if row[provenance.support_channel_column("person")] == "asec" + else row["employment_income_before_lsr"] + for _, row in people.iterrows() + ] + wages = pd.Series( + values, index=pd.Index(people.person_id.to_numpy()), dtype="float64" + ) + new, new_mask = detail.recipient_matrix(frame, person_wages=wages) + assert new == old + np.testing.assert_array_equal(new_mask, old_mask) + # Independent hand expectations: actual JOINT spouse included, dependent + # wages 999/800 excluded, zero-weight single unit retained. + assert sorted( + decode_recipient_matrix(new).features.reported_wage_proxy.tolist() + ) == [10.0, 20.0, 30.0, 170.0] + with pytest.raises(ValueError, match="HOST_WAGE_ROW_AXIS"): + detail.recipient_matrix(frame, person_wages=wages.iloc[::-1]) + wrong = wages.copy() + role = people[provenance.support_clone_index_column("person")].to_numpy() + wrong.iloc[np.flatnonzero(role == 1)[0]] += 1 + with pytest.raises(ValueError, match="HOST_WAGE_PAIR_BYTES"): + detail.recipient_matrix(frame, person_wages=wrong) + missing = wages.copy() + first = people.iloc[0][provenance.support_source_id_column("person")] + pair = people[provenance.support_source_id_column("person")].eq(first).to_numpy() + missing.iloc[np.flatnonzero(pair)] = np.nan + with pytest.raises(ValueError, match="HOST_MISSING_FEATURE"): + detail.recipient_matrix(frame, person_wages=missing) + with pytest.raises(ValueError, match="HOST_WAGE_ROW_AXIS"): + detail.recipient_matrix(frame, person_wages=wages.astype("object")) + + +def test_current_route_is_explicit_and_legacy_declarations_are_separate(): + columns = frame_column_declarations(_legacy_clone()) + current = bridge.current_survey_host_nodes( + columns, + preparation_sha256="1" * 64, + allocation_sha256="2" * 64, + clone_sha256="3" * 64, + host_pins={ + e.name: { + "producer_key": "4" * 64, + "artifact_key": "5" * 64, + "payload_sha256": "6" * 64, + } + for e in bridge.current_survey_host_edges() + }, + ) + assert len(current) == 2 + assert all(not n.outputs and not n.sources for n in current) + assert [e.name for e in current[0].artifact_inputs] == [ + "preparation", + "allocation", + "frame_context", + ] + assert [e.name for e in current[1].artifact_inputs] == ["projection"] + assert current[0].params["host_route"] == consumer.CURRENT_SURVEY_ROUTE + assert len(bridge.host_edges(native_binding="native", clone_binding="clones")) == 11 + assert not any( + word in n.kernel + for n in current + for word in ("donor", "fit", "attach", "apply") + ) + + +def test_unissued_byte_equal_preparation_refuses_before_projection(): + fake = object.__new__(owner.AuthenticatedSurveyPopulationPreparation) + object.__setattr__(fake, "payload", b"{}") + with pytest.raises( + owner.SurveyPopulationPreparationError, match="UNISSUED_OR_CHANGED" + ): + bridge.qualify_current_survey_host(fake, None, None) + + +@pytest.mark.parametrize("year,field", [(2024, "ANN_VAL"), (2022, "WSAL_VAL")]) +def test_current_host_preserves_full_parent_readiness( + tmp_path, monkeypatch, year, field +): + arguments = source_fixture(tmp_path, monkeypatch, missing_asec_money=(year, field)) + live = survey.run_authenticated_survey_population( + **arguments, + store_root=tmp_path / "graph-store", + clones=True, + return_values=True, + ) + # Reaching this point establishes actual preparation and native issuance; + # an earlier SOURCE_CHANGED error is not the readiness result under test. + preparation_entry = owner._ISSUED[id(live.preparation)] + native = preparation_entry[2].native[1] + parent = owner.asec_native._ISSUED[id(native)][2].parent + decoded = money.classify_asec_money(parent.views(), parent.spec, parent.scope) + years = np.asarray(parent.scope.person_years) + absent = ~decoded.field(field).validity.astype(bool) + assert int(absent.sum()) == 1 + assert years[absent].tolist() == [year] + assert decoded.field("WSAL_VAL").validity[years == 2024].all() + with pytest.raises(money.MoneyRefusalError) as caught: + bridge.qualify_current_survey_host( + live.preparation, live.allocated_population, live.clone_population + ) + assert caught.value.reason == "MISSING_REQUIRED_AMOUNT" + assert caught.value.field == field + + +def test_actual_current_sources_cold_and_replayed_host(tmp_path, monkeypatch): + arguments = authenticated_arguments(tmp_path, monkeypatch) + live = survey.run_authenticated_survey_population( + **arguments, clones=True, return_values=True + ) + preparation, allocated, expanded = ( + live.preparation, + live.allocated_population, + live.clone_population, + ) + before = owner._frame_identity(expanded.frame) + projection, matrix, mask, evidence = bridge.qualify_current_survey_host( + preparation, allocated, expanded + ) + document = codec.decode_json(projection) + assert evidence["release_eligible"] is False + assert evidence["source_period_equivalence_claim"] is False + assert len(document["asec"]["rows"]) == 4 + assert [r[2] for r in document["asec"]["rows"]] == [2024] * 4 + assert [ + np.frombuffer(bytes.fromhex(r[3]), dtype=" 1 + for changed_slice in ( + replace(first, columns=first.columns[:-1]), + replace(first, columns=first.columns[::-1]), + replace(first, columns=(*first.columns, "__extra_input__")), + # A declared column lets Node construction succeed; this direct + # contract check must reject non-ALL rows before any mask evaluation. + replace(first, rows=first.columns[0]), + ): + bad_node = replace(nodes[0], inputs=(changed_slice, *nodes[0].inputs[1:])) + with pytest.raises(ValueError, match="^SURVEY_HOST_INPUTS$"): + bridge._current_context_frame( + replace(context, node=bad_node), expanded.frame + ) + for field, values, reason in ( + ("tables", context.tables, "SURVEY_HOST_TABLES"), + ("weights", context.weights, "SURVEY_HOST_WEIGHTS"), + ): + missing = dict(values) + del missing[next(iter(missing))] + extra = {**values, "__extra_entity__": next(iter(values.values()))} + for changed_values in (missing, extra): + with pytest.raises(ValueError, match="^" + reason + "$"): + bridge._current_context_frame( + replace(context, **{field: changed_values}), expanded.frame + ) + + person = context.tables[expanded.frame.schema.person_entity] + changed_tables = dict(context.tables) + changed_tables[expanded.frame.schema.person_entity] = person.iloc[:, ::-1] + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + changed_tables[expanded.frame.schema.person_entity] = person.rename_axis( + "__changed_columns__", axis="columns" + ) + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + changed_person = person.copy(deep=True) + changed_person.iloc[0, changed_person.columns.get_loc("age")] += 1 + changed_tables[expanded.frame.schema.person_entity] = changed_person + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_NATIVE_BITS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + + # Positive and negative zero are both valid weights; only their exact bits + # differ. This must reach the replay comparison rather than Frame refusal. + entity = next(e for e, w in context.weights.items() if np.any(w.values == 0)) + value = context.weights[entity] + changed_values = value.values.copy() + zero = int(np.flatnonzero(changed_values == 0)[0]) + changed_values[zero] = 0.0 if np.signbit(changed_values[zero]) else -0.0 + changed_weights = {**context.weights, entity: Weights(changed_values, value.kind)} + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_WEIGHT_BYTES$"): + bridge._current_context_frame( + replace(context, weights=changed_weights), expanded.frame + ) + with pytest.raises(ValueError, match="^SURVEY_HOST_STRATA_NAME$"): + bridge._current_context_frame( + replace(context, strata=context.strata.rename("__changed_name__")), + expanded.frame, + ) + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, strata=context.strata.rename_axis("__changed_index__")), + expanded.frame, + ) + assert owner._frame_identity(expanded.frame) == before + + graph = replace(live.compiled.graph, nodes=(*live.compiled.graph.nodes, *nodes)) + compiled = compile_graph(graph) + assert len(compiled.graph.nodes) == 6 + live.kernels.register( + bridge.CurrentSurveyHostProjectionKernel(preparation, allocated, expanded) + ) + live.kernels.register(bridge.CurrentSurveyMatrixKernel()) + outputs = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume=resume, + _population_observer=lambda name, population, observed=observed: ( + observed.__setitem__(name, population) + ), + ) + actual = [] + for node_id, name, type_, expected, kernel in ( + ( + bridge.CURRENT_PROJECTION_NODE, + "projection", + bridge.CURRENT_PROJECTION_TYPE, + projection, + bridge.CurrentSurveyHostProjectionKernel, + ), + ( + bridge.CURRENT_MATRIX_NODE, + "matrix", + bridge.model_input.RECIPIENT_MATRIX_TYPE, + matrix, + bridge.CurrentSurveyMatrixKernel, + ), + ): + survey._final_artifact( + manifest, + live.store, + node_id=node_id, + name=name, + type_=type_, + payload=expected, + capabilities=kernel.capabilities, + ) + receipt = manifest.node(node_id) + assert receipt.opaque_artifacts[name] == opaque_artifact_key( + receipt.key, name + ) + actual.append(live.store.load_bytes(receipt.opaque_artifacts[name])) + replay.same_replayed_population(expanded, observed[node_id]) + checked = bridge.verify_materialized_current_survey_host( + preparation, + allocated, + expanded, + population=observed[bridge.CURRENT_MATRIX_NODE], + projection=actual[0], + matrix=actual[1], + ) + assert checked == evidence + outputs.append(tuple(actual)) + assert outputs[0] == outputs[1] == (projection, matrix) + assert owner._frame_identity(expanded.frame) == before + wrong_pins = copy.deepcopy(pins) + wrong_pins["preparation"]["producer_key"] = "f" * 64 + bad_projection_node = replace( + nodes[0], + params={ + **dict(nodes[0].params), + "host_edges": codec.encode_json(wrong_pins).decode(), + }, + ) + bad_graph = replace( + graph, nodes=(*live.compiled.graph.nodes, bad_projection_node, nodes[1]) + ) + with pytest.raises(NodeRejected, match="SURVEY_HOST_EDGE_PIN"): + run_graph( + compile_graph(bad_graph), + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume="auto", + ) + + # A well-formed amount mutation can pass the pure value parser; the live + # materialized verifier must reject it against the retained current owner. + changed = copy.deepcopy(document) + changed["asec"]["rows"][0][3] = np.asarray([60001.0], dtype=" None: +@pytest.mark.parametrize("context", ["empty", "metadata", "mass_log", "both"]) +def test_aca_source_runtime_refreshes_degenerate_release_inputs( + monkeypatch, context +) -> None: builder = _load_builder_module() + has_metadata = context in ("metadata", "both") + has_mass_log = context in ("mass_log", "both") person = pd.DataFrame( { "person_id": np.asarray([1, 2, 3], dtype="int64"), @@ -7272,11 +7277,20 @@ def test_aca_source_runtime_refreshes_degenerate_release_inputs(monkeypatch) -> builder.US_SCHEMA, { "household": builder.Weights( - values=np.asarray([1.0, 1.0]), + values=np.asarray([0.5, 0.5] if has_mass_log else [1.0, 1.0]), kind=WeightKind.DESIGN, ) }, + metadata={"invented_native_context": {"period": 2024, "flags": [False, True]}} + if has_metadata + else None, ) + if has_mass_log: + frame = frame.with_weights( + "household", + Weights(np.asarray([1.0, 1.0]), WeightKind.DESIGN), + mass=MassChange(factor=2.0, reason="Invented pre-ACA support expansion"), + ) specs = ( TargetSpec( name="cms_aca.oep2024.state_marketplace.al.aptc_recipients", @@ -7336,9 +7350,37 @@ def fake_calculate_array(simulation, variable, *, map_to=None): frame.table("tax_unit")["selected_marketplace_plan_benchmark_ratio"].nunique() == 1 ) + # ACA owns only its two tax-unit outputs. Context and all other data must + # survive the real writeback, including a retained native handoff's history. + assert refreshed.metadata == frame.metadata + assert refreshed.mass_log == frame.mass_log + assert bool(refreshed.metadata) is has_metadata + assert bool(refreshed.mass_log) is has_mass_log + assert refreshed.schema == frame.schema + pd.testing.assert_series_equal(refreshed.strata, frame.strata) + assert refreshed.weighted_entities == frame.weighted_entities + for entity in frame.weighted_entities: + before, after = frame.weights_for(entity), refreshed.weights_for(entity) + assert after.kind is before.kind + assert after.values.dtype == before.values.dtype + assert after.values.tobytes() == before.values.tobytes() + for entity in frame.entities: + columns = [ + name + for name in frame.table(entity) + if name not in builder.US_ACA_SOURCE_OUTPUT_COLUMNS + ] + pd.testing.assert_frame_equal( + refreshed.table(entity)[columns], + frame.table(entity)[columns], + check_exact=True, + ) -def test_aca_source_tax_unit_table_batches_policyengine_inputs(monkeypatch) -> None: +@pytest.mark.parametrize("explicit_metadata", [False, True]) +def test_aca_source_tax_unit_table_batches_policyengine_inputs( + monkeypatch, explicit_metadata +) -> None: builder = _load_builder_module() person = pd.DataFrame( { @@ -7411,6 +7453,10 @@ def test_aca_source_tax_unit_table_batches_policyengine_inputs(monkeypatch) -> N seen_tax_unit_batches: list[tuple[int, ...]] = [] formula_owned_assertions: list[int] = [] dataset_assert_flags: list[bool | None] = [] + formula_metadata = object() if explicit_metadata else None + seen_metadata = [] + before = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + before_weights = frame.weights_for("household").values.copy() class FakeMicrosimulation: def __init__(self, *, dataset): @@ -7440,11 +7486,13 @@ def fake_calculate_array(simulation, variable, *, map_to=None): dtype=np.float64, ) - def fake_assert_no_formula_owned_columns(frame_arg): + def fake_assert_no_formula_owned_columns(frame_arg, *, formula_metadata=None): formula_owned_assertions.append(frame_arg.n("household")) + seen_metadata.append(formula_metadata) def fake_dataset_from_frame(frame_arg, **kwargs): dataset_assert_flags.append(kwargs.get("assert_no_formula_owned_columns")) + seen_metadata.append(kwargs.get("formula_metadata")) return frame_arg monkeypatch.setattr( @@ -7460,11 +7508,17 @@ def fake_dataset_from_frame(frame_arg, **kwargs): target_tables, microsimulation_cls=FakeMicrosimulation, maximum_microsim_batch_size=1, + formula_metadata=formula_metadata, ).set_index("tax_unit_id") assert seen_tax_unit_batches == [(10,), (20,), (30,)] assert formula_owned_assertions == [3] assert dataset_assert_flags == [False, False, False] + assert len(seen_metadata) == 4 + assert all(metadata is formula_metadata for metadata in seen_metadata) + for entity, original in before.items(): + pd.testing.assert_frame_equal(frame.table(entity), original, check_exact=True) + np.testing.assert_array_equal(frame.weights_for("household").values, before_weights) assert tax_unit.loc[10, "tax_unit_weight"] == 20.0 assert tax_unit.loc[20, "tax_unit_weight"] == 20.0 assert tax_unit.loc[30, "tax_unit_weight"] == 0.0 @@ -7525,6 +7579,8 @@ class FakeFrame: schema = object() weighted_entities = () strata = None + mass_log = () + metadata = {} def table(self, entity): assert entity == "tax_unit" @@ -7561,7 +7617,9 @@ def fake_run_source_stage( monkeypatch.setattr( builder, "Frame", - lambda tables, schema, weights, strata: SimpleNamespace(tables=tables), + lambda tables, schema, weights, strata, *, mass_log, metadata: SimpleNamespace( + tables=tables + ), ) specs = ( @@ -7801,6 +7859,8 @@ def fake_dataset_from_frame( zero_variables=(), system=None, assert_no_formula_owned_columns=True, + formula_metadata=None, + dataset_cls=None, ): datasets.append( ( @@ -7817,7 +7877,7 @@ def fake_make_zero_variable_reform(system, variable_name): assert variable_name == "mock_credit" return object() - def fake_assert_no_formula_owned_columns(frame_arg): + def fake_assert_no_formula_owned_columns(frame_arg, *, formula_metadata=None): formula_owned_assertions.append(frame_arg.n("household")) monkeypatch.setitem( @@ -9836,7 +9896,7 @@ def test_dataset_from_frame_rejects_formula_owned_columns_by_default( ) -> None: builder = _load_builder_module() - def fake_assert_no_formula_owned_columns(frame): + def fake_assert_no_formula_owned_columns(frame, *, formula_metadata=None): assert frame is small_frame raise ValueError("formula-owned guard fired") @@ -9876,7 +9936,9 @@ def _engine_computed_columns(self, tables, *, period): def test_l0_refit_export_subsets_clean_base_frame(monkeypatch, small_frame) -> None: builder = _load_builder_module() - monkeypatch.setattr(builder, "_assert_no_formula_owned_columns", lambda frame: None) + monkeypatch.setattr( + builder, "_assert_no_formula_owned_columns", lambda frame, **kwargs: None + ) result = SimpleNamespace( selected_entity_ids=np.asarray([2], dtype="int64"), weight_entity="household", @@ -10434,6 +10496,8 @@ def fake_dataset_from_frame( zero_variables=(), system=None, assert_no_formula_owned_columns=True, + formula_metadata=None, + dataset_cls=None, ): return {"frame": frame_arg, "zero_variables": tuple(zero_variables)} @@ -10453,7 +10517,7 @@ def fake_make_zero_variable_reform(system, variable_name): monkeypatch.setattr( builder, "_assert_no_formula_owned_columns", - lambda frame_arg: None, + lambda frame_arg, **kwargs: None, ) monkeypatch.setattr(builder, "_dataset_from_frame", fake_dataset_from_frame) monkeypatch.setattr( diff --git a/packages/microcosm-build/tests/test_us_fiscal_shared_solve.py b/packages/microcosm-build/tests/test_us_fiscal_shared_solve.py new file mode 100644 index 000000000..3b10f6e13 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_fiscal_shared_solve.py @@ -0,0 +1,173 @@ +"""The shared release solve preserves the maintained calibrators and support.""" + +import argparse +import json +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_fiscal_refresh_builder import _load_builder_module + +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.frame import WeightKind + + +def _args(dense): + return argparse.Namespace( + exact_k=None, + dense_default_dataset=dense, + epochs=12, + learning_rate=0.03, + max_weight_ratio=4.0, + seed=17, + l2_lambda=0.01, + refit_l2_lambda=0.02, + l0_refit_lambda_share=0.01, + ) + + +def _registry(): + return TargetRegistry( + ( + TargetSpec( + name="invented.income", + entity="person", + measure="income", + value=750_000.0, + source="Invented fixture", + ), + ), + country="us", + ) + + +@pytest.mark.parametrize("dense", [True, False]) +def test_shared_solve_matches_maintained_solver_and_preserves_parent( + small_frame, dense +): + builder = _load_builder_module() + args = _args(dense) + registry = _registry() + before = {e: small_frame.table(e).copy(deep=True) for e in small_frame.entities} + initial = small_frame.weights_for("household").values.copy() + loss_weights = np.asarray([1.7]) + warm = np.asarray([1100.0, 1900.0]) if dense else None + events = [] + result, metadata = builder._calibrate_fiscal_support( + small_frame, + registry, + args=args, + target_loss_weights=loss_weights, + warm_start_weights=warm, + progress_callback=events.append, + ) + common = dict( + epochs=12, + learning_rate=0.03, + max_weight_ratio=4.0, + seed=17, + mass="conserve", + l2_lambda=0.01, + target_loss_weights=loss_weights, + target_loss_cap=builder.US_FISCAL_TARGET_LOSS_CAP, + warm_start_weights=warm, + ) + if dense: + expected = builder.calibrate(small_frame, registry.to_target_set(), **common) + else: + expected = builder.calibrate_l0_refit( + small_frame, + registry.to_target_set(), + refit_epochs=12, + l0_lambda=0.005, + refit_l2_lambda=0.02, + **common, + ) + np.testing.assert_array_equal(result.weights, expected.weights) + assert result.final_loss == expected.final_loss + assert events + assert metadata["method"] == ("dense_no_l0" if dense else "l0_refit") + assert metadata["n_candidate_households"] == 2 + json.dumps(metadata, allow_nan=False) + for entity, table in before.items(): + pd.testing.assert_frame_equal(small_frame.table(entity), table) + np.testing.assert_array_equal(small_frame.weights_for("household").values, initial) + assert small_frame.weights_for("household").kind == WeightKind.DESIGN + + class LeafMetadata: + def _engine_computed_columns(self, tables, *, period): + return set() + + if dense: + exported = builder._with_calibrated_weights( + small_frame, result.weights, formula_metadata=LeafMetadata() + ) + ids = small_frame.table("household")["household_id"].to_numpy() + else: + exported = builder._with_l0_refit_weights( + small_frame, result, formula_metadata=LeafMetadata() + ) + ids = result.selected_entity_ids + np.testing.assert_array_equal(exported.table("household")["household_id"], ids) + np.testing.assert_array_equal( + exported.weights_for("household").values, result.weights + ) + assert exported.weights_for("household").kind == WeightKind.CALIBRATED + assert set(exported.table("person").columns) == set(before["person"].columns) + assert set(exported.table("household").columns) == set(before["household"].columns) + + +def test_sparse_failure_losses_still_write_strict_diagnostics(monkeypatch, small_frame): + builder = _load_builder_module() + result = SimpleNamespace( + selection=SimpleNamespace(n_nonzero=1, final_loss=float("inf")), + frame=small_frame, + l0_lambda=0.005, + initial_loss=float("nan"), + final_loss=float("inf"), + ) + monkeypatch.setattr(builder, "calibrate_l0_refit", lambda *a, **kw: result) + _, metadata = builder._calibrate_fiscal_support( + small_frame, + _registry(), + args=_args(False), + target_loss_weights=np.asarray([1.0]), + warm_start_weights=None, + ) + assert metadata["selection_final_loss"] is None + assert metadata["refit_initial_loss"] is None + assert metadata["refit_final_loss"] is None + json.dumps(metadata, allow_nan=False) + + +def test_exact_k_cannot_silently_enter_ordinary_solve(monkeypatch, small_frame): + builder = _load_builder_module() + args = _args(False) + args.exact_k = 1 + + def forbidden(*a, **kw): + raise AssertionError("ordinary solve must not run") + + monkeypatch.setattr(builder, "calibrate", forbidden) + monkeypatch.setattr(builder, "calibrate_l0_refit", forbidden) + with pytest.raises(ValueError, match="Exact-k"): + builder._calibrate_fiscal_support( + small_frame, + _registry(), + args=args, + target_loss_weights=np.asarray([1.0]), + warm_start_weights=None, + ) + + +def test_sparse_warm_start_keeps_existing_refusal(small_frame): + builder = _load_builder_module() + with pytest.raises(ValueError, match="warm_start_weights is not supported"): + builder._calibrate_fiscal_support( + small_frame, + _registry(), + args=_args(False), + target_loss_weights=np.asarray([1.0]), + warm_start_weights=np.asarray([1100.0, 1900.0]), + ) diff --git a/packages/microcosm-build/tests/test_us_fiscal_shared_targets.py b/packages/microcosm-build/tests/test_us_fiscal_shared_targets.py new file mode 100644 index 000000000..51569b6b5 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_fiscal_shared_targets.py @@ -0,0 +1,110 @@ +"""The shared target compiler preserves pins, substitutions, and parity refusal.""" + +import argparse +from types import SimpleNamespace + +import pytest +from test_us_fiscal_refresh_builder import _load_builder_module + + +def _args(tmp_path): + return argparse.Namespace( + ledger_facts=tmp_path / "facts.jsonl", + ledger_facts_sha256="a" * 64, + ledger_manifest_sha256="b" * 64, + age_targets=True, + allow_unaged_dollar_targets=False, + ) + + +@pytest.mark.parametrize("parity_passes", [True, False]) +def test_shared_target_compilation_preserves_pins_configuration_and_gate_order( + tmp_path, monkeypatch, parity_passes +): + builder = _load_builder_module() + args = _args(tmp_path) + facts = object() + artifact = SimpleNamespace(facts=facts) + initial_registry, substituted_registry, crosswalk = object(), object(), object() + records = ("invented-substitution",) + gate = SimpleNamespace(passed=parity_passes, failures=("invented missing family",)) + events = [] + + def load(path, **kwargs): + assert path == args.ledger_facts + assert kwargs == { + "expected_facts_sha256": "a" * 64, + "expected_manifest_sha256": "b" * 64, + } + events.append("load") + return artifact + + def compile_targets(actual, **kwargs): + assert actual is facts + assert kwargs == { + "target_period": builder.PERIOD, + "congressional_district_vintage_crosswalk": crosswalk, + "age_targets": True, + "allow_unaged_dollar_targets": False, + } + events.append("compile") + return initial_registry + + def substitute(actual): + assert actual is initial_registry + events.append("substitute") + return substituted_registry, records + + def check_manifest(*, registry): + assert registry is substituted_registry + events.append("manifest") + + def check_parity(actual): + assert actual is substituted_registry + events.append("parity") + return gate + + monkeypatch.setattr(builder, "load_ledger_consumer_artifact", load) + monkeypatch.setattr(builder, "compile_us_fiscal_target_registry", compile_targets) + monkeypatch.setattr( + builder, "apply_us_medicaid_enrollment_substitutions", substitute + ) + monkeypatch.setattr( + builder, "assert_target_parity_manifest_current", check_manifest + ) + monkeypatch.setattr(builder, "us_release_target_parity_gate", check_parity) + if parity_passes: + result = builder._compile_fiscal_release_target_registry( + args, congressional_district_vintage_crosswalk=crosswalk + ) + assert all( + a is b + for a, b in zip( + result, (artifact, substituted_registry, records, gate), strict=True + ) + ) + else: + with pytest.raises( + RuntimeError, match="Target parity coverage failed: invented missing family" + ): + builder._compile_fiscal_release_target_registry( + args, congressional_district_vintage_crosswalk=crosswalk + ) + assert events == ["load", "compile", "substitute", "manifest", "parity"] + + +def test_failed_ledger_identity_stops_before_compilation(tmp_path, monkeypatch): + builder = _load_builder_module() + + def refuse(*a, **kw): + raise ValueError("invented source identity mismatch") + + def forbidden(*a, **kw): + raise AssertionError("must not compile refused source") + + monkeypatch.setattr(builder, "load_ledger_consumer_artifact", refuse) + monkeypatch.setattr(builder, "compile_us_fiscal_target_registry", forbidden) + with pytest.raises(ValueError, match="source identity mismatch"): + builder._compile_fiscal_release_target_registry( + _args(tmp_path), congressional_district_vintage_crosswalk=None + ) diff --git a/packages/microcosm-build/tests/test_us_fiscal_targets.py b/packages/microcosm-build/tests/test_us_fiscal_targets.py index 0252714f2..01c6fe2d2 100644 --- a/packages/microcosm-build/tests/test_us_fiscal_targets.py +++ b/packages/microcosm-build/tests/test_us_fiscal_targets.py @@ -3774,6 +3774,379 @@ def test_stale_soi_capital_gains_without_source_total_is_dropped() -> None: assert state_record_id not in source_record_ids +def _soi_table_1_1_fact( + source_period: int, + *, + income_range: str = "all", + lower: float | None = None, + upper: float | None = None, + measure_id: str = "adjusted_gross_income", + value: float, + table: str = "table_1_1", + geography_level: str = "country", + geography_id: str = "0100000US", + filing_status: str = "all", +) -> dict[str, object]: + """A Publication 1304 size-of-AGI fact shaped like the Chronicle export.""" + constraints: list[dict[str, object]] = [] + if lower is not None: + constraints.append( + { + "variable": "us:statutes/26/62#adjusted_gross_income", + "operator": ">=", + "value": lower, + } + ) + if upper is not None: + constraints.append( + { + "variable": "us:statutes/26/62#adjusted_gross_income", + "operator": "<", + "value": upper, + } + ) + return _dynamic_ledger_fact( + source_record_id=( + f"irs_soi.ty{source_period}.{table}.{income_range}.{measure_id}" + ), + source_name="irs_soi", + measure_id=measure_id, + value=value, + period_value=source_period, + geography_level=geography_level, + geography_id=geography_id, + dimensions={"income_range": income_range, "filing_status": filing_status}, + universe_constraints=constraints, + layout_record_set_id=f"irs_soi.ty{source_period}.{table}", + groupby_dimension="us:statutes/26/62#adjusted_gross_income", + groupby_value_id=income_range, + ) + + +def _soi_table_1_1_totals(source_period: int, *, returns: float, agi: float): + return ( + _soi_table_1_1_fact(source_period, measure_id="return_count", value=returns), + _soi_table_1_1_fact(source_period, value=agi), + ) + + +def test_cross_period_soi_table_1_1_agi_size_classes_bind_as_shares() -> None: + # microcosm#958: with only the all-returns totals bound, certified + # populace-us-2024-spm-20260915 met national AGI at +0.3% while holding + # 3.4x SOI's returns from $1M to $2M and none above $10M. The size + # classes publish one tax year behind the build, so without the rescue + # every one of them is refused as a stale nominal bin. + facts = [ + *packaged_reference_facts(), + *_soi_table_1_1_totals(2023, returns=160_000_000, agi=16_000_000_000_000), + _soi_table_1_1_fact( + 2023, + income_range="10m_plus", + lower=10_000_000, + value=900_000_000_000, + ), + _soi_table_1_1_fact( + 2023, + income_range="10m_plus", + lower=10_000_000, + measure_id="return_count", + value=30_000, + ), + _soi_table_1_1_fact( + 2023, + income_range="5m_to_10m", + lower=5_000_000, + upper=10_000_000, + value=330_000_000_000, + ), + ] + registry = compile_us_fiscal_target_registry( + facts, + target_period=2024, + allow_unaged_dollar_targets=True, + ) + by_name = {spec.name: spec for spec in registry.specs} + + amount = by_name["irs_soi.ty2023.table_1_1.10m_plus.adjusted_gross_income"] + assert amount.value == 900_000_000_000 + assert amount.metadata["variable"] == "adjusted_gross_income" + assert amount.metadata["measure_mode"] == "sum" + assert amount.metadata["agi_lower_bound"] == "10000000.0" + assert amount.metadata["agi_upper_bound"] == "inf" + assert amount.metadata["requires_agi_size_distribution_rebase"] == "true" + assert amount.metadata["stale_distribution_rebased_to_active_total"] == "true" + assert amount.metadata["uprating_index"] == "total_adjusted_gross_income" + assert amount.metadata["uprating_from_period"] == "2023" + assert amount.metadata["uprating_to_period"] == "2023" + assert amount.metadata["uprating_index_source_record_id"] == ( + "irs_soi.ty2023.table_1_1.all.adjusted_gross_income" + ) + assert float(amount.metadata["uprating_factor"]) == 1.0 + + count = by_name["irs_soi.ty2023.table_1_1.10m_plus.return_count"] + assert count.value == 30_000 + assert count.metadata["measure_mode"] == "indicator_sum" + assert count.metadata["uprating_index"] == "total_return_count" + + closed = by_name["irs_soi.ty2023.table_1_1.5m_to_10m.adjusted_gross_income"] + assert closed.metadata["agi_lower_bound"] == "5000000.0" + assert closed.metadata["agi_upper_bound"] == "10000000.0" + # The national all-returns rows still own the national concept. + assert by_name["irs_soi.ty2023.table_1_1.all.adjusted_gross_income"].value == ( + 16_000_000_000_000 + ) + assert "uprating_factor" not in ( + by_name["irs_soi.ty2023.table_1_1.all.return_count"].metadata + ) + + +def test_soi_table_1_1_agi_size_classes_bind_only_the_latest_vintage() -> None: + # Chronicle keeps old tax years. One class shape activates one fact, the + # latest not after the build period, and it rebases on its OWN vintage. + facts = [ + *packaged_reference_facts(), + *_soi_table_1_1_totals(2022, returns=150_000_000, agi=15_000_000_000_000), + *_soi_table_1_1_totals(2023, returns=160_000_000, agi=16_000_000_000_000), + _soi_table_1_1_fact( + 2022, income_range="10m_plus", lower=10_000_000, value=1_050_000_000_000 + ), + _soi_table_1_1_fact( + 2023, income_range="10m_plus", lower=10_000_000, value=900_000_000_000 + ), + ] + registry = compile_us_fiscal_target_registry( + facts, + target_period=2024, + allow_unaged_dollar_targets=True, + ) + names = {spec.name for spec in registry.specs} + + assert "irs_soi.ty2023.table_1_1.10m_plus.adjusted_gross_income" in names + assert "irs_soi.ty2022.table_1_1.10m_plus.adjusted_gross_income" not in names + + +def test_soi_table_1_1_agi_size_class_rebases_onto_a_newer_national_control() -> None: + # When the classes lag the national total by a vintage, the class is a + # SHARE of its own year scaled to the newer control, landed at the + # control's period so target aging completes only the remaining links. + facts = [ + *packaged_reference_facts(), + *_soi_table_1_1_totals(2022, returns=150_000_000, agi=15_000_000_000_000), + *_soi_table_1_1_totals(2023, returns=160_000_000, agi=16_500_000_000_000), + _soi_table_1_1_fact( + 2022, income_range="10m_plus", lower=10_000_000, value=1_000_000_000_000 + ), + _soi_table_1_1_fact( + 2022, + income_range="10m_plus", + lower=10_000_000, + measure_id="return_count", + value=30_000, + ), + ] + registry = compile_us_fiscal_target_registry( + facts, + target_period=2024, + allow_unaged_dollar_targets=True, + ) + by_name = {spec.name: spec for spec in registry.specs} + + amount = by_name["irs_soi.ty2022.table_1_1.10m_plus.adjusted_gross_income"] + assert amount.value == pytest.approx(1_000_000_000_000 * 16.5 / 15.0) + assert amount.metadata["uprating_from_period"] == "2022" + assert amount.metadata["uprating_to_period"] == "2023" + count = by_name["irs_soi.ty2022.table_1_1.10m_plus.return_count"] + assert count.value == pytest.approx(30_000 * 160 / 150) + + +@pytest.mark.parametrize( + ("fact_kwargs", "reason"), + [ + ( + {"income_range": "50k_to_75k", "lower": 50_000, "upper": 75_000}, + "below the declared lower edge, where a tax-unit count is not a " + "return count", + ), + ( + {"income_range": "10m_plus", "lower": 10_000_000, "table": "table_1_2"}, + "another Publication 1304 table carrying the same measure id", + ), + ( + { + "income_range": "10m_plus", + "lower": 10_000_000, + "filing_status": "single", + }, + "a filing-status slice, which has no same-status national control", + ), + ( + { + "income_range": "10m_plus", + "lower": 10_000_000, + "geography_level": "state", + "geography_id": "0400000US06", + }, + "a state slice, which the national control cannot anchor", + ), + ], +) +def test_soi_agi_size_rescue_stays_narrow( + fact_kwargs: dict[str, object], + reason: str, +) -> None: + fact = _soi_table_1_1_fact(2023, value=900_000_000_000, **fact_kwargs) + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + *_soi_table_1_1_totals(2023, returns=160_000_000, agi=16_000_000_000_000), + fact, + ], + target_period=2024, + allow_unaged_dollar_targets=True, + ) + + name = fact["lineage"]["source_record_id"] + assert name not in {spec.name for spec in registry.specs}, reason + + +def test_soi_agi_size_rescue_starts_exactly_at_the_declared_lower_edge() -> None: + from microcosm.build.us_runtime.fiscal_targets import ( + US_SOI_AGI_SIZE_DISTRIBUTION_MINIMUM_LOWER_BOUND, + ) + + floor = US_SOI_AGI_SIZE_DISTRIBUTION_MINIMUM_LOWER_BOUND + at_edge = _soi_table_1_1_fact( + 2023, + income_range="100k_to_200k", + lower=floor, + upper=2 * floor, + measure_id="return_count", + value=27_000_000, + ) + below_edge = _soi_table_1_1_fact( + 2023, + income_range="75k_to_100k", + lower=0.75 * floor, + upper=floor, + measure_id="return_count", + value=15_000_000, + ) + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + *_soi_table_1_1_totals(2023, returns=160_000_000, agi=16_000_000_000_000), + at_edge, + below_edge, + ], + target_period=2024, + allow_unaged_dollar_targets=True, + ) + names = {spec.name for spec in registry.specs} + + assert floor == 100_000.0 + assert at_edge["lineage"]["source_record_id"] in names + assert below_edge["lineage"]["source_record_id"] not in names + + +def test_soi_table_1_1_agi_size_class_without_a_national_total_is_dropped() -> None: + # An unanchored class is unverifiable: dropped, never shipped stale. + fact = _soi_table_1_1_fact( + 2023, income_range="10m_plus", lower=10_000_000, value=900_000_000_000 + ) + registry = compile_us_fiscal_target_registry( + [*packaged_reference_facts(), fact], + target_period=2024, + allow_unaged_dollar_targets=True, + ) + + assert fact["lineage"]["source_record_id"] not in { + spec.name for spec in registry.specs + } + + +def test_same_period_soi_table_1_1_agi_size_class_needs_no_rescue() -> None: + # A class published AT the build period is an ordinary level, not a share. + fact = _soi_table_1_1_fact( + 2024, income_range="10m_plus", lower=10_000_000, value=950_000_000_000 + ) + registry = compile_us_fiscal_target_registry( + [*packaged_reference_facts(), fact], + target_period=2024, + allow_unaged_dollar_targets=True, + ) + spec = next( + spec + for spec in registry.specs + if spec.name == fact["lineage"]["source_record_id"] + ) + + assert spec.value == 950_000_000_000 + assert "requires_agi_size_distribution_rebase" not in spec.metadata + assert "uprating_factor" not in spec.metadata + + +def test_soi_table_1_1_agi_size_classes_age_with_the_national_rows() -> None: + # The classes must keep summing to the national rows they decompose: + # class AGI ages on the CBO AGI series from the control's period, exactly + # like the all-returns AGI row, and class counts stay raw like the + # all-returns count. + facts = [ + *packaged_reference_facts(), + *_soi_table_1_1_totals(2023, returns=160_000_000, agi=16_000_000_000_000), + _soi_table_1_1_fact( + 2023, income_range="10m_plus", lower=10_000_000, value=900_000_000_000 + ), + _soi_table_1_1_fact( + 2023, + income_range="10m_plus", + lower=10_000_000, + measure_id="return_count", + value=30_000, + ), + _cbo_income_source_projection_fact( + 2023, "adjusted_gross_income", value=15_000_000_000_000 + ), + _cbo_income_source_projection_fact( + 2024, "adjusted_gross_income", value=16_500_000_000_000 + ), + ] + registry = compile_us_fiscal_target_registry( + facts, + target_period=2024, + age_targets=True, + ) + by_name = {spec.name: spec for spec in registry.specs} + + national = by_name["irs_soi.ty2023.table_1_1.all.adjusted_gross_income"] + amount = by_name["irs_soi.ty2023.table_1_1.10m_plus.adjusted_gross_income"] + assert national.value == pytest.approx(16_000_000_000_000 * 1.1) + assert amount.value == pytest.approx(900_000_000_000 * 1.1) + assert amount.value / national.value == pytest.approx(900 / 16_000) + count = by_name["irs_soi.ty2023.table_1_1.10m_plus.return_count"] + assert count.value == 30_000 + + +def test_published_soi_table_1_1_size_classes_stay_feasible_when_aged() -> None: + # Counts stay raw while class AGI ages, so the implied mean AGI of a + # closed class rises by the aging factor. If it left the class's own + # edges the count and amount rows could not both bind. Values are IRS + # SOI Publication 1304 Table 1.1, tax year 2023 (23in11si.xls); the + # factor is the CBO AGI TY2023->TY2024 ratio the 2026-09-18 feed yields. + cbo_agi_growth_2023_to_2024 = 1.08721346938244 + published = { + (100_000, 200_000): (27_602_755, 3_818_295_141_000), + (200_000, 500_000): (10_955_818, 3_153_877_437_000), + (500_000, 1_000_000): (1_779_720, 1_194_934_962_000), + (1_000_000, 1_500_000): (368_931, 443_658_405_000), + (1_500_000, 2_000_000): (147_290, 252_903_399_000), + (2_000_000, 5_000_000): (203_229, 602_653_109_000), + (5_000_000, 10_000_000): (49_262, 336_334_760_000), + } + for (lower, upper), (returns, agi) in published.items(): + aged_mean = agi * cbo_agi_growth_2023_to_2024 / returns + assert lower <= aged_mean < upper, (lower, upper, aged_mean) + + def test_cross_period_soi_taxable_interest_agi_slice_without_total_is_dropped() -> None: source_record_id = ( "irs_soi.ty2022.historic_table_2.us.200k_to_500k.taxable_interest_amount" diff --git a/packages/microcosm-build/tests/test_us_full_original_amount_donors.py b/packages/microcosm-build/tests/test_us_full_original_amount_donors.py new file mode 100644 index 000000000..9030f27e9 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_full_original_amount_donors.py @@ -0,0 +1,455 @@ +"""Full original donor projections and tiny amount graphs over invented inputs.""" + +import json +import shutil +from dataclasses import replace +from fractions import Fraction + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_amount_donor as donors +from microcosm.build.us_runtime import current_survey_amounts as amounts +from microcosm.build.us_runtime import graph_us_survey_enrichment as graph + + +@pytest.mark.parametrize("option", [None, 1, "true", [], {}]) +def test_full_original_option_refuses_before_owner_access(option): + with pytest.raises(ValueError, match="FULL_ORIGINAL_DONORS_OPTION"): + amounts.qualify_current_survey_amounts(object(), full_original_donors=option) + with pytest.raises(ValueError, match="FULL_ORIGINAL_DONORS_OPTION"): + graph.Boundary( + object(), + groups=("unemployment",), + n_estimators=2, + full_original_amount_donors=option, + ) + with pytest.raises(ValueError, match="FULL_ORIGINAL_OPTION"): + donors.receipts._qualify_receipt_amount( + object(), family="unemployment", full_original=option + ) + + +def test_detached_frame_does_not_supply_full_donor_authority(): + with pytest.raises(ValueError, match="PREPARATION_TYPE"): + donors.qualify_full_original_amount_donor( + object(), amounts.selected_groups(("workers_compensation",)) + ) + + +def test_same_items_state_mapping_replacement_changes_live_seal(monkeypatch): + class ChangedMembership(dict): + def __contains__(self, key): + return True + + before = graph._live() + original = donors.US_STATE_NUMERIC_FIPS_TO_POSTAL + changed = ChangedMembership(original) + assert tuple(changed.items()) == tuple(original.items()) + assert 999 in changed and 999 not in original + monkeypatch.setattr(donors, "US_STATE_NUMERIC_FIPS_TO_POSTAL", changed) + assert graph._live() != before + + +def test_default_scope_is_selected_and_additional_source_is_opt_in(): + import inspect + + from test_us_native_workers_compensation import _invented_wc_family + + qualified, receiving = _invented_wc_family() + assert qualified.full_donor_source_frame is None + assert qualified.donor_source_frame is qualified.source_frame + nodes = graph.amount_nodes( + qualified, receiving, parent_digest="a" * 64, n_estimators=2 + ) + assert graph.FULL_DONOR_SOURCE_NODE not in {n.id for n in nodes} + for function in (graph.run_us_survey_enrichment, graph._construct, graph.Boundary): + assert ( + inspect.signature(function) + .parameters["full_original_amount_donors"] + .default + is False + ) + assert ( + inspect.signature(amounts.qualify_current_survey_amounts) + .parameters["full_original_donors"] + .default + is False + ) + assert ( + inspect.signature(donors.receipts._qualify_receipt_amount) + .parameters["full_original"] + .default + is False + ) + + +@pytest.mark.parametrize( + "function,target", + [(graph.run_us_survey_enrichment, "_construct"), (graph._construct, "Boundary")], +) +@pytest.mark.parametrize("enabled", [False, True]) +def test_optional_scope_forwarding(function, target, enabled, monkeypatch): + class StopError(Exception): + pass + + def observed(*args, **kwargs): + assert kwargs["full_original_amount_donors"] is enabled + raise StopError + + monkeypatch.setattr(graph, target, observed) + with pytest.raises(StopError): + function( + object(), + full_original_amount_donors=enabled, + groups=("unemployment",), + n_estimators=2, + ) + + +def test_genuine_full_donor_survives_receiving_selection(tmp_path, monkeypatch): + from test_us_native_workers_compensation import _wc_source_arguments + + args = _wc_source_arguments(tmp_path, monkeypatch) + partial = tmp_path / "partial" + shutil.copytree(args["source_dir"], partial) + request = json.loads((partial / "selection-request.json").read_bytes()) + request["fraction"], request["seed"] = [2, 3], 41 + (partial / "selection-request.json").write_text( + json.dumps(request, sort_keys=True, separators=(",", ":")) + ) + captures = tmp_path / "partial-captures" + captures.mkdir() + full = donors.source.prepare_authenticated_survey_population(**args) + selected = donors.source.prepare_authenticated_survey_population( + source_dir=partial, snapshot_root=captures, fraction=Fraction(2, 3), seed=41 + ) + specs = amounts.selected_groups( + ("unemployment", "health_costs", "workers_compensation") + ) + complete = donors.qualify_full_original_amount_donor( + full, specs, demographic_conditioning=True + ) + subset = donors.qualify_full_original_amount_donor( + selected, specs, demographic_conditioning=True + ) + native_selected = set(selected._checked()[2].source_frames[1].person.person_id) + assert native_selected < set(subset.features.index) + assert 105 not in native_selected + assert subset.amounts.loc[105, "WC_VAL"] == 120 + assert len(subset.features) == 4 + assert subset.frame.weights_for("household").kind.value == "design" + assert subset.frame.weights_for("household").values.tolist() == [2552.12, 100.0] + for entity in complete.frame.entities: + pd.testing.assert_frame_equal( + complete.frame.table(entity), subset.frame.table(entity), check_exact=True + ) + pd.testing.assert_frame_equal(complete.features, subset.features, check_exact=True) + pd.testing.assert_frame_equal(complete.amounts, subset.amounts, check_exact=True) + np.testing.assert_array_equal( + subset.amounts.loc[[105, 106, 107, 108], "WC_VAL"], [120.0, 0.0, np.nan, np.nan] + ) + assert set(subset.features) == set(amounts.predictors.DEMOGRAPHIC_FEATURES) + assert not any("last_year" in c or "prior" in c for c in subset.frame.person) + before = donors.source._frame_identity(selected._checked()[2].frame) + selected_receipt = donors.receipts._qualify_receipt_amount( + selected, family="workers_compensation" + ) + full_receipt = donors.receipts._qualify_receipt_amount( + selected, family="workers_compensation", full_original=True + ) + assert len(selected_receipt[0]) == 2 and len(full_receipt[0]) == 4 + assert "donor_scope" not in selected_receipt[1] + assert full_receipt[1]["donor_scope"] == "full_original_current_asec" + assert donors.source._frame_identity(selected._checked()[2].frame) == before + # Carry the genuine full-source projection into the real numerical nodes. + # The receiving family is explicitly an invented component fixture, not a + # replacement for a genuine full financial/enrichment host acceptance run. + _run_full_source_graph(tmp_path, monkeypatch, projection=subset) + + +def test_actual_full_source_filter_fit_apply_and_clone_replay(tmp_path, monkeypatch): + _run_full_source_graph(tmp_path, monkeypatch) + + +def _run_full_source_graph(tmp_path, monkeypatch, *, projection=None): + from types import SimpleNamespace + + from test_us_current_survey_health_coverage import _frame + from test_us_native_workers_compensation import _invented_wc_family + + from microcosm.fit import model_input + from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel + from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel + from microcosm.graph import ( + ArtifactOutput, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, + ) + from microcosm.graph import population as population_ops + + monkeypatch.setenv("POPULACE_FIT_N_JOBS", "1") + monkeypatch.setenv("POPULACE_FIT_PREDICT_WORKERS", "1") + qualified, receiving = _invented_wc_family() + spec = qualified.groups[0].spec + full = _frame(pd.DataFrame({"person_id": [2, 3, 999], "age": [40, 40, 80]})) + columns = pd.DataFrame( + { + qualified.features[0]: [40.0, 40.0, 80.0], + spec.targets[0]: [120.0, 0.0, 900.0], + }, + index=pd.Index([2, 3, 999], name="person_id"), + ) + keep = np.ones(3, dtype=bool) + matrix = qualified.groups[0].matrix + omitted = 999 + if projection is not None: + full = projection.frame + keep = np.isfinite(projection.features.to_numpy()).all(axis=1) & np.isfinite( + projection.amounts.WC_VAL.to_numpy() + ) + columns = projection.features.loc[keep].copy() + columns[spec.targets[0]] = projection.amounts.loc[keep, "WC_VAL"] + recipient = columns.loc[:, list(projection.features)].iloc[:1].copy() + recipient.index = pd.Index([1], name="person_id") + matrix = model_input.encode_recipient_matrix( + recipient, entity="person", entity_ids=np.array([1], dtype=" 0 + before = {e: receiving.table(e).copy(deep=True) for e in receiving.entities} + weight = receiving.weights_for("household") + weight_bytes, weight_kind = weight.values.tobytes(), weight.kind + nodes = graph.amount_nodes( + qualified, receiving, parent_digest="a" * 64, n_estimators=2 + ) + assert sum(n.id == graph.FULL_DONOR_SOURCE_NODE for n in nodes) == 1 + donor_id = graph._ids(group)[0] + assert ( + next(n for n in nodes if n.id == donor_id).base == graph.FULL_DONOR_SOURCE_NODE + ) + + class Receiving(KernelBase): + ref = "test.full-amount.receiving@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return "b" * 64 + + def run(self, context): + return KernelResult(frame=receiving) + + class Finalization(KernelBase): + ref = "test.full-amount.finalization@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def implementation_hash(self): + return "c" * 64 + + def run(self, context): + return KernelResult( + artifacts={"finalization": b"invented; no native parent authority"} + ) + + create = Node( + graph.parent.attach.FILTER_NODE, + Receiving.ref, + sources=(graph.health_graph.SOURCE_NAME,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned( + selection.entity, + c, + population_ops.token_for_dtype( + receiving.table(selection.entity)[c].dtype + ), + ) + for selection in graph.predictor_graph._inputs(receiving) + for c in selection.columns + ), + ) + finalization = Node( + graph.parent.attach.ATTACH_NODE, + Finalization.ref, + population=create.id, + artifact_outputs=( + ArtifactOutput("finalization", graph.parent.attach.FINALIZATION_TYPE), + ), + ) + compiled = compile_graph( + Graph( + "us", + (SourceRef(graph.health_graph.SOURCE_NAME, "raw-bytes-v1"),), + (create, finalization, *nodes), + ) + ) + original_seal = amounts.seal(qualified) + + def pure(): + assert amounts.seal(qualified) == original_seal + + def context(value): + pure() + assert value.node in nodes + return qualified + + boundary = SimpleNamespace( + qualified=qualified, + context=context, + pure=pure, + run=SimpleNamespace(population=SimpleNamespace(frame=receiving)), + parent_view=SimpleNamespace(digest="a" * 64), + n_estimators=2, + compiled=compiled, + ) + registry = KernelRegistry() + for kernel in ( + Receiving(), + Finalization(), + graph.CurrentSurveyAmountProjectionKernel(boundary), + graph.CurrentSurveyFullAmountSourceKernel(boundary), + graph.CurrentSurveyAmountDonorKernel(boundary), + graph.CurrentSurveyAmountColumnsKernel(boundary), + graph.CurrentSurveyAmountAttachKernel(boundary), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + ): + registry.register(kernel) + path = tmp_path / "invented.txt" + path.write_bytes(b"invented full-original amount graph") + store = ContentStore(tmp_path / "store") + kwargs = dict( + sources={graph.health_graph.SOURCE_NAME: path}, store=store, kernels=registry + ) + cold = run_graph(compiled, **kwargs) + warm = run_graph(compiled, **kwargs, resume="require") + assert cold.key == warm.key and all(n.hit for n in warm.nodes.values()) + fitted = warm.population(donor_id) + assert fitted.person.person_id.tolist() == columns.index.tolist() + assert fitted.person[spec.targets[0]].tolist() == columns[spec.targets[0]].tolist() + assert omitted in fitted.person.person_id.values + output = warm.population(create.id) + for e in receiving.entities: + pd.testing.assert_frame_equal( + output.table(e)[before[e].columns], before[e], check_exact=True + ) + assert output.weights_for("household").kind is weight_kind + assert output.weights_for("household").values.tobytes() == weight_bytes + # Native observed cells survive; only the eligible ACS original gets a draw. + expected = output.person.set_index("person_id").workers_compensation + assert expected.loc[2] == expected.loc[12] == 120 + assert expected.loc[3] == expected.loc[13] == 0 + assert np.isnan(expected.loc[0]) and np.isnan(expected.loc[10]) + assert expected.loc[1] == expected.loc[11] + loaded = { + (node.id, name): store.load_bytes(key) + for node in compiled.graph.nodes + for name, key in warm.node(node.id).opaque_artifacts.items() + } + graph._verify_models( + boundary, + loaded, + {donor_id: population_ops.Population.from_frame(fitted, donor_id)}, + ) + assert model_input.decode_recipient_matrix( + group.matrix + ).features.index.tolist() == [1] + pure() + + +@pytest.mark.parametrize( + "mutation,reason", + [ + ("axis", "FULL_DONOR_TABLE"), + ("value", "FULL_DONOR_TABLE"), + ("weight", "FULL_DONOR_WEIGHT_VALUES"), + ("weight_kind", "FULL_DONOR_WEIGHT_VALUES"), + ("strata", "FULL_DONOR_STRATA"), + ("tables", "FULL_DONOR_TABLES"), + ("declaration", "FULL_DONOR_INPUTS"), + ], +) +def test_full_donor_context_checks_exact_declared_projection(mutation, reason): + from types import SimpleNamespace + + from test_us_current_survey_health_coverage import _frame + + from microcosm.frame import WeightKind, Weights + + frame = _frame(pd.DataFrame({"person_id": [2, 9], "age": [40, 80]})) + inputs = graph._full_donor_inputs(frame) + assert len(inputs) == 1 and inputs[0].entity == "person" + columns = ( + ["person_id"] + + [frame.schema.membership_column(e) for e in frame.schema.group_entities] + + ["age"] + ) + original_weights = frame.resolve_weights("person") + context = SimpleNamespace( + node=SimpleNamespace(inputs=inputs), + tables={"person": frame.person.loc[:, columns].copy(deep=True)}, + weights={"person": original_weights}, + strata=frame.strata.copy(deep=True), + ) + graph._full_donor_context(context, frame) + if mutation == "axis": + context.tables["person"] = context.tables["person"].iloc[::-1] + elif mutation == "value": + context.tables["person"].loc[0, "age"] += 1 + elif mutation in {"weight", "weight_kind"}: + changed = original_weights.values.copy() + kind = original_weights.kind + if mutation == "weight": + changed[0] += 1 + else: + kind = WeightKind.IMPORTANCE + context.weights["person"] = Weights(changed, kind) + elif mutation == "strata": + context.strata.iloc[0] = "changed" + elif mutation == "tables": + context.tables["household"] = frame.table("household") + else: + context.node.inputs = () + with pytest.raises(ValueError, match=reason): + graph._full_donor_context(context, frame) + + +@pytest.mark.parametrize( + "name", + [ + "current_asec_amount_donor.py", + "current_asec_unemployment_source.py", + "current_survey_amounts.py", + "graph_us_survey_enrichment.py", + ], +) +def test_full_donor_changed_modules_preserve_source_classification(name): + import test_us_spine_blindness as guard + + assert not guard._non_owner_source_spine_accesses( + name, (guard._US_RUNTIME / name).read_text() + ) + if name == "current_asec_amount_donor.py": + assert name in guard._US_LAUNCH_GRAPH_RUNTIME_MODULES + assert name not in guard._SOURCE_SPINE_PROVENANCE_OWNERS + assert guard._non_owner_source_spine_accesses( + name, 'def wrong(f):\n return f.person["person_spine_source_id"]\n' + ) diff --git a/packages/microcosm-build/tests/test_us_full_puf_enrichment.py b/packages/microcosm-build/tests/test_us_full_puf_enrichment.py new file mode 100644 index 000000000..f173636ba --- /dev/null +++ b/packages/microcosm-build/tests/test_us_full_puf_enrichment.py @@ -0,0 +1,477 @@ +"""Ordinary canonical-column controls; no genuine donor/source authority mocks.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import full_puf_enrichment as full +from microcosm.build.us_runtime import puf_support as support +from microcosm.build.us_runtime.acs_income_universe import ( + apply_acs_pums_earnings_universe_zeros, +) +from microcosm.build.us_runtime.spine_assembly import assemble_spines +from microcosm.fit import model_input +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactOutput, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.codecs import load_frame_store +from microcosm.graph.keys import opaque_artifact_key + + +def _donor_columns(): + links = np.repeat(np.arange(1001, 1005, dtype=np.int64), 2) + person = pd.DataFrame( + { + "person_id": np.arange(101, 109, dtype=np.int64), + "person_tax_unit_id": links, + } + ) + for index, target in enumerate(full.PERSON_OUTPUTS): + person[target] = ( + np.arange(8) % 2 == 1 + if target in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + else (np.arange(8, dtype=np.float64) + 1) * (index + 1) + ) + person["short_term_capital_gains"] = [-10.0, 0.0, 5.0, 10.0, -3.0, 0.0, 12.0, 20.0] + tax_unit = pd.DataFrame( + { + "tax_unit_id": np.arange(1001, 1005, dtype=np.int64), + "weight": [1.0, 0.0, 2.0, 3.0], + "filing_status_code": [1.0, 2.0, 3.0, 4.0], + } + ) + for target in full.TAX_UNIT_OUTPUTS: + tax_unit[target] = ( + [0.0, 2010.0, 2020.0, 2024.0] + if target in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS + else [1.0, 2.0, 3.0, 4.0] + ) + pk = pd.DataFrame(True, index=person.index, columns=full.PERSON_OUTPUTS) + tk = pd.DataFrame( + True, + index=tax_unit.index, + columns=("weight", "filing_status_code", *full.TAX_UNIT_OUTPUTS), + ) + return person, tax_unit, pk, tk + + +def _donor(): + p, t, pk, tk = _donor_columns() + return full.canonical_full_puf_donor(p, t, person_known=pk, tax_unit_known=tk) + + +def _recipient(): + ids = np.arange(1, 5, dtype=np.int64) + links = np.repeat(ids, 2) + person = pd.DataFrame({"person_id": np.arange(1, 9, dtype=np.int64)}) + for entity in US_SCHEMA.group_entities: + person[US_SCHEMA.membership_column(entity)] = links + person["age"] = [14.0, 45.0] * 4 + for name, amount in ( + ("employment_income_before_lsr", 100.0), + ("self_employment_income_before_lsr", 20.0), + ("taxable_interest_income", 5.0), + ("qualified_dividend_income", 2.0), + ("non_qualified_dividend_income", 3.0), + ("short_term_capital_gains", 10.0), + ("long_term_capital_gains_before_response", 12.0), + ): + person[name] = [0.0, amount] * 4 + for name in support.PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS: + person[name] = [0.0, 10.0] * 4 + person["is_full_time_college_student"] = [False, True] * 4 + tables = { + entity: pd.DataFrame({US_SCHEMA.entity_id_column(entity): ids}) + for entity in US_SCHEMA.group_entities + } + tables["tax_unit"]["filing_status_input"] = [ + "SINGLE", + "JOINT", + "SEPARATE", + "HEAD_OF_HOUSEHOLD", + ] + tables["person"] = person + native = Frame( + tables, + US_SCHEMA, + {"household": Weights(np.array([1.0, 0.0, 2.0, 3.0]), WeightKind.DESIGN)}, + metadata={"scope": "invented_full_puf_contract", "source_admission": False}, + ) + asec = native.select(person.person_household_id.le(2)) + acs = native.select(person.person_household_id.gt(2)) + acs_person = acs.table("person") + child = acs_person.age.lt(15) + for mapped, raw in ( + ("employment_income_before_lsr", "WAGP"), + ("self_employment_income_before_lsr", "SEMP"), + ): + acs_person[raw] = acs_person[mapped] + acs_person.loc[child, [mapped, raw]] = np.nan + assembled = assemble_spines( + {"asec": asec, "acs": acs}, + household_mass_shares={"asec": 0.5, "acs": 0.5}, + ) + cloned = support.clone_us_frame_for_puf_support(assembled) + application = apply_acs_pums_earnings_universe_zeros( + cloned, boundary="invented full65 recipient ACS materialization" + ) + # This is the maintained source-universe operator, with its actual receipt; + # the mapped child zeros are produced here and the raw blanks survive. + assert application.receipt["structurally_absent_person_rows"] == 4 + people = application.frame.table("person") + acs_child = people[support.support_channel_column("person")].eq( + "acs" + ) & people.age.lt(15) + assert people.loc[acs_child, ["WAGP", "SEMP"]].isna().all().all() + return application.frame + + +def _known(frame): + table = frame.table("tax_unit") + mask = support.puf_tax_detail_clone_mask(table, entity="tax_unit") + return pd.DataFrame( + True, + columns=full.PREDICTORS, + index=pd.Index(table.loc[mask, "tax_unit_id"].to_numpy(), name="tax_unit_id"), + ) + + +def test_complete_canonical_reduction_retains_zero_weights_signed_values_and_mortgage(): + p, t, pk, tk = _donor_columns() + before = p.copy(deep=True) + donor = full.canonical_full_puf_donor(p, t, person_known=pk, tax_unit_known=tk) + assert len(full.TARGETS) == 65 and len(full.PERSON_OUTPUTS) == 56 + assert "prior_year_wages" not in full.TARGETS + assert tuple(donor) == (*full.PREDICTORS, *full.TARGETS, "weight") + assert donor.weight.tolist() == [1.0, 0.0, 2.0, 3.0] + assert donor.short_term_capital_gains.tolist() == [-10.0, 15.0, -3.0, 32.0] + assert donor.business_is_sstb.tolist() == [1.0] * 4 + # Canonical mortgage has already been interpreted upstream: no E19200 split. + np.testing.assert_array_equal( + donor.home_mortgage_interest, + p.home_mortgage_interest.groupby(p.person_tax_unit_id).sum(), + ) + pd.testing.assert_frame_equal(p, before) + + +@pytest.mark.parametrize( + "value", + [None, np.nan, np.inf, "0", True, 1 + 2j, pd.Timestamp("2024-01-01"), 2**53 + 1], +) +def test_rejects_bad_member_before_groupby_can_turn_it_into_a_valid_total(value): + p, t, pk, tk = _donor_columns() + p["taxable_interest_income"] = p.taxable_interest_income.astype(object) + p.loc[0, "taxable_interest_income"] = value + with pytest.raises( + ValueError, match="PUF_(UNKNOWN|NONFINITE|PHYSICAL_TYPE|FLOAT64_INTEGER_RANGE)" + ): + full.canonical_full_puf_donor(p, t, person_known=pk, tax_unit_known=tk) + + +def test_unknown_zero_and_numeric_boolean_are_not_complete_canonical_values(): + p, t, pk, tk = _donor_columns() + p.loc[0, "taxable_interest_income"] = 0.0 + pk.loc[0, "taxable_interest_income"] = False + with pytest.raises(ValueError, match="PUF_UNKNOWN:person.taxable_interest_income"): + full.canonical_full_puf_donor(p, t, person_known=pk, tax_unit_known=tk) + pk.loc[0, "taxable_interest_income"] = True + p["business_is_sstb"] = p.business_is_sstb.astype(np.int64) + with pytest.raises(ValueError, match="PUF_PHYSICAL_TYPE:person.business_is_sstb"): + full.canonical_full_puf_donor(p, t, person_known=pk, tax_unit_known=tk) + + +def test_explicit_return_grain_for_person_destination_and_collision_refusal(): + p, t, pk, tk = _donor_columns() + name = "self_employed_pension_contributions_desired" + t[name] = [10.0, 20.0, 30.0, 40.0] + tk[name] = True + tk = tk.loc[:, ["weight", "filing_status_code", name, *full.TAX_UNIT_OUTPUTS]] + kwargs = dict( + person_known=pk.drop(columns=name), + tax_unit_known=tk, + person_targets_at_tax_unit=(name,), + ) + with pytest.raises(ValueError, match="PUF_DONOR_GRAIN_COLLISION"): + full.canonical_full_puf_donor(p, t, **kwargs) + donor = full.canonical_full_puf_donor(p.drop(columns=name), t, **kwargs) + assert donor[name].tolist() == [10.0, 20.0, 30.0, 40.0] + + +def test_full_recipient_surface_keeps_zero_weight_rows_and_rejects_unknowns(): + frame, donor = _recipient(), _donor() + known = _known(frame) + inputs = full.prepare_full_puf_inputs(frame, donor, predictor_known=known) + matrix = model_input.decode_recipient_matrix(inputs.matrix) + assert len(matrix.features) == 4 and tuple(matrix.features) == full.PREDICTORS + assert matrix.entity_ids.tolist() == known.index.tolist() + assert inputs.recipient_universe["structurally_absent_person_rows"] == 2 + assert matrix.features[full.PREDICTORS[2]].eq(100.0).all() + assert matrix.features[full.PREDICTORS[3]].eq(20.0).all() + assert (frame.weights_for("household").values == 0).sum() == 2 + assert ( + inputs.recipient_universe["person_output_allocation"][ + "out_of_universe_person_rows" + ] + == 4 + ) + known.iloc[0, 2] = False + with pytest.raises(ValueError, match="PUF_UNKNOWN:recipient_predictor"): + full.prepare_full_puf_inputs(frame, donor, predictor_known=known) + known.iloc[0, 2] = True + person = frame.table("person") + person.loc[person.index[-1], "taxable_interest_income"] = np.nan + with pytest.raises(ValueError, match="missing values before coercion"): + full.prepare_full_puf_inputs(frame, donor, predictor_known=known) + + +def test_return_only_donor_does_not_invent_persons(): + expected = _donor() + _, returns, _, _ = _donor_columns() + columns = ( + "weight", + "filing_status_code", + "tax_unit_person_count", + *full.PERSON_OUTPUTS, + *full.TAX_UNIT_OUTPUTS, + ) + returns["tax_unit_person_count"] = expected[full.PREDICTORS[1]] + for name in full.PERSON_OUTPUTS: + returns[name] = expected[name] + known = pd.DataFrame(True, index=returns.index, columns=columns) + actual = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PERSON_OUTPUTS, + ) + pd.testing.assert_frame_equal(actual, expected) + + +class InventedSource(KernelBase): + ref = "test.full_puf.frame_source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + return KernelResult( + frame=load_frame_store(context.sources[context.node.sources[0]]) + ) + + +class InventedMatrix(KernelBase): + ref = "test.full_puf.matrix@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.PLATFORM_BITWISE + ) + + def run(self, context): + table = context.tables["tax_unit"] + ids = table.tax_unit_id.to_numpy() + features = table.loc[:, list(full.PREDICTORS)].copy() + features.index = pd.Index(ids, name="tax_unit_id") + return KernelResult( + artifacts={ + "matrix": model_input.encode_recipient_matrix( + features, entity="tax_unit", entity_ids=ids + ) + } + ) + + +def test_real_full_65_target_graph_draw_chain_finalization_and_replay(tmp_path): + frame, donor = _recipient(), _donor() + known = _known(frame) + prepared = full.prepare_full_puf_inputs(frame, donor, predictor_known=known) + matrix = model_input.decode_recipient_matrix(prepared.matrix) + # An explicit minimal recipient model Frame for the ordinary graph control; + # finalization below consumes the actual retained US clone Frame. + table = matrix.features.copy() + table.insert(0, "tax_unit_id", matrix.entity_ids) + from microcosm.frame import EntitySchema + + recipient = Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + { + "person_id": matrix.entity_ids, + "person_tax_unit_id": matrix.entity_ids, + } + ), + }, + EntitySchema(group_entities=("tax_unit",)), + { + "tax_unit": Weights( + frame.resolve_weights("tax_unit").values[ + support.puf_tax_detail_clone_mask( + frame.table("tax_unit"), entity="tax_unit" + ) + ], + WeightKind.DESIGN, + ) + }, + ) + store = ContentStore(tmp_path / "store", codecs={"frame-store": load_frame_store}) + sources, nodes = {}, [] + for name, value in (("donor", prepared.donor_frame), ("recipient", recipient)): + key = full.codec.sha(("invented-full-puf:" + name).encode()) + sources[name] = store.put_frame(key, value) + nodes.append( + Node( + name, + InventedSource.ref, + sources=(name,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned("tax_unit", col, "float64") + for col in value.table("tax_unit") + if col != "tax_unit_id" + ), + ) + ) + fits, applies = full.full_puf_train_apply_nodes( + donor_population="donor", + recipient_population="recipient", + matrix_producer="matrix", + seed=578, + n_estimators=2, + zero_atol=0, + ) + assert len(fits) == len(applies) == 65 + assert [ + e.name for e in applies[-1].artifact_inputs if e.name.startswith("prior_") + ] == [f"prior_{i:03d}" for i in range(64)] + nodes.append( + Node( + "matrix", + InventedMatrix.ref, + population="recipient", + inputs=(Slice("tax_unit", full.PREDICTORS),), + artifact_outputs=( + ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE), + ), + ) + ) + compiled = compile_graph( + Graph( + "us", + tuple(SourceRef(name, "frame-store") for name in sources), + (*nodes, *fits, *applies), + ) + ) + registry = KernelRegistry() + for kernel in ( + InventedSource(), + InventedMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + ): + registry.register(kernel) + actual = run_graph(compiled, sources=sources, store=store, kernels=registry) + + def payload(manifest, node_id, artifact): + record = manifest.node(node_id) + key = opaque_artifact_key(record.key, artifact) + assert key == record.opaque_artifacts[artifact] + return store.load_bytes(key) + + raw = { + target: payload(actual, node.id, "raw_draw") + for target, node in zip(full.TARGETS, applies, strict=True) + } + kwargs = dict( + matrix=payload(actual, "matrix", "matrix"), + matrix_producer_key=actual.node("matrix").key, + raw_draws=raw, + apply_state=payload(actual, applies[-1].id, "apply_state"), + training_state=payload(actual, fits[-1].id, "training_state"), + seed=578, + ) + assert kwargs["matrix"] == prepared.matrix + before = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + last_model = payload(actual, fits[-1].id, "model") + result, receipt = full.finalize_full_puf( + frame, donor, predictor_known=known, last_model=last_model, **kwargs + ) + assert receipt["release_eligible"] is False and receipt["tail_bounds"] + assert receipt["target_order"] == list(full.TARGETS) + for entity in frame.entities: + pd.testing.assert_frame_equal(frame.table(entity), before[entity]) + np.testing.assert_array_equal( + result.weights_for("household").values, frame.weights_for("household").values + ) + for entity, outputs in ( + ("person", full.PERSON_OUTPUTS), + ("tax_unit", full.TAX_UNIT_OUTPUTS), + ): + mask = support.puf_tax_detail_clone_mask(result.table(entity), entity=entity) + for output in outputs: + assert result.table(entity).loc[mask, output].notna().all() + if output not in before[entity]: + assert result.table(entity).loc[~mask, output].isna().all() + else: + np.testing.assert_array_equal( + result.table(entity).loc[~mask, output], + before[entity].loc[~mask, output], + ) + people = result.table("person") + child = support.puf_tax_detail_clone_mask(people, entity="person") & people.age.lt( + 15 + ) + assert people.loc[child, "employment_income_before_lsr"].eq(0).all() + changed_raw = dict(raw) + values = full.codec.read_raw_target( + raw[full.TARGETS[0]], target=full.TARGETS[0], index=matrix.features.index + ).copy() + values[0] += 1.0 + changed_raw[full.TARGETS[0]] = full.codec.encode_raw_target( + values, target=full.TARGETS[0], index=matrix.features.index + ) + with pytest.raises(ValueError, match="PUF_RAW_HISTORY"): + full.decode_full_puf_draws(**{**kwargs, "raw_draws": changed_raw}) + with pytest.raises(ValueError, match="PUF_FULL_MATRIX_BINDING"): + full.decode_full_puf_draws(**{**kwargs, "matrix_producer_key": "f" * 64}) + changed_donor = donor.copy(deep=True) + changed_donor.loc[0, "casualty_loss"] += 1.0 + with pytest.raises(ValueError, match="PUF_DONOR_CONSUMED_BYTES"): + full.finalize_full_puf( + frame, changed_donor, predictor_known=known, last_model=last_model, **kwargs + ) + incomplete = dict(raw) + incomplete.pop(full.TARGETS[-1]) + with pytest.raises(ValueError, match="PUF_RAW_ROSTER"): + full.decode_full_puf_draws(**{**kwargs, "raw_draws": incomplete}) + changed_training = full.codec.decode_json(kwargs["training_state"]) + changed_training["models"][0]["sha256"] = "e" * 64 + with pytest.raises(ValueError, match="PUF_FULL_CHAIN_IDENTITY"): + full.decode_full_puf_draws( + **{**kwargs, "training_state": full.codec.encode_json(changed_training)} + ) + warm = run_graph( + compiled, sources=sources, store=store, kernels=registry, resume="require" + ) + assert warm.key == actual.key and all(record.hit for record in warm.nodes.values()) + for node in applies: + assert payload(warm, node.id, "raw_draw") == payload( + actual, node.id, "raw_draw" + ) diff --git a/packages/microcosm-build/tests/test_us_full_puf_output_profiles.py b/packages/microcosm-build/tests/test_us_full_puf_output_profiles.py new file mode 100644 index 000000000..53fa59024 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_full_puf_output_profiles.py @@ -0,0 +1,1118 @@ +"""Ordinary profile controls with invented donors and real graph/QRF execution. + +The helper fixtures retain their explicit lack of source admission. These tests +run ordinary maintained fit/draw/finalizer and Population/store/replay paths; +they do not replace source qualification or an independent production gate. +""" + +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_full_puf_enrichment import ( + InventedMatrix, + InventedSource, + _donor_columns, + _known, +) +from test_us_graph_full_puf_enrichment import ( + InventedCompleteBoundary, + _complete_frame, + _copy_population, + _load_artifact, +) +from test_us_graph_full_puf_enrichment import ( + attached_full65 as original_attached_full65, +) + +from microcosm.build.us_runtime import full_puf_enrichment as full +from microcosm.build.us_runtime import graph_full_puf_enrichment as placement +from microcosm.build.us_runtime import puf_support as support +from microcosm.build.us_runtime import survey_population_replay as replay +from microcosm.build.us_runtime.graph_sources import frame_column_declarations +from microcosm.fit import model_input +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ContentStore, + Graph, + KernelRegistry, + KernelResult, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph import population as population_ops +from microcosm.graph.codecs import load_frame_store +from microcosm.graph.population import dtype_for_token + +SCF_MORTGAGE_OUTPUTS = ( + "first_home_mortgage_balance", + "second_home_mortgage_balance", + "first_home_mortgage_interest", + "second_home_mortgage_interest", + "first_home_mortgage_origination_year", + "second_home_mortgage_origination_year", +) +PUF59_TAX_UNIT_OUTPUTS = ( + "domestic_production_ald", + "unrecaptured_section_1250_gain", + "health_savings_account_ald", +) + +# Register the existing ordinary fixture unchanged, without invoking its body. +attached_full65 = original_attached_full65 + + +def _puf59_columns(): + person, tax_unit, person_known, tax_unit_known = _donor_columns() + # Invented upstream leaves, not a source measurement implementation or receipt. + tax_unit[full.PUF59.predictors[0]] = [1.0, 2.0, 3.0, 4.0] + tax_unit[full.PUF59.predictors[1]] = [3.0, 5.0, 2.0, 4.0] + for name in full.PUF59.predictors[:2]: + tax_unit_known[name] = True + return ( + person, + tax_unit.drop(columns=list(SCF_MORTGAGE_OUTPUTS)), + person_known, + tax_unit_known.drop(columns=list(SCF_MORTGAGE_OUTPUTS)), + ) + + +def _puf59_donor(): + person, tax_unit, person_known, tax_unit_known = _puf59_columns() + return full.canonical_full_puf_donor( + person, + tax_unit, + person_known=person_known, + tax_unit_known=tax_unit_known, + profile=full.PUF59, + ) + + +def _puf59_return_only_columns(): + """Invented return incidence bridge; no physical people or source admission.""" + reduced = _puf59_donor() + _, returns, _, _ = _puf59_columns() + for name in full.PUF59.person_outputs: + returns[name] = ( + np.array([0.0, 1.0, 1.0, 0.0]) + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + else reduced[name] + ) + returns["puf_person_incidence_capacity"] = 1.0 + known = pd.DataFrame( + True, + index=returns.index, + columns=( + "weight", + "filing_status_code", + "puf_person_incidence_capacity", + *full.PUF59.person_outputs, + *full.PUF59.tax_unit_outputs, + *full.PUF59.predictors[:2], + ), + ) + return returns, known + + +def _puf59_return_only_donor(): + returns, known = _puf59_return_only_columns() + return full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + + +def _puf59_frame(): + frame = _complete_frame() + tax_unit = frame.table("tax_unit") + tax_unit[full.PUF59.predictors[0]] = np.resize( + np.array([2.0, 4.0, 1.0, 3.0]), len(tax_unit) + ) + tax_unit[full.PUF59.predictors[1]] = np.resize( + np.array([5.0, 4.0, 3.0, 2.0]), len(tax_unit) + ) + return frame + + +def _puf59_known(frame): + return _known(frame).set_axis(full.PUF59.predictors, axis="columns") + + +class InventedProfileMatrix(InventedMatrix): + """Real ordinary projection of the declared selected predictor columns.""" + + ref = "test.puf_profile.matrix@1" + + def run(self, context): + table = context.tables["tax_unit"] + ids = table.tax_unit_id.to_numpy() + features = table.loc[:, list(context.node.inputs[0].columns)].copy() + features.index = pd.Index(ids, name="tax_unit_id") + return KernelResult( + artifacts={ + "matrix": model_input.encode_recipient_matrix( + features, entity="tax_unit", entity_ids=ids + ) + } + ) + + +def _finalizer_arguments(case, result): + artifacts = result.artifacts + return dict( + predictor_known=case.binding.predictor_known, + matrix=artifacts["matrix"].payload, + matrix_producer_key=artifacts["matrix"].producer_key, + raw_draws={ + target: artifacts[f"raw_{index:03d}"].payload + for index, target in enumerate(case.binding.profile.targets) + }, + apply_state=artifacts["apply_state"].payload, + training_state=artifacts["training_state"].payload, + last_model=artifacts["last_model"].payload, + seed=578, + ) + + +def _independent_puf59_population(binding, nodes, artifacts): + """Actual finalizer and graph patch, independent of attachment result code.""" + case = SimpleNamespace(binding=binding) + arguments = _finalizer_arguments(case, SimpleNamespace(artifacts=artifacts)) + candidate, receipt = full.finalize_full_puf( + binding.expected_population.frame, + binding.donor, + **arguments, + profile=full.PUF59, + ) + expected = binding.expected_population + masks = {} + for entity in ("person", "tax_unit"): + table = expected.frame.table(entity) + id_column = expected.frame.schema.entity_id_column(entity) + masks[entity] = pd.Series( + support.puf_tax_detail_clone_mask(table, entity=entity), + index=pd.Index(table[id_column], name=id_column), + dtype=bool, + ) + expected = population_ops.patch( + expected, + nodes[0], + KernelResult( + columns={ + (entity, placement.MASKS[entity]): mask + for entity, mask in masks.items() + } + ), + ) + columns = {} + for owned in nodes[1].outputs: + table = candidate.table(owned.entity) + mask = masks[owned.entity].to_numpy() + id_column = candidate.schema.entity_id_column(owned.entity) + columns[(owned.entity, owned.column)] = pd.Series( + table.loc[mask, owned.column].array, + index=pd.Index(table.loc[mask, id_column], name=id_column), + dtype=dtype_for_token(owned.dtype), + ) + return population_ops.patch( + expected, nodes[1], KernelResult(columns=columns) + ), receipt + + +@pytest.fixture(scope="module", params=("absent", "incumbent")) +def attached_puf59(tmp_path_factory, request): + root = tmp_path_factory.mktemp("puf59_" + request.param) + frame = _puf59_frame() + donor = _puf59_return_only_donor() if request.param == "absent" else _puf59_donor() + if request.param == "incumbent": + # The actual Frame contract requires global column-name uniqueness. + # These six details belong to tax units; the separate observed person + # home_mortgage_interest destination remains part of the PUF59 profile. + table = frame.table("tax_unit") + for index, name in enumerate(SCF_MORTGAGE_OUTPUTS): + # Independent incumbent bytes, including nulls and signed zero, + # survive on native AND clone rows without entering the PUF model. + table[name] = np.resize( + np.array([-0.0, np.nan, -12.5, 1800.5 + index]), len(table) + ) + assert name not in frame.person + frame.revalidate() + known = _puf59_known(frame) + prepared = full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + decoded = model_input.decode_recipient_matrix(prepared.matrix) + table = decoded.features.copy() + table.insert(0, "tax_unit_id", decoded.entity_ids) + matrix_frame = Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + { + "person_id": decoded.entity_ids, + "person_tax_unit_id": decoded.entity_ids, + } + ), + }, + EntitySchema(group_entities=("tax_unit",)), + {"tax_unit": Weights(np.ones(len(table), dtype="float64"), WeightKind.DESIGN)}, + ) + retained_store = ContentStore( + root / "retained", codecs={"frame-store": load_frame_store} + ) + sources, nodes = {}, [] + for name, value in ( + ("survey", frame), + ("donor", prepared.donor_frame), + ("matrix_input", matrix_frame), + ): + source = "invented." + name + sources[source] = retained_store.put_frame( + full.codec.sha(("ordinary-puf59:" + request.param + ":" + name).encode()), + value, + ) + outputs = ( + frame_column_declarations(value) + if name == "survey" + else tuple( + Owned("tax_unit", name, "float64") + for name in value.table("tax_unit") + if name != "tax_unit_id" + ) + ) + nodes.append( + Node( + name, + InventedSource.ref, + sources=(source,), + structural=StructuralDelta.CREATE, + outputs=outputs, + ) + ) + boundary = Node( + "complete_upstream", + InventedCompleteBoundary.ref, + base="survey", + structural=StructuralDelta.FILTER, + inputs=(Slice("person", ("age",)),), + ) + matrix_node = Node( + "matrix", + InventedProfileMatrix.ref, + population="matrix_input", + inputs=(Slice("tax_unit", full.PUF59.predictors),), + artifact_outputs=(ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE),), + ) + base_graph = Graph( + "us", + tuple(SourceRef(name, "frame-store") for name in sources), + (*nodes, boundary, matrix_node), + ) + kernels = KernelRegistry() + for kernel in ( + InventedSource(), + InventedCompleteBoundary(), + InventedProfileMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + ): + kernels.register(kernel) + observed = {} + retained_manifest = run_graph( + compile_graph(base_graph), + sources=sources, + store=retained_store, + kernels=kernels, + _population_observer=lambda name, value: observed.__setitem__(name, value), + ) + upstream = observed[boundary.id] + matrix = _load_artifact( + retained_manifest, + retained_store, + ArtifactInput( + "matrix", matrix_node.id, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ), + ) + assert matrix.payload == prepared.matrix + fits, applies = full.full_puf_train_apply_nodes( + donor_population="donor", + recipient_population=boundary.id, + matrix_producer=matrix_node.id, + seed=578, + n_estimators=2, + zero_atol=0, + profile=full.PUF59, + ) + retention_arguments = dict( + population=upstream, + expected_population=_copy_population(upstream), + population_node=boundary, + input_owners=dict(upstream.owners), + donor=donor, + predictor_known=known, + matrix=matrix, + fit_nodes=fits, + apply_nodes=applies, + profile=full.PUF59, + ) + binding = placement.retain_full_puf_attachment(**retention_arguments) + attachment_nodes = placement.full_puf_attachment_nodes(binding) + kernels.register(placement.FullPufMaskKernel(binding)) + kernels.register(placement.FullPufAttachKernel(binding)) + compiled = compile_graph( + replace( + base_graph, nodes=(*base_graph.nodes, *fits, *applies, *attachment_nodes) + ) + ) + store = ContentStore(root / "application", codecs={"frame-store": load_frame_store}) + results = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=lambda name, value, target=observed: ( + target.__setitem__(name, value) + ), + ) + assert all( + record.hit is (resume == "require") for record in manifest.nodes.values() + ) + artifacts, producer_keys = placement.load_full_puf_attachment_artifacts( + binding, compiled=compiled, manifest=manifest, store=store + ) + actual = observed[attachment_nodes[-1].id] + evidence = placement.verify_materialized_full_puf_attachment( + binding, + upstream_population=observed[boundary.id], + population=actual, + artifacts=artifacts, + producer_keys=producer_keys, + ) + expected, receipt = _independent_puf59_population( + binding, attachment_nodes, artifacts + ) + replay.same_replayed_population(expected, actual) + frame_key = full.codec.sha( + ( + "puf59-materialized:" + manifest.node(attachment_nodes[-1].id).key + ).encode() + ) + store.put_frame(frame_key, actual.frame) + replay.same_replayed_frame(expected.frame, store.load_frame(frame_key)) + results.append( + SimpleNamespace( + manifest=manifest, + population=_copy_population(actual), + upstream=_copy_population(observed[boundary.id]), + artifacts=artifacts, + producer_keys=producer_keys, + evidence=evidence, + finalizer_receipt=receipt, + ) + ) + replay.same_replayed_population(results[0].population, results[1].population) + assert results[0].artifacts == results[1].artifacts + assert results[0].manifest.key == results[1].manifest.key + return SimpleNamespace( + binding=binding, + retention_arguments=retention_arguments, + nodes=attachment_nodes, + fits=fits, + applies=applies, + results=results, + compiled=compiled, + sources=sources, + store=store, + kernels=kernels, + mortgage_mode=request.param, + ) + + +def test_closed_profiles_keep_observed_mortgage_and_original_conditioning_order(): + assert ( + full.FULL65.person_outputs == full.PUF59.person_outputs == full.PERSON_OUTPUTS + ) + assert full.FULL65.tax_unit_outputs == full.TAX_UNIT_OUTPUTS + assert full.FULL65.targets == full.TARGETS + assert full.PUF59.tax_unit_outputs == PUF59_TAX_UNIT_OUTPUTS + assert len(full.PUF59.person_outputs) == 56 and len(full.PUF59.targets) == 59 + assert full.PUF59.targets == tuple( + t for t in full.TARGETS if t not in SCF_MORTGAGE_OUTPUTS + ) + assert "home_mortgage_interest" in full.PUF59.person_outputs + assert not {"prior_year_wages", "employment_income_last_year"} & set( + full.PUF59.targets + ) + assert full.FULL65.predictors == full.PREDICTORS + assert full.FULL65.donor_auxiliary_columns == () + assert full.PUF59.donor_auxiliary_columns == ("puf_person_incidence_capacity",) + assert full.PUF59.predictors == ( + "puf_2015_filing_status_code", + "puf_2015_capped_return_size", + *full.PREDICTORS[2:], + ) + + +@pytest.mark.parametrize("invalid", (None, "puf59", "full65", 59, True, ("puf59",))) +def test_profiles_require_explicit_enum_members(invalid): + with pytest.raises(ValueError, match="PUF_OUTPUT_PROFILE"): + full.require_puf_output_profile(invalid) + + +def test_puf59_canonical_donor_accepts_truly_absent_scf_columns_and_keeps_observed_interest(): + person, tax_unit, pk, tk = _puf59_columns() + # Actual canonical arithmetic after an invented QBI allocation: entirely + # SSTB positive, ordinary positive, entirely SSTB loss, ordinary positive. + person["self_employment_income_before_lsr"] = [ + 0.0, + 0.0, + 50.0, + 25.0, + 0.0, + 0.0, + 700.0, + 300.0, + ] + person["sstb_self_employment_income_before_lsr"] = [ + 900.0, + 100.0, + 0.0, + 0.0, + -250.0, + -50.0, + 0.0, + 0.0, + ] + before = tuple(value.copy(deep=True) for value in (person, tax_unit, pk, tk)) + donor = full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + assert tuple(donor) == ( + *full.PUF59.predictors, + *full.PUF59.targets, + "weight", + "puf_person_incidence_capacity", + ) + assert not set(SCF_MORTGAGE_OUTPUTS) & set(donor) + np.testing.assert_array_equal( + donor.home_mortgage_interest, + person.home_mortgage_interest.groupby(person.person_tax_unit_id).sum(), + ) + assert donor.weight.tolist() == [1.0, 0.0, 2.0, 3.0] + np.testing.assert_array_equal( + donor[full.PUF59.predictors[3]], [1000.0, 75.0, -300.0, 1000.0] + ) + np.testing.assert_array_equal( + donor.self_employment_income_before_lsr, [0.0, 75.0, 0.0, 1000.0] + ) + np.testing.assert_array_equal( + donor.sstb_self_employment_income_before_lsr, [1000.0, 0.0, -300.0, 0.0] + ) + for actual, expected in zip((person, tax_unit, pk, tk), before, strict=True): + pd.testing.assert_frame_equal(actual, expected) + with pytest.raises(ValueError, match="PUF_DONOR_COLUMNS:tax_unit"): + full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk + ) + + +def test_puf59_donor_does_not_validate_excluded_years_or_consume_scf_payloads(): + person, tax_unit, pk, tk = _puf59_columns() + expected = _puf59_donor() + for name in SCF_MORTGAGE_OUTPUTS: + tax_unit[name] = ["independent SCF", None, False, 1800.5] + actual = full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + pd.testing.assert_frame_equal(actual, expected) + person["business_is_sstb"] = person.business_is_sstb.astype("float64") + with pytest.raises(ValueError, match="PUF_PHYSICAL_TYPE:person.business_is_sstb"): + full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + + +def test_puf59_return_only_donor_uses_explicit_leaves_without_inventing_persons(): + expected = _puf59_donor() + returns, known = _puf59_return_only_columns() + before = returns.copy(deep=True), known.copy(deep=True) + assert "tax_unit_person_count" not in returns + assert "tax_unit_person_count" not in known + for name in full.PUF59.person_outputs: + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + assert returns[name].dtype == np.dtype("float64") + assert set(returns[name]) == {0.0, 1.0} + expected[name] = returns[name] + expected["puf_person_incidence_capacity"] = 1.0 + actual = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + pd.testing.assert_frame_equal(actual, expected) + assert bool((returns.sstb_self_employment_income_before_lsr != 0).any()) + np.testing.assert_array_equal( + actual[full.PUF59.predictors[3]], + returns.self_employment_income_before_lsr + + returns.sstb_self_employment_income_before_lsr, + ) + pd.testing.assert_frame_equal(returns, before[0]) + pd.testing.assert_frame_equal(known, before[1]) + + +@pytest.mark.parametrize( + ("failure", "error"), + ( + ("absent", "PUF_DONOR_COLUMNS:tax_unit"), + ("physical_count_alias", "PUF_DONOR_COLUMNS:tax_unit"), + ("unknown", "PUF_UNKNOWN:tax_unit.puf_person_incidence_capacity"), + ("missing_knownness", "PUF_KNOWNNESS_AXIS:tax_unit"), + ), +) +def test_puf59_return_only_requires_exact_known_incidence_capacity(failure, error): + returns, known = _puf59_return_only_columns() + name = "puf_person_incidence_capacity" + if failure == "absent": + returns.drop(columns=name, inplace=True) + elif failure == "physical_count_alias": + returns.rename(columns={name: "tax_unit_person_count"}, inplace=True) + known.rename(columns={name: "tax_unit_person_count"}, inplace=True) + elif failure == "unknown": + known.loc[0, name] = False + else: + known.drop(columns=name, inplace=True) + with pytest.raises(ValueError, match=error): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + + +@pytest.mark.parametrize( + ("value", "error"), + ( + (0.0, "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN"), + (-1.0, "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN"), + (0.5, "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN"), + ("1", "PUF_PHYSICAL_TYPE:puf_person_incidence_capacity"), + (True, "PUF_PHYSICAL_TYPE:puf_person_incidence_capacity"), + (None, "PUF_UNKNOWN:puf_person_incidence_capacity"), + (np.inf, "PUF_NONFINITE:puf_person_incidence_capacity"), + ), +) +def test_puf59_return_only_incidence_capacity_requires_physical_positive_integer( + value, error +): + returns, known = _puf59_return_only_columns() + name = "puf_person_incidence_capacity" + returns[name] = returns[name].astype(object) + returns.loc[0, name] = value + with pytest.raises(ValueError, match=error): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + + +def test_puf59_return_only_rejects_boolean_incidence_above_source_capacity(): + returns, known = _puf59_return_only_columns() + returns.loc[0, "business_is_sstb"] = 2.0 + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + donor = _puf59_return_only_donor() + donor.loc[0, "business_is_sstb"] = 2.0 + frame = _puf59_frame() + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +@pytest.mark.parametrize("predictor_index", range(8)) +def test_puf59_requires_all_eight_known_predictors(predictor_index): + frame, donor = _puf59_frame(), _puf59_donor() + known = _puf59_known(frame) + known.iloc[0, predictor_index] = False + with pytest.raises(ValueError, match="PUF_UNKNOWN:recipient_predictor"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + known.iloc[0, predictor_index] = True + with pytest.raises(ValueError, match="PUF_KNOWNNESS_AXIS:recipient_predictor"): + full.prepare_full_puf_inputs( + frame, + donor, + predictor_known=known.drop(columns=known.columns[predictor_index]), + profile=full.PUF59, + ) + + +@pytest.mark.parametrize("predictor_index", (0, 1)) +def test_puf59_requires_new_leaves_and_never_falls_back_to_generic_measurements( + predictor_index, +): + frame, donor = _puf59_frame(), _puf59_donor() + known = _puf59_known(frame) + before = frame.table("tax_unit").filing_status_input.copy(deep=True) + prepared = full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + assert prepared.profile is full.PUF59 + decoded = model_input.decode_recipient_matrix(prepared.matrix) + assert tuple(decoded.features) == full.PUF59.predictors + tax_unit = frame.table("tax_unit") + mask = support.puf_tax_detail_clone_mask(tax_unit, entity="tax_unit") + for name in full.PUF59.predictors[:2]: + np.testing.assert_array_equal(decoded.features[name], tax_unit.loc[mask, name]) + pd.testing.assert_series_equal(tax_unit.filing_status_input, before) + name = full.PUF59.predictors[predictor_index] + tax_unit.drop(columns=name, inplace=True) + with pytest.raises(ValueError, match="PUF_PROFILE_PREDICTOR_SOURCE:" + name): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + # A same-named person column is also insufficient for the explicit return-grain leaf. + frame.table("person")[name] = 1.0 + with pytest.raises(ValueError, match="PUF_PROFILE_PREDICTOR_SOURCE:" + name): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + person, returns, pk, tk = _puf59_columns() + with pytest.raises(ValueError, match="PUF_DONOR_COLUMNS:tax_unit"): + full.canonical_full_puf_donor( + person, + returns.drop(columns=name), + person_known=pk, + tax_unit_known=tk.drop(columns=name), + profile=full.PUF59, + ) + + +@pytest.mark.parametrize("value", (0.5, 6.0)) +def test_puf59_prepare_still_rejects_invalid_selected_boolean_counts(value): + frame, donor = _puf59_frame(), _puf59_donor() + donor.loc[0, "business_is_sstb"] = value + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +@pytest.mark.parametrize( + ("predictor_index", "value", "error"), + ( + (0, "1", "PUF_PHYSICAL_TYPE"), + (1, "2", "PUF_PHYSICAL_TYPE"), + (0, True, "PUF_PHYSICAL_TYPE"), + (1, False, "PUF_PHYSICAL_TYPE"), + (0, None, "missing values before coercion"), + (1, np.inf, "PUF_NONFINITE"), + (0, 5.0, "PUF_PROFILE_FILING_STATUS_DOMAIN"), + (1, 1.5, "PUF_PROFILE_RETURN_SIZE_DOMAIN"), + ), +) +def test_puf59_measured_predictor_leaves_require_physical_values_and_domains( + predictor_index, value, error +): + frame, donor = _puf59_frame(), _puf59_donor() + tax_unit = frame.table("tax_unit") + selected = support.puf_tax_detail_clone_mask(tax_unit, entity="tax_unit") + name = full.PUF59.predictors[predictor_index] + tax_unit[name] = tax_unit[name].astype(object) + tax_unit.loc[tax_unit.index[selected][0], name] = value + with pytest.raises(ValueError, match=error): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +def test_puf59_person_donor_derives_incidence_capacity_from_actual_membership(): + person, returns, pk, tk = _puf59_columns() + person.loc[:1, "business_is_sstb"] = True + returns.loc[0, full.PUF59.predictors[1]] = 1.0 + # The bound is derived from membership here, not read as a return source fact. + returns["puf_person_incidence_capacity"] = "not a source fact" + returns["tax_unit_person_count"] = "not a source fact" + donor = full.canonical_full_puf_donor( + person, returns, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + assert ( + donor.loc[0, "business_is_sstb"] + == donor.loc[0, "puf_person_incidence_capacity"] + == 2 + ) + assert donor.loc[0, full.PUF59.predictors[1]] == 1 + frame = _puf59_frame() + prepared = full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + for auxiliary in ("puf_person_incidence_capacity", "tax_unit_person_count"): + assert auxiliary not in prepared.donor + assert auxiliary not in prepared.donor_frame.table("tax_unit") + donor.loc[0, "puf_person_incidence_capacity"] = 1.0 + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +@pytest.mark.parametrize( + "year", + ("first_home_mortgage_origination_year", "second_home_mortgage_origination_year"), +) +def test_default_full65_retains_selected_discrete_year_checks(year): + person, tax_unit, pk, tk = _donor_columns() + tax_unit.loc[0, year] = 2010.5 + with pytest.raises(ValueError, match="PUF_YEAR_DOMAIN:" + year): + full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk + ) + + +def test_default_full65_return_only_retains_legacy_person_count_contract(): + person, returns, person_known, tax_unit_known = _donor_columns() + for name in full.FULL65.person_outputs: + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + person.loc[:1, name] = True + expected = full.canonical_full_puf_donor( + person, + returns, + person_known=person_known, + tax_unit_known=tax_unit_known, + ) + returns["tax_unit_person_count"] = 2.0 + for name in full.FULL65.person_outputs: + returns[name] = expected[name] + known = pd.DataFrame( + True, + index=returns.index, + columns=( + "weight", + "filing_status_code", + "tax_unit_person_count", + *full.FULL65.person_outputs, + *full.FULL65.tax_unit_outputs, + ), + ) + assert "puf_person_incidence_capacity" not in returns + for arguments in ({}, {"profile": full.FULL65}): + actual = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.FULL65.person_outputs, + **arguments, + ) + pd.testing.assert_frame_equal(actual, expected) + assert actual.loc[0, "business_is_sstb"] == 2.0 + # FULL65 compatibility is explicit: its historical alias excludes the + # nonzero SSTB component rather than silently changing old evidence. + assert bool((returns.sstb_self_employment_income_before_lsr != 0).any()) + np.testing.assert_array_equal( + actual[full.FULL65.predictors[3]], returns.self_employment_income_before_lsr + ) + returns["puf_person_incidence_capacity"] = 1.0 + actual = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.FULL65.person_outputs, + ) + pd.testing.assert_frame_equal(actual, expected) + returns.drop(columns="tax_unit_person_count", inplace=True) + with pytest.raises(ValueError, match="PUF_DONOR_COLUMNS:tax_unit"): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.FULL65.person_outputs, + ) + + +def test_real_puf59_chain_finalization_attachment_and_required_replay(attached_puf59): + case = attached_puf59 + assert case.binding.profile is full.PUF59 + assert len(case.fits) == len(case.applies) == 59 + assert [ + edge.name + for edge in case.applies[-1].artifact_inputs + if edge.name.startswith("prior_") + ] == [f"prior_{i:03d}" for i in range(58)] + assert tuple(owned.column for owned in case.nodes[-1].outputs) == full.PUF59.targets + for node in (*case.fits, *case.applies): + assert node.params["phase"] == full.PUF59.phase + assert not { + *SCF_MORTGAGE_OUTPUTS, + "puf_person_incidence_capacity", + "tax_unit_person_count", + } & {column for item in node.inputs for column in item.columns} + assert all(node.params["predictors"] == full.PUF59.predictors for node in case.fits) + assert all(node.params["targets"] == full.PUF59.targets for node in case.fits) + assert "raw_058" in case.results[0].artifacts + assert "raw_059" not in case.results[0].artifacts + if case.mortgage_mode == "absent": + assert case.binding.donor.puf_person_incidence_capacity.eq(1.0).all() + for name in full.PUF59.person_outputs: + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + assert set(case.binding.donor[name]) == {0.0, 1.0} + for result in case.results: + assert result.finalizer_receipt["output_profile"] == "puf59" + assert result.finalizer_receipt["person_target_count"] == 56 + assert result.finalizer_receipt["tax_unit_target_count"] == 3 + assert result.finalizer_receipt["target_count"] == 59 + assert result.finalizer_receipt["predictor_order"] == list( + full.PUF59.predictors + ) + assert result.finalizer_receipt["donor_auxiliary_columns"] == [ + "puf_person_incidence_capacity" + ] + assert result.finalizer_receipt["target_order"] == list(full.PUF59.targets) + assert result.finalizer_receipt["person_target_count"] == 56 + assert result.finalizer_receipt["tax_unit_target_count"] == 3 + assert result.finalizer_receipt["target_count"] == 59 + assert result.evidence["output_profile"] == "puf59" + assert result.evidence["source_admission_issued"] is False + assert result.evidence["release_eligible"] is False + for entity in ("person", "tax_unit"): + before, after = ( + result.upstream.frame.table(entity), + result.population.frame.table(entity), + ) + for name in SCF_MORTGAGE_OUTPUTS: + if case.mortgage_mode == "absent" or entity != "tax_unit": + assert name not in before and name not in after + else: + assert population_ops.storage_equal( + before[name], after[name], np.ones(len(before), dtype=bool) + ) + assert ( + result.population.owners[(entity, name)] + == result.upstream.owners[(entity, name)] + ) + for owned in case.nodes[-1].outputs: + before = result.upstream.frame.table(owned.entity) + after = result.population.frame.table(owned.entity) + mask = support.puf_tax_detail_clone_mask(before, entity=owned.entity) + assert after.loc[mask, owned.column].notna().all() + if owned.column in before: + assert population_ops.storage_equal( + before[owned.column], after[owned.column], ~mask + ) + else: + assert after.loc[~mask, owned.column].isna().all() + + +def test_puf59_incidence_capacity_remains_bound_to_attachment(attached_puf59): + case = attached_puf59 + donor = case.binding.donor.copy(deep=True) + donor["puf_person_incidence_capacity"] += 1.0 + stale = replace(case.binding, donor=donor) + with pytest.raises(ValueError, match="FULL_PUF_DONOR_CHANGED"): + placement.full_puf_attachment_nodes(stale) + rebound = placement.retain_full_puf_attachment( + **{**case.retention_arguments, "donor": donor} + ) + assert rebound.fit_nodes == case.binding.fit_nodes + assert rebound.apply_nodes == case.binding.apply_nodes + assert rebound.matrix == case.binding.matrix + nodes = placement.full_puf_attachment_nodes(rebound) + for original, changed in zip(case.nodes, nodes, strict=True): + assert original.params["donor_values"] != changed.params["donor_values"] + + +def test_default_full65_matches_explicit_full65_and_runs_actual_complete_chain( + attached_full65, +): + case = attached_full65 + kwargs = dict( + donor_population="donor", + recipient_population="complete_upstream", + matrix_producer="matrix", + seed=578, + n_estimators=2, + zero_atol=0, + ) + default = full.full_puf_train_apply_nodes(**kwargs) + explicit = full.full_puf_train_apply_nodes(**kwargs, profile=full.FULL65) + assert default == explicit == (case.fits, case.applies) + assert len(case.fits) == len(case.applies) == 65 + assert tuple(owned.column for owned in case.nodes[-1].outputs) == full.TARGETS + assert case.binding.profile is full.FULL65 + assert set(SCF_MORTGAGE_OUTPUTS) <= set(full.FULL65.tax_unit_outputs) + for result in case.results: + assert "raw_064" in result.artifacts + assert result.finalizer_receipt["output_profile"] == "full65" + assert result.finalizer_receipt["target_order"] == list(full.TARGETS) + + +def test_actual_wrong_profile_chains_and_manifests_are_refused( + attached_puf59, attached_full65 +): + for correct, wrong in ( + (attached_puf59, attached_full65), + (attached_full65, attached_puf59), + ): + with pytest.raises(ValueError, match="FULL_PUF_CHAIN_ROSTER"): + placement.retain_full_puf_attachment( + **{**correct.retention_arguments, "profile": wrong.binding.profile} + ) + result = correct.results[-1] + arguments = _finalizer_arguments(correct, result) + arguments.pop("predictor_known") + arguments.pop("last_model") + with pytest.raises(ValueError, match="PUF_"): + full.decode_full_puf_draws(**arguments, profile=wrong.binding.profile) + # Feed the real other-profile compiled graph, manifest and ContentStore; + # no fabricated manifest, source issuer or artifact readiness is involved. + with pytest.raises(ValueError, match="FULL_PUF_"): + placement.load_full_puf_attachment_artifacts( + correct.binding, + compiled=wrong.compiled, + manifest=wrong.results[-1].manifest, + store=wrong.store, + ) + + +def test_puf59_wrong_profile_model_and_trimmed_full65_history_are_refused( + attached_puf59, attached_full65 +): + case, result = attached_puf59, attached_puf59.results[-1] + arguments = _finalizer_arguments(case, result) + arguments["last_model"] = ( + attached_full65.results[-1].artifacts["last_model"].payload + ) + with pytest.raises(ValueError, match="Legacy QRF artifact content digest mismatch"): + full.finalize_full_puf( + case.binding.expected_population.frame, + case.binding.donor, + **arguments, + profile=full.PUF59, + ) + full_arguments = _finalizer_arguments(attached_full65, attached_full65.results[-1]) + full_arguments.pop("predictor_known") + full_arguments.pop("last_model") + full_arguments["raw_draws"] = { + target: full_arguments["raw_draws"][target] for target in full.PUF59.targets + } + with pytest.raises(ValueError, match="PUF_"): + full.decode_full_puf_draws(**full_arguments, profile=full.PUF59) + + +def test_puf59_rebuilt_binding_survives_nullable_frame_store_normalization( + attached_puf59, tmp_path +): + case = attached_puf59 + retained = _copy_population(case.binding.expected_population) + nullable = retained.frame.table("person").unrelated_nullable.array + missing = nullable._mask.copy() + assert missing.any() + nullable._data[missing] = True + arguments = {**case.retention_arguments, "expected_population": retained} + physical_binding = placement.retain_full_puf_attachment(**arguments) + assert placement.full_puf_attachment_nodes(physical_binding) == case.nodes + + directory = tmp_path / "puf59-new-session" + key = full.codec.sha(b"invented-puf59-retained-upstream") + ContentStore(directory).put_frame(key, retained.frame) + # Fresh ContentStore decoding performs the actual v2 masked-byte normalization. + restored_frame = ContentStore(directory).load_frame(key) + assert ( + not restored_frame.table("person").unrelated_nullable.array._data[missing].any() + ) + replay.same_replayed_frame(retained.frame, restored_frame) + restored = replace(_copy_population(retained), frame=restored_frame) + upstream = _copy_population(case.results[-1].upstream) + rebuilt = placement.retain_full_puf_attachment( + **{ + **arguments, + "population": upstream, + "expected_population": restored, + "input_owners": dict(upstream.owners), + } + ) + assert physical_binding.expected_stamp != rebuilt.expected_stamp + nodes = placement.full_puf_attachment_nodes(rebuilt) + assert nodes == case.nodes + replacements = {node.id: node for node in nodes} + compiled = compile_graph( + replace( + case.compiled.graph, + nodes=tuple( + replacements.get(node.id, node) for node in case.compiled.graph.nodes + ), + ) + ) + registry = KernelRegistry() + for kernel in ( + InventedSource(), + InventedCompleteBoundary(), + InventedProfileMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + placement.FullPufMaskKernel(rebuilt), + placement.FullPufAttachKernel(rebuilt), + ): + registry.register(kernel) + observed = {} + manifest = run_graph( + compiled, + sources=case.sources, + store=case.store, + kernels=registry, + resume="require", + _population_observer=lambda name, value: observed.__setitem__(name, value), + ) + assert all(record.hit for record in manifest.nodes.values()) + assert manifest.key == case.results[-1].manifest.key + artifacts, producer_keys = placement.load_full_puf_attachment_artifacts( + rebuilt, compiled=compiled, manifest=manifest, store=case.store + ) + evidence = placement.verify_materialized_full_puf_attachment( + rebuilt, + upstream_population=observed[rebuilt.population_node.id], + population=observed[nodes[-1].id], + artifacts=artifacts, + producer_keys=producer_keys, + ) + assert evidence["output_profile"] == "puf59" + replay.same_replayed_population(case.results[-1].population, observed[nodes[-1].id]) diff --git a/packages/microcosm-build/tests/test_us_geography_integration.py b/packages/microcosm-build/tests/test_us_geography_integration.py new file mode 100644 index 000000000..7e2521de9 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_geography_integration.py @@ -0,0 +1,276 @@ +"""Ordinary invented geography controls for the combined graph contract. + +All NPZ/CSV bytes are created in the test. The national-sized fixture satisfies +the shape validator with invented equal cell masses and district allocations; +it is not a Census crosswalk or evidence of genuine source admission. +""" + +import hashlib +import json +from collections import Counter +from dataclasses import fields +from io import BytesIO + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_geography as geography +from microcosm.build.us_runtime.congressional_district_vintage import ( + CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, + SOURCE_CONGRESSIONAL_DISTRICT_PREFIX, +) +from microcosm.build.us_runtime.puma_ladder import ( + assign_us_puma_ladder, + decode_us_puma_ladder, + load_us_puma_ladder, +) +from microcosm.frame import WeightKind, Weights +from microcosm.graph import ArtifactValue, KernelContext, Owned +from microcosm.graph.kernel import NumericScope + + +def _ordinary_lookup_bytes(**overrides): + # The production validator requires 436 districts across 51 jurisdictions. + # Allocate invented counts: 7 at-large + 33*10 + 11*9 = 436. + states = [i for i in range(1, 57) if i not in {3, 7, 14, 43, 52}] + at_large = {2, 10, 11, 38, 46, 50, 56} + multi = [i for i in states if i not in at_large] + rows, crosswalk = [], [] + for state in states: + districts = ( + [0] + if state in at_large + else range(1, 11 if multi.index(state) < 33 else 10) + ) + for district in districts: + county = max(1, district) + if state == 36: + county = 61 if district == 1 else (1 if district == 2 else district) + puma = state * 100_000 + 100 + tract = (state * 1000 + county) * 1_000_000 + 100 + cd = state * 100 + district + rows.append((puma, tract, cd)) + crosswalk.append( + { + "source_geography_id": f"{SOURCE_CONGRESSIONAL_DISTRICT_PREFIX}{cd:04d}", + "target_geography_id": f"{CURRENT_CONGRESSIONAL_DISTRICT_PREFIX}{cd:04d}", + "weight": 1.0, + "pair_population": 1, + } + ) + rows.sort() + pumas = Counter(row[0] for row in rows) + arrays = { + "puma": np.asarray(sorted(pumas), dtype=np.int64), + "puma_population": np.asarray([pumas[p] for p in sorted(pumas)]), + "joint_overlap_puma": np.asarray([r[0] for r in rows]), + "joint_overlap_tract": np.asarray([r[1] for r in rows]), + "joint_overlap_cd": np.asarray([r[2] for r in rows]), + "joint_overlap_population": np.ones(len(rows), dtype=np.int64), + "metadata_json": np.asarray( + json.dumps( + { + "schema_version": 2, + "kind": "us_puma_ladder", + "puma_vintage": "2020_puma", + "sampling_basis": "population", + "layers": { + name: { + "vintage": vintage, + "source": "invented ordinary fixture", + } + for name, vintage in ( + ("congressional_district", "119th_congress"), + ("county", "2020_census"), + ("tract", "2020_census"), + ) + }, + } + ) + ), + } + for layer, coordinate in (("cd", 2), ("county", 1), ("tract", 1)): + counts = Counter( + ( + row[0], + row[coordinate] // 1_000_000 if layer == "county" else row[coordinate], + ) + for row in rows + ) + keys = sorted(counts) + arrays[f"{layer}_overlap_puma"] = np.asarray([key[0] for key in keys]) + arrays[f"{layer}_overlap_{layer}"] = np.asarray([key[1] for key in keys]) + arrays[f"{layer}_overlap_population"] = np.asarray( + [counts[key] for key in keys] + ) + arrays.update(overrides) + stream = BytesIO() + np.savez_compressed(stream, **arrays) + return stream.getvalue(), pd.DataFrame(crosswalk).to_csv(index=False).encode() + + +def _nodes(): + return geography.us_geography_nodes( + (Owned("person", "age", "int64"), Owned("household", "state_fips", "int64")), + base="ordinary_base", + context_producer="ordinary_context", + ) + + +def _context(household, weights, ladder_bytes, crosswalk_bytes): + artifacts = {} + for alias, payload, type_ in ( + ("ladder", ladder_bytes, geography.US_PUMA_LOOKUP_TYPE), + ("crosswalk", crosswalk_bytes, geography.US_CD_CROSSWALK_TYPE), + ): + artifacts[alias] = ArtifactValue( + payload, + type_, + hashlib.sha256(payload).hexdigest(), + "a" * 64, + NumericScope(), + ) + node = _nodes()[-1] + return KernelContext( + node=node, + tables={"household": household}, + weights={"household": Weights(np.asarray(weights), WeightKind.DESIGN)}, + strata=pd.Series(dtype="int64"), + params=node.params, + rng=np.random.default_rng(0), + artifacts=artifacts, + ) + + +def _households(): + # Same ordinary mass control as test_us_puma_ladder._gated_household: + # NYC = 2.6% nationally and 2.6/6.2 of New York's mass. + return pd.DataFrame( + { + "household_id": [1, 2, 3, 4, 5], + "state_fips": [36, 36, 6, 48, 1], + "puma": ["3600100", "3600100", "0600100", "4800100", "0100100"], + "county_fips": ["36061", "36001", "06001", "48001", "01001"], + "congressional_district_geoid": [3601, 3602, 601, 4801, 101], + } + ) + + +def test_byte_and_path_joint_decode_and_assignments_match(tmp_path): + payload, _ = _ordinary_lookup_bytes() + path = tmp_path / "invented-ladder.npz" + path.write_bytes(payload) + path_ladder = load_us_puma_ladder(path) + byte_ladder = decode_us_puma_ladder(payload) + for field in fields(path_ladder): + left, right = getattr(path_ladder, field.name), getattr(byte_ladder, field.name) + if isinstance(left, np.ndarray): + np.testing.assert_array_equal(left, right) + assert left.dtype == right.dtype + else: + assert left == right + for assign_tract in (False, True): + pd.testing.assert_frame_equal( + assign_us_puma_ladder( + _households(), path_ladder, seed=17, assign_tract=assign_tract + ), + assign_us_puma_ladder( + _households(), byte_ladder, seed=17, assign_tract=assign_tract + ), + ) + + +@pytest.mark.parametrize("defect", ["schema_v1", "joint_population"]) +def test_byte_and_path_reject_the_same_joint_defect(tmp_path, defect): + payload, _ = _ordinary_lookup_bytes() + with np.load(BytesIO(payload), allow_pickle=False) as archive: + arrays = {key: archive[key] for key in archive.files} + if defect == "schema_v1": + metadata = json.loads(str(arrays["metadata_json"].item())) + metadata["schema_version"] = 1 + arrays["metadata_json"] = np.asarray(json.dumps(metadata)) + else: + arrays["joint_overlap_population"][0] += 1 + path = tmp_path / "invalid-invented-ladder.npz" + np.savez_compressed(path, **arrays) + with pytest.raises(ValueError) as path_error: + load_us_puma_ladder(path) + with pytest.raises(ValueError) as byte_error: + decode_us_puma_ladder(path.read_bytes()) + assert str(path_error.value) == str(byte_error.value) + + +@pytest.mark.parametrize("wrapper", [bytearray, memoryview]) +def test_byte_decoder_requires_immutable_bytes(wrapper): + with pytest.raises(TypeError, match="immutable bytes"): + decode_us_puma_ladder(wrapper(b"not an archive")) + + +@pytest.mark.parametrize("unsupported_weight", [0.0, 1.0]) +def test_graph_gate_rejects_real_unsupported_pair_even_at_zero_weight( + unsupported_weight, +): + ladder_bytes, crosswalk_bytes = _ordinary_lookup_bytes() + household = _households() + weights = [2.6, 3.6, 73.8, 20.0, unsupported_weight] + kernel = geography.USGeographyGateKernel() + assert ( + kernel.run(_context(household, weights, ladder_bytes, crosswalk_bytes)).receipt[ + "outcome" + ] + == "pass" + ) + # County 01001 and CD 102 are individually present in PUMA 0100100, + # but the only supported pairs are county 01001/CD 101, county 01002/CD 102. + household.loc[4, "congressional_district_geoid"] = 102 + result = kernel.run(_context(household, weights, ladder_bytes, crosswalk_bytes)) + assert result.receipt["outcome"] == "fail" + expected_mass = geography.us_puma_ladder_gate(household, np.asarray(weights)) + ladder = decode_us_puma_ladder(ladder_bytes) + assert (100100, 1001) in set( + zip(ladder.county_overlap_puma, ladder.county_overlap_county, strict=True) + ) + assert (100100, 102) in set( + zip(ladder.cd_overlap_puma, ladder.cd_overlap_cd, strict=True) + ) + assert (100100, 1001, 102) not in set( + zip( + ladder.joint_overlap_puma, + ladder.joint_overlap_tract // 1_000_000, + ladder.joint_overlap_cd, + strict=True, + ) + ) + expected_joint = geography.us_puma_ladder_joint_support_gate(household, ladder) + assert expected_mass.passed + assert not expected_joint.passed + assert expected_joint.details["unsupported_rows"] == 1 + assert ( + result.receipt["evidence"] + == geography.GateReport((expected_mass, expected_joint)).to_manifest() + ) + + +def test_graph_gate_retains_independent_legacy_mass_failure(): + ladder_bytes, crosswalk_bytes = _ordinary_lookup_bytes() + household = _households() + weights = np.asarray([0.0, 6.2, 73.8, 20.0, 0.0]) + ladder = decode_us_puma_ladder(ladder_bytes) + assert geography.us_puma_ladder_joint_support_gate(household, ladder).passed + assert not geography.us_puma_ladder_gate(household, weights).passed + result = geography.USGeographyGateKernel().run( + _context(household, weights, ladder_bytes, crosswalk_bytes) + ) + assert result.receipt["outcome"] == "fail" + + +def test_graph_gate_declares_both_lookup_producer_dependencies(): + lookup, _, _, gate = _nodes() + assert { + (item.name, item.producer, item.artifact, item.type) + for item in gate.artifact_inputs + } == { + ("ladder", lookup.id, "ladder", geography.US_PUMA_LOOKUP_TYPE), + ("crosswalk", lookup.id, "crosswalk", geography.US_CD_CROSSWALK_TYPE), + } diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py b/packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py new file mode 100644 index 000000000..f5436b02c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py @@ -0,0 +1,876 @@ +"""Source-authored optional host acceptance. All cases are UNRUN in this task. + +One shared invented source fixture supplies actual ACS/ASEC owners and the real +compiler, store, executor and numeric kernels. No owner or runtime is replaced. +""" + +import ast +import csv +import hashlib +import io +import json +import shutil +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_child_property_income_graph import _options +from test_us_graph_atomic_survey_population import _support_payload + +from microcosm.build.us_runtime import graph_atomic_survey_financial as host +from microcosm.build.us_runtime import graph_child_property_income as child +from microcosm.build.us_runtime import graph_property_tax_leaves as tax +from microcosm.build.us_runtime import graph_survey_completion as receiving +from microcosm.build.us_runtime import graph_survey_completion_host as extension + + +def _source_arguments( + root, patch, *, incomplete_adult=False, household_roles=True, person_status=True +): + """Compose all literals and restore attachments before any host is issued.""" + import test_us_child_property_income_source_owner as children + import test_us_current_asec_demographics as demographics + import test_us_current_asec_income_routing as routing + import test_us_survey_population_preparation as preparation + from test_us_current_survey_person_status_source import ( + _source_arguments as statuses, + ) + from test_us_survey_social_security import _social_security_arguments + + from microcosm.build.us_runtime import asec_person_income_source as restoration + from microcosm.build.us_runtime import ( + current_survey_person_status_source as status_source, + ) + + original_build = preparation.build_fixture + + def supported_housing(*args, **kwargs): + # GQ reference roles are deliberately unsupported. This positive fixture + # contains only genuine housing reference-person observations; no GQ + # incumbent is erased or given a fabricated role after issuance. + for kind in ("household_members", "person_members"): + members = [] + for member in kwargs[kind]: + rows = list(csv.DictReader(io.StringIO(member.data.decode()))) + rows = [row for row in rows if "GQ" not in row["SERIALNO"]] + if kind == "person_members": + for row in rows: + if int(row["AGEP"]) >= 15: + row["INTP"] = ( + "bad" + if incomplete_adult + and row["SERIALNO"] == "2024HU0000002" + else "100" + ) + members.append(replace(member, data=preparation._csv(rows))) + kwargs[kind] = tuple(members) + return original_build(*args, **kwargs) + + patch.setattr(preparation, "build_fixture", supported_housing) + + def social(path, monkeypatch, *, zero): + assert zero is False + return _social_security_arguments(path, monkeypatch, ambiguous=False) + + def status(path, monkeypatch, *, zero): + assert zero is False + return statuses(path, monkeypatch) + + patch.setattr(demographics, "fixture", social) + patch.setattr(preparation, "fixture", status) + # Either selected ASEC household remains a valid property donor. The child + # donor itself remains the one full-source teenager outside the selection. + patch.setattr(routing, "AMOUNTS", {**routing.AMOUNTS, 107: routing.AMOUNTS[105]}) + patch.setattr( + routing, "LITERALS", {**routing.LITERALS, 107: dict(routing.LITERALS[105])} + ) + original_change = children._change_unissued_literals + + def final_literals(arguments, monkeypatch, donor, changes): + original_change(arguments, monkeypatch, donor, changes) + folder = arguments["source_dir"] / "asec" + pins, paths = [], {} + for year, member, archive, *_ in child.child.routing.coverage._MEMBER_PINS: + path = folder / member + table = pd.read_csv(path, dtype=str, keep_default_na=False) + if year == 2024: + adults = table.A_AGE.astype(int).ge(15) + table.loc[adults, "SUR_YN"] = "2" + table.loc[adults, "SUR_SC2"] = "0" + table.to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(table), + len(data), + ) + ) + paths[year] = path + for module in ( + child.child.routing.coverage, + restoration, + demographics.owner.demographic, + status_source.student, + ): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + restored = root / "completion-restored-money" + restoration.restore_asec_person_income_source( + folder / "parent.h5", + folder / "household-attachment.h5", + member_paths=paths, + output_dir=restored, + ) + shutil.copyfile( + restored / restoration.CHECKPOINT_FILENAME, + folder / "person-income-attachment.h5", + ) + + patch.setattr(children, "_change_unissued_literals", final_literals) + full, partial, donor = children.source_arguments(root, patch) + # These adapters only compose unissued sources. Release them before issuing + # a run so a later independently selected fixture cannot wrap this closure + # again and overwrite its own literals or reuse this fixture's output path. + patch.setattr(preparation, "build_fixture", original_build) + patch.setattr(children, "_change_unissued_literals", original_change) + arguments = full if incomplete_adult else partial + payload, source_ids = _support_payload() + support = root / "invented-block-support.npz" + support.write_bytes(payload) + config = host.reconstruction.AtomicSurveyReconstruction( + support_path=str(support), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + options = host._property_module().PropertyIncomeOptions( + scales=(1.0, 1.0, 1.0, 1.0), + atol=1e-10, + rtol=1e-12, + n_estimators=2, + completion_routing=True, + ) + return dict( + **arguments, + store_root=root / "store", + geography_config=config, + demographic_conditioning=True, + n_estimators=2, + property_income=options, + rebase_property_taxes=True, + person_status=person_status, + household_roles=household_roles, + child_property=_options(only_positive=True), + return_values=True, + ), donor + + +@pytest.fixture(scope="module") +def completion_host(tmp_path_factory): + with pytest.MonkeyPatch.context() as patch: + call, donor = _source_arguments( + tmp_path_factory.mktemp("completion-host"), patch + ) + cold = host.run_atomic_survey_financial(**call) + warm = host.run_atomic_survey_financial(**call, resume="require") + yield SimpleNamespace(cold=cold, warm=warm, call=call, donor=donor) + + +@pytest.mark.parametrize( + "changes,reason", + [ + ({"household_roles": 1}, "HOUSEHOLD_ROLES_FLAG"), + ({"household_roles": None}, "HOUSEHOLD_ROLES_FLAG"), + ({"household_roles": True}, "ROLES_REQUIRE_CHILD"), + ({"child_property": object()}, "CHILD_REQUIRES_PROPERTY"), + ( + {"child_property": object(), "property_income": object()}, + "CHILD_REQUIRES_TAX_REBASE", + ), + ( + { + "child_property": object(), + "property_income": object(), + "rebase_property_taxes": True, + }, + "CHILD_OPTIONS_TYPE", + ), + ], +) +def test_refused_options_precede_source_io(changes, reason): + with pytest.raises(ValueError, match=reason): + host.run_atomic_survey_financial( + None, + snapshot_root=None, + store_root=None, + fraction=1, + seed=17, + geography_config=None, + **changes, + ) + + +def _assert_all_row_lineage(run, *, household_roles, person_status): + boundary = host._run_entry(run)[2].completion_boundary + order = run.compiled.order + writers = [ + *boundary.base.prefix.compiled.order, + host.financial.ATTACH_NODE, + host._property_module().ATTACH_NODE, + ] + if person_status: + writers.append(host._person_status_module().BIND_NODE) + assert all(order.index(n) < order.index(receiving.NODE) for n in writers) + base = boundary.base.financial_population + received = boundary.observed[receiving.NODE] + completed = boundary.observed[child.VERIFY] + tax_received = boundary.observed[tax.RECEIVING_NODE] + for before, after, node_id in ( + (base, received, receiving.NODE), + (completed, tax_received, tax.RECEIVING_NODE), + ): + node = run.compiled.graph.node(node_id) + assert node.base == before.version + assert after.version == node_id + assert before.version != after.version + assert before.frame.entities == after.frame.entities + assert before.frame.schema == after.frame.schema + for entity in before.frame.entities: + assert before.frame.table(entity).shape == after.frame.table(entity).shape + pd.testing.assert_frame_equal( + before.frame.table(entity), after.frame.table(entity), check_exact=True + ) + pd.testing.assert_series_equal(before.frame.strata, after.frame.strata) + for entity in before.design_weights: + np.testing.assert_array_equal( + before.design_weights[entity], after.design_weights[entity] + ) + for entity in before.frame.weighted_entities: + np.testing.assert_array_equal( + before.frame.weights_for(entity).values, + after.frame.weights_for(entity).values, + ) + assert set(after.owners) == set(before.owners) + assert set(after.owners.values()) == {node_id} + assert after.mass_ledger[:-1] == before.mass_ledger + record = after.mass_ledger[-1] + assert (record.node_id, record.operation, record.policy) == ( + node_id, + "filter", + "conserve", + ) + assert record.before_total == record.after_total + assert record.before_by_stratum == record.after_by_stratum + assert record.before_by_partition_stratum == record.after_by_partition_stratum + assert len(run.financial_population.mass_ledger) == len(base.mass_ledger) + 2 + assert boundary.parent.version == completed.version == receiving.NODE + assert received.owners["person", child.child.TARGETS[0]] == receiving.NODE + for name in child.child.TARGETS: + assert base.owners["person", name] == host._property_module().ATTACH_NODE + assert completed.owners["person", name] == child.ATTACH + if household_roles: + roles = extension._roles() + assert ( + boundary.parent.owners["person", roles.roles.CANONICAL_COLUMN] + == roles.BIND_NODE + ) + if person_status: + status = host._person_status_module() + bind = run.compiled.graph.node(status.BIND_NODE) + for output in bind.outputs: + cell = output.entity, output.column + assert base.owners[cell] == status.BIND_NODE + assert received.owners[cell] == receiving.NODE + assert completed.owners[cell] == receiving.NODE + assert tax_received.owners[cell] == tax.RECEIVING_NODE + assert run.financial_population.version == tax.RECEIVING_NODE + for name in tax.TAX_LEAF_COLUMNS: + assert run.financial_population.owners["person", name] == tax.TAX_LEAVES_NODE + + +def _roles_disabled_pair(root, patch, *, person_status, count): + # Explicit partial ACS+ASEC selection from the same pre-issuance source + # composition: the full-source teenage donor is outside the selected sample. + call, _ = _source_arguments( + root, patch, household_roles=False, person_status=person_status + ) + cold = host.run_atomic_survey_financial(**call) + warm = host.run_atomic_survey_financial(**call, resume="require") + assert cold.compiled.graph == warm.compiled.graph + assert cold.manifest.key == warm.manifest.key + assert cold.checked_view().payload == warm.checked_view().payload + assert ( + host._run_entry(cold)[2].artifact_hashes + == host._run_entry(warm)[2].artifact_hashes + ) + roles = extension._roles() + for run in (cold, warm): + state = host._run_entry(run)[2] + boundary = state.completion_boundary + assert len(run.compiled.order) == count + assert len(boundary.base.compiled.order) == count - 10 + assert (state.person_status_boundary is not None) is person_status + assert boundary.household_roles is False + assert boundary.role_nodes == () and boundary.role_qualified is None + assert boundary.parent is boundary.receiving + assert roles.roles.CANONICAL_COLUMN not in boundary.owned_columns() + assert {roles.SOURCE_NODE, roles.BIND_NODE}.isdisjoint(run.compiled.order) + assert all( + output.column != roles.roles.CANONICAL_COLUMN + for node in run.compiled.graph.nodes + if node.id not in boundary.base.compiled.order + for output in node.outputs + ) + before = boundary.base.financial_population.frame.person + after = run.financial_population.frame.person + assert (roles.roles.CANONICAL_COLUMN in before) == ( + roles.roles.CANONICAL_COLUMN in after + ) + if roles.roles.CANONICAL_COLUMN in before: + pd.testing.assert_series_equal( + before[roles.roles.CANONICAL_COLUMN], + after[roles.roles.CANONICAL_COLUMN], + check_exact=True, + ) + document = json.loads(run.checked_view().payload) + assert document["household_roles"] is False + assert document["household_role_node_count"] == 0 + assert document["child_property_node_count"] == 6 + assert document["child_source_knownness_changed"] is False + assert json.loads(state.tax_verification)["complete"] is True + _assert_all_row_lineage(run, household_roles=False, person_status=person_status) + base_order = host._run_entry(cold)[2].completion_boundary.base.compiled.order + assert all(cold.manifest.node(n).hit for n in base_order) + assert all( + not cold.manifest.node(n).hit + for n in cold.compiled.order + if n not in base_order + ) + assert all(record.hit for record in warm.manifest.nodes.values()) + return cold, warm + + +def test_roles_disabled_status_enabled_49_cold_required_and_output_identity(tmp_path): + with pytest.MonkeyPatch.context() as patch: + cold, warm = _roles_disabled_pair(tmp_path, patch, person_status=True, count=49) + boundary = host._run_entry(warm)[2].completion_boundary + original = boundary.completed + boundary.completed = replace(original) + assert boundary.completed is not original + assert host.reconstruction._population_stamp( + boundary.completed + ) == host.reconstruction._population_stamp(original) + with pytest.raises(ValueError, match="COMPLETION_OUTPUT_IDENTITY$"): + boundary.pure() + boundary.completed = original + assert boundary.revoked and boundary.child.revoked + with pytest.raises(ValueError, match="COMPLETION_HOST_REVOKED$"): + boundary.pure() + boundary.base.checked_view() + cold.checked_view() + + +def _assert_private_support_custody(run): + """Actual executor observations and manifest views keep distinct full seals.""" + state = host._run_entry(run)[2] + boundary = state.completion_boundary + retained = dict(zip(run.compiled.order, state.node_populations, strict=True)) + for node_id in (child.DONOR, child.RECIPIENT, child.FIT, child.DRAW): + actual = boundary.observed[node_id] + expected_version = ( + child.DONOR if node_id in (child.DONOR, child.FIT) else child.RECIPIENT + ) + assert actual.version == run.compiled.versions[node_id] == expected_version + stamp = extension._population_stamp(boundary, run.compiled, node_id, actual) + assert stamp == dict(boundary.observed_stamps)[node_id] + assert retained[node_id][0] is actual and retained[node_id][1] == stamp + assert stamp[1] == child.child.physical._population_stamp(actual) + with pytest.raises(ValueError, match="^FRAME_TYPE$"): + host.reconstruction._population_stamp(actual) + for version in (child.DONOR, child.RECIPIENT): + attached = run.manifest.population(version) + assert type(attached) is extension.PopulationView + assert attached is not boundary.observed[version].frame + actual = host.Population.from_frame( + attached, version, mass_ledger=run.manifest.mass_ledger(version) + ) + assert ( + extension._population_stamp( + boundary, run.compiled, version, actual, manifest=True + ) + == dict(state.manifest_populations)[version] + ) + + +def test_roles_disabled_status_disabled_45_cold_required(tmp_path): + with pytest.MonkeyPatch.context() as patch: + cold, warm = _roles_disabled_pair( + tmp_path, patch, person_status=False, count=45 + ) + for run in (cold, warm): + _assert_private_support_custody(run) + # All original positive cold/required checks finish before either issued + # handle is deliberately poisoned. The observer is detached from the + # manifest; both actual custody paths need their own refusal. + for run, surface, reason in ( + (cold, "observed_fit", "COMPLETION_OBSERVED_CHANGED"), + (warm, "manifest_recipient", "FINANCIAL_RUN_ATTACHED_POPULATION_CHANGED"), + ): + boundary = host._run_entry(run)[2].completion_boundary + frame = ( + boundary.observed[child.FIT].frame + if surface == "observed_fit" + else run.manifest.population(child.RECIPIENT) + ) + original = frame.person["source_year"].copy(deep=True) + frame.person.loc[0, "source_year"] += 1 + try: + with pytest.raises( + ValueError, match="^CURRENT_SURVEY_PREDICTOR_" + reason + "$" + ): + run.checked_view() + finally: + frame.person["source_year"] = original + assert boundary.revoked and boundary.child.revoked + with pytest.raises( + ValueError, match="^CURRENT_SURVEY_PREDICTOR_COMPLETION_HOST_REVOKED$" + ): + run.checked_view() + boundary.base.checked_view() + + +def test_disabled_signature_and_serialized_option_parity(completion_host): + tree = ast.parse(Path(host.__file__).read_text()) + function = next( + n + for n in tree.body + if isinstance(n, ast.FunctionDef) and n.name == "run_atomic_survey_financial" + ) + defaults = dict( + zip( + (a.arg for a in function.args.kwonlyargs), + function.args.kw_defaults, + strict=True, + ) + ) + assert ast.literal_eval(defaults["household_roles"]) is False + assert ast.literal_eval(defaults["child_property"]) is None + bases = [ + host._run_entry(run)[2].completion_boundary.base + for run in (completion_host.cold, completion_host.warm) + ] + assert bases[0].compiled.graph == bases[1].compiled.graph + assert bases[0].checked_view().payload == bases[1].checked_view().payload + for base in bases: + document = json.loads(base.checked_view().payload) + assert len(base.compiled.order) == 39 + assert ( + not {"child_property", "household_roles", "child_property_node_count"} + & document.keys() + ) + assert host._run_entry(base)[2].completion_boundary is None + + +def test_exact_51_roster_order_and_actual_required_hits(completion_host): + cold, warm = completion_host.cold, completion_host.warm + boundary = host._run_entry(cold)[2].completion_boundary + expected = ( + *boundary.base.compiled.graph.nodes, + boundary.receiving_node, + *boundary.role_nodes, + *boundary.child.nodes, + *host._tax_nodes( + boundary.completed.frame, + boundary.completed.version, + host._run_entry(cold)[2].property_income, + completion=boundary.tax_edge(), + ), + ) + assert tuple(cold.compiled.graph.nodes) == expected + assert len(expected) == len(cold.compiled.order) == len(warm.compiled.order) == 51 + assert all(cold.manifest.node(n).hit for n in boundary.base.compiled.order) + assert all(not cold.manifest.node(n.id).hit for n in expected[39:]) + assert all(record.hit for record in warm.manifest.nodes.values()) + assert cold.manifest.key == warm.manifest.key + assert cold.checked_view().payload == warm.checked_view().payload + _assert_all_row_lineage(cold, household_roles=True, person_status=True) + _assert_all_row_lineage(warm, household_roles=True, person_status=True) + order = cold.compiled.order + assert ( + order.index(receiving.NODE) + < order.index(extension._roles().BIND_NODE) + < order.index(child.ATTACH) + ) + assert ( + order.index(child.VERIFY) + < order.index(tax.RECEIVING_NODE) + < order.index(tax.GATE_NODE) + ) + assert cold.compiled.graph.node(child.ATTACH).population == receiving.NODE + assert cold.compiled.graph.node(tax.RECEIVING_NODE).base == receiving.NODE + for name in (tax.RECEIVING_NODE, tax.TAX_LEAVES_NODE, tax.GATE_NODE): + assert boundary.tax_edge() in cold.compiled.graph.node(name).artifact_inputs + assert set(boundary.base.kernels.refs()).isdisjoint( + {node.kernel for node in expected[39:]} + ) + boundary.base.checked_view() + + +def test_actual_donor_outside_selection_and_paired_child_draws(completion_host): + run = completion_host.cold + boundary = host._run_entry(run)[2].completion_boundary + donor = boundary.observed[child.DONOR].frame.person + assert len(donor) == 1 + selected = boundary.child.qualified.recipients + selected_people = boundary.base.prefix.preparation.checked_view().frame.person + asec = selected_people[child.provenance.support_channel_column("person")].eq("asec") + assert not selected_people.loc[asec, "A_AGE"].between(15, 17).any() + assert donor[child.WEIGHT].iloc[0] > 0 + output = boundary.completed.frame.person + original_id = child.provenance.support_source_id_column("person") + eligible = selected.index[selected.eligible] + assert len(eligible) > 0 + for identity in eligible: + pair = output.loc[output[original_id].eq(identity)] + assert len(pair) == 2 and pair[child.IMPUTED].all() + np.testing.assert_array_equal( + pair[list(child.child.TARGETS)].iloc[0], + pair[list(child.child.TARGETS)].iloc[1], + ) + assert pair[child.STATUS].eq("imputed_positive").all() + # Full nonrecipient and source-known O/D preserve storage and null masks. + before = boundary.parent.frame.person + unchanged = ~output[child.IMPUTED] + for name in child.child.TARGETS: + pd.testing.assert_series_equal( + before.loc[unchanged, name], output.loc[unchanged, name], check_exact=True + ) + for name in before: + if name not in child.child.TARGETS: + pd.testing.assert_series_equal(before[name], output[name], check_exact=True) + + +def test_full_weights_geography_status_roles_and_complete_gate(completion_host): + run = completion_host.warm + state = host._run_entry(run)[2] + boundary = state.completion_boundary + before, after = boundary.parent, run.financial_population + for entity in before.frame.entities: + unchanged = [ + c + for c in before.frame.table(entity) + if c not in (*child.child.TARGETS, *tax.TAX_LEAF_COLUMNS) + ] + pd.testing.assert_frame_equal( + before.frame.table(entity)[unchanged], + after.frame.table(entity)[unchanged], + check_exact=True, + ) + pd.testing.assert_series_equal(before.frame.strata, after.frame.strata) + assert before.frame.schema == after.frame.schema + for entity in before.design_weights: + np.testing.assert_array_equal( + before.design_weights[entity], after.design_weights[entity] + ) + np.testing.assert_array_equal( + before.frame.weights_for(entity).values, + after.frame.weights_for(entity).values, + ) + assert json.loads(state.tax_verification)["complete"] is True + host.require_complete_property_taxes(run) + assert host.financial_output_node(run) == tax.GATE_NODE + assert ( + json.loads(run.checked_view().payload)["property_completion_routing_scope"] + == "pre_child_source_gaps" + ) + document = json.loads(run.checked_view().payload) + verification = boundary.payloads[child.VERIFY, "verification"] + assert ( + document["child_verification_sha256"] + == hashlib.sha256(verification).hexdigest() + ) + assert ( + document["child_source_knownness_changed"] + is json.loads(verification)["source_knownness_changed"] + ) + assert ( + child.VERIFY, + "verification", + document["child_verification_sha256"], + ) in state.artifact_hashes + + +def test_detached_parent_refuses_without_owner_substitution(completion_host): + run = completion_host.cold + base = host._run_entry(run)[2].completion_boundary.base + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN"): + extension.extend( + replace(base), + options=completion_host.call["child_property"], + household_roles=True, + resume="require", + ) + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN"): + replace(run).checked_view() + base.checked_view() + + +def test_actual_tax_rebased_parent_refuses(completion_host): + run = completion_host.cold + with pytest.raises(ValueError, match="COMPLETION_ACTUAL_PROPERTY_PARENT$"): + extension.extend( + run, + options=completion_host.call["child_property"], + household_roles=True, + resume="require", + ) + run.checked_view() + + +@pytest.mark.parametrize("defect", ["compensating_child_tax", "receiving", "roles"]) +def test_wrong_fragment_roster_refuses_before_compile(completion_host, defect): + run = completion_host.cold + boundary = host._run_entry(run)[2].completion_boundary + tax_nodes = host._tax_nodes( + boundary.completed.frame, + boundary.completed.version, + host._run_entry(run)[2].property_income, + completion=boundary.tax_edge(), + ) + original = boundary.receiving_node, boundary.role_nodes, boundary.child.nodes + assert ( + extension._completion_nodes(boundary, tax_nodes) + == tuple(run.compiled.graph.nodes)[39:] + ) + if defect == "compensating_child_tax": + # Preserve the union count while moving a real tax node into child7. + boundary.child.nodes = (*boundary.child.nodes, tax_nodes[0]) + tax_nodes = tax_nodes[1:] + elif defect == "receiving": + boundary.receiving_node = replace(boundary.receiving_node, id="wrong.receiving") + else: + boundary.role_nodes = tuple(reversed(boundary.role_nodes)) + try: + with pytest.raises(ValueError, match="COMPLETION_FRAGMENT_ROSTER$"): + extension._completion_nodes(boundary, tax_nodes) + finally: + boundary.receiving_node, boundary.role_nodes, boundary.child.nodes = original + boundary.pure() + + +@pytest.mark.parametrize( + "defect,reason", + [ + ("base_miss", "COMPLETION_BASE_MUST_HIT"), + ("missing_observation", "COMPLETION_OBSERVER_ROSTER"), + ], +) +def test_actual_union_observation_and_base_cache_refusals( + completion_host, defect, reason +): + run = completion_host.warm + boundary = host._run_entry(run)[2].completion_boundary + extension._check_observed_union( + boundary.base, run.compiled, run.manifest, boundary.observed + ) + manifest, observed = run.manifest, dict(boundary.observed) + if defect == "base_miss": + node_id = boundary.base.compiled.order[-1] + manifest = replace( + manifest, + nodes={ + **manifest.nodes, + node_id: replace(manifest.node(node_id), hit=False), + }, + ) + # Hits are deliberately outside manifest content identity; the host + # must enforce this separate staged-execution invariant itself. + assert manifest.key == run.manifest.key + else: + observed.pop(child.VERIFY) + with pytest.raises(ValueError, match=reason + "$"): + extension._check_observed_union(boundary.base, run.compiled, manifest, observed) + boundary.pure() + + +@pytest.mark.parametrize( + "defect", ["source_key", "node_key", "implementation", "artifact"] +) +def test_actual_union_final_io_identity_refusals_without_execution( + completion_host, defect +): + run = completion_host.warm + state = host._run_entry(run)[2] + source_keys, keys, implementations = ( + dict(state.source_keys), + dict(state.keys), + dict(state.implementations), + ) + loaded = host._artifacts( + run.manifest, run.compiled, run.store, run.kernels, keys, implementations + ) + expected = dict( + source_keys=source_keys, + keys=keys, + implementations=implementations, + loaded=loaded, + ) + assert extension._checked_final_io(run, **expected) == (source_keys, loaded) + # Change a retained pre-I/O identity; the maintained boundary re-reads the + # authentic source/store/implementations. No runner or issuer is replaced. + selected = expected[ + { + "source_key": "source_keys", + "node_key": "keys", + "implementation": "implementations", + "artifact": "loaded", + }[defect] + ] + key = next(iter(selected)) + selected[key] = ( + b"changed retained payload" + if defect == "artifact" + else "changed retained identity" + ) + with pytest.raises(ValueError, match="COMPLETION_FINAL_IO_CHANGED$"): + extension._checked_final_io(run, **expected) + state.completion_boundary.pure() + + +@pytest.mark.parametrize("field", ["scenario_id", "stream", "transport"]) +def test_explicit_scientific_identity_changes_declarations(completion_host, field): + boundary = host._run_entry(completion_host.cold)[2].completion_boundary + options = boundary.options + replacement = { + "scenario_id": "another-explicit-scenario", + "stream": (*options.stream[:-1], options.stream[-1] + 1), + "transport": replace(options.transport, receipt_factor=0.0), + }[field] + changed = replace(options, **{field: replacement}) + nodes = child.child_property_nodes( + boundary.child.qualified, + boundary.child.origins, + boundary.parent, + options=changed, + host_edges=boundary.host_edges, + host_pins=boundary.host_pins, + ) + assert tuple(n.normative() for n in nodes) != boundary.child.declaration + + +def test_incomplete_adult_refuses_actual_puf_qualification_before_fitting(tmp_path): + from microcosm.build.us_runtime import graph_puf55_survey_recipients as recipients + + with pytest.MonkeyPatch.context() as patch: + call, _ = _source_arguments(tmp_path, patch, incomplete_adult=True) + run = host.run_atomic_survey_financial(**call) + state = host._run_entry(run)[2] + completed = state.completion_boundary.completed.frame.person + assert ( + completed.loc[~completed[child.IMPUTED], list(child.child.TARGETS)] + .isna() + .any() + .any() + ) + assert json.loads(state.tax_verification)["complete"] is False + with pytest.raises(ValueError, match="PROPERTY_TAX_INPUTS_INCOMPLETE"): + recipients.values.qualify_puf55_survey_recipients(run) + assert len(run.compiled.order) == 51 # No PUF fit graph was constructed. + + +@pytest.mark.parametrize( + "defect,reason", + [ + ("parent", "RETAINED_VALUES_CHANGED"), + ("artifact", "RECONSTRUCTED_ARTIFACT_CHANGED"), + ("identity", "MATERIALIZED_VERIFICATION"), + ], +) +def test_real_child_verifier_refuses_changed_custody(completion_host, defect, reason): + run = completion_host.cold + original = host._run_entry(run)[2].completion_boundary + # Reuse the actual executed support/output and the same issued source/base; + # a new private boundary isolates permanent revocation. Construction does + # not fit child income, but verification reconstructs and can refit it. + boundary = extension._CompletionHost(original.base, original.options, True) + keys = dict(host._run_entry(run)[2].keys) + loaded = host._artifacts( + run.manifest, + run.compiled, + run.store, + run.kernels, + keys, + dict(host._run_entry(run)[2].implementations), + ) + artifacts = extension._inputs( + boundary.child.nodes[-1], run.compiled, run.kernels, keys, loaded + ) + verification = extension._value( + boundary.tax_edge(), run.compiled, run.kernels, keys, loaded + ) + if defect == "parent": + # FILTER -> Frame.select -> Frame.__init__ copies each table; role + # patching also uses _copied_tables(deep=True). Check isolation before + # asking the verifier to refuse, without substituting any owner. + base_stamp = host.reconstruction._population_stamp( + original.base.financial_population + ) + boundary.parent.frame.person.loc[0, child.child.TARGETS[0]] = 314.0 + assert ( + host.reconstruction._population_stamp(original.base.financial_population) + == base_stamp + ) + elif defect == "artifact": + artifacts["draws"] = replace(artifacts["draws"], payload=b"{}") + else: + verification = replace(verification, producer_key="f" * 64) + with pytest.raises(ValueError, match="^CHILD_PROPERTY_" + reason + "$"): + boundary.child.verify_materialized( + original.completed, + artifacts, + support_populations={ + n: original.observed[n] for n in (child.DONOR, child.RECIPIENT) + }, + verification=verification, + ) + original.base.checked_view() + assert boundary.child.revoked + with pytest.raises(ValueError, match="REVOKED"): + boundary.child._pure() + + +@pytest.mark.parametrize("when", ["first", "final"]) +def test_final_source_io_mutation_permanently_revokes_host( + completion_host, monkeypatch, when +): + # Last controls: each poisons one independently issued handle permanently. + run = completion_host.cold if when == "first" else completion_host.warm + boundary = host._run_entry(run)[2].completion_boundary + read = Path.read_bytes + count = [] + target = 1 if when == "first" else 2 + before = run.financial_population.frame.person[child.STATUS].copy() + + def mutate(path): + data = read(path) + if path == Path(extension.__file__): + count.append(True) + if len(count) == target: + run.financial_population.frame.person.loc[0, child.STATUS] = "changed" + return data + + with monkeypatch.context() as patch: + patch.setattr(Path, "read_bytes", mutate) + with pytest.raises(ValueError): + run.checked_view() + assert len(count) >= target + run.financial_population.frame.person[child.STATUS] = before + assert boundary.revoked and boundary.child.revoked + with pytest.raises(ValueError, match="REVOKED"): + run.checked_view() diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_person_status.py b/packages/microcosm-build/tests/test_us_graph_atomic_person_status.py new file mode 100644 index 000000000..ebb072156 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_person_status.py @@ -0,0 +1,343 @@ +"""Optional status on the actual common host, over finite invented sources.""" + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_current_survey_person_status_source import _source_arguments +from test_us_graph_atomic_survey_population import _support_payload + +from microcosm.build.us_runtime import graph_atomic_survey_financial as runner +from microcosm.build.us_runtime import graph_current_survey_person_status as graph + + +@pytest.fixture(scope="module") +def status_host(tmp_path_factory): + root = tmp_path_factory.mktemp("atomic-person-status") + with pytest.MonkeyPatch.context() as patch: + arguments = _source_arguments(root, patch) + payload, source_ids = _support_payload() + support = root / "invented-block-support.npz" + support.write_bytes(payload) + config = runner.reconstruction.AtomicSurveyReconstruction( + support_path=str(support), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + call = { + **arguments, + "store_root": root / "store", + "geography_config": config, + "demographic_conditioning": True, + "n_estimators": 2, + "return_values": True, + } + base = runner.run_atomic_survey_financial(**call, person_status=False) + cold = runner.run_atomic_survey_financial(**call, person_status=True) + warm = runner.run_atomic_survey_financial( + **call, person_status=True, resume="require" + ) + yield SimpleNamespace(base=base, cold=cold, warm=warm, call=call) + for run in (base, cold, warm): + run.checked_view() + + +def test_optional_23_node_host_preserves_default_19_and_nine_node_prefix(status_host): + case = status_host + assert len(case.base.compiled.order) == 19 + assert len(case.cold.compiled.order) == len(case.warm.compiled.order) == 23 + assert case.cold.manifest.key == case.warm.manifest.key + assert all(record.hit for record in case.warm.manifest.nodes.values()) + assert not set(graph.BIND_COLUMNS) & set( + case.base.financial_population.frame.person + ) + assert runner._run_entry(case.base)[2].person_status_boundary is None + for run in (case.cold, case.warm): + boundary = runner._run_entry(run)[2].person_status_boundary + runner.atomic.same_replayed_population( + case.base.financial_population, + boundary.complement(run.financial_population), + ) + for name in ( + "allocated_population", + "observed_population", + "expanded_population", + "geography_population", + "clone_population", + ): + runner.atomic.same_replayed_population( + getattr(case.base.prefix, name), getattr(run.prefix, name) + ) + assert len(run.prefix.compiled.order) == 9 + for node in case.base.prefix.compiled.graph.nodes: + assert run.compiled.graph.node(node.id) == node + assert graph.BIND_NODE in run.compiled.predecessors[runner.financial.DONOR_NODE] + assert ( + graph.BIND_NODE + in run.compiled.order[ + : run.compiled.order.index(runner.financial.ATTACH_NODE) + ] + ) + doc = json.loads(run.checked_view().payload) + assert doc["person_status"]["node_count"] == 4 + assert doc["person_status"]["source_artifact_required"] + assert doc["person_status"]["descriptive_only"] and not doc["release_eligible"] + assert set(graph.BIND_COLUMNS) <= set(doc["owned_columns"]) + + +@pytest.mark.parametrize("column", ["survey_vision_difficulty", "employment_income"]) +def test_common_host_refuses_status_only_and_financial_only_drift(status_host, column): + run = status_host.cold + people = run.financial_population.frame.person + # Check the original financial complement remains identical for a status-only + # mutation; acceptance still fails because the full reconstruction includes it. + if column == "employment_income": + column = runner.values.OUTPUTS[0] + saved = people[column].copy(deep=True) + complement_columns = [c for c in people if c not in graph.BIND_COLUMNS] + complement = people.loc[:, complement_columns].copy(deep=True) + try: + people.loc[0, column] = ( + True if column in graph.BIND_COLUMNS else float(saved.iloc[0]) + 11 + ) + if column in graph.BIND_COLUMNS: + pd.testing.assert_frame_equal( + people.loc[:, complement_columns], complement, check_exact=True + ) + with pytest.raises(ValueError): + run.checked_view() + finally: + people[column] = saved + run.checked_view() + + +@pytest.mark.parametrize("defect", ["copied_run", "status_owner", "qualifier_receipt"]) +def test_common_host_retains_live_issued_owner_and_writers(status_host, defect): + run = status_host.cold + boundary = runner._run_entry(run)[2].person_status_boundary + if defect == "copied_run": + with pytest.raises(ValueError): + replace(run).checked_view() + return + if defect == "status_owner": + owners = dict(run.financial_population.owners) + owners["person", graph.BIND_COLUMNS[0]] = "unissued" + detached = replace(run.financial_population, owners=owners) + with pytest.raises(ValueError, match="PERSON_STATUS_WRITER"): + boundary.complement(detached) + return + saved = boundary.qualified.receipt + try: + object.__setattr__(boundary.qualified, "receipt", b"{}") + with pytest.raises(ValueError): + run.checked_view() + finally: + object.__setattr__(boundary.qualified, "receipt", saved) + run.checked_view() + + +def test_final_status_source_io_cannot_mutate_only_financial_complement( + status_host, monkeypatch +): + from pathlib import Path + + run = status_host.cold + boundary = runner._run_entry(run)[2].person_status_boundary + column = runner.values.OUTPUTS[0] + saved = run.financial_population.frame.person[column].copy(deep=True) + read = Path.read_bytes + changed = [] + + def mutate(path): + payload = read(path) + if not changed: + changed.append(True) + run.financial_population.frame.person.loc[0, column] += 1 + return payload + + try: + with monkeypatch.context() as patch: + patch.setattr(Path, "read_bytes", mutate) + boundary.requalify() + assert changed + with pytest.raises(ValueError): + runner._pure_run(run, runner._run_entry(run)) + finally: + run.financial_population.frame.person[column] = saved + run.checked_view() + + +@pytest.mark.parametrize("defect", ["declaration", "binding", "artifact_roster"]) +def test_ordering_adapter_refuses_changes_before_original_donor(status_host, defect): + from microcosm.graph import ArtifactValue + from microcosm.graph.artifact_edges import numeric_scope + from microcosm.graph.executor import _project_context + + run = status_host.cold + node = run.compiled.graph.node(runner.financial.DONOR_NODE) + kernel = run.kernels.get(node.kernel) + graph_edge = graph.binding_edge() + assert node == replace( + kernel.original_node, + artifact_inputs=(*kernel.original_node.artifact_inputs, graph_edge), + ) + artifacts = {} + for edge in node.artifact_inputs: + record = run.manifest.node(edge.producer) + artifacts[edge.name] = ArtifactValue( + run.store.load_bytes(record.opaque_artifacts[edge.artifact]), + edge.type, + record.opaque_artifacts[edge.artifact], + record.key, + numeric_scope( + run.kernels.get( + run.compiled.graph.node(edge.producer).kernel + ).capabilities + ), + ) + incoming = runner.Population.from_frame( + run.prefix.preparation.checked_view().frame, runner.survey.CREATE_NODE + ) + context = _project_context( + node, + incoming, + key=run.manifest.node(node.id).key, + sources={}, + tolerances={}, + numerics={}, + artifacts=artifacts, + ) + if defect == "declaration": + context = replace( + context, node=replace(node, params={**node.params, "changed": True}) + ) + elif defect == "binding": + artifacts[graph_edge.name] = replace( + artifacts[graph_edge.name], payload=b"changed" + ) + context = replace(context, artifacts=artifacts) + else: + artifacts.pop(graph_edge.name) + context = replace(context, artifacts=artifacts) + with pytest.raises(ValueError, match="PERSON_STATUS_DONOR_"): + kernel.run(context) + + +@pytest.fixture(scope="module", params=[False, True], ids=["property39", "tax42"]) +def extended_status_host(tmp_path_factory, request): + import test_us_graph_atomic_property_financial as fixtures + + with pytest.MonkeyPatch.context() as patch: + + def status_demographics(path, monkeypatch, *, unknown=False, zero=True): + assert not unknown and zero is False + return _source_arguments(path, monkeypatch) + + # Compose source constructors before any preparation is issued. The + # existing property helper retains status columns while updating money. + patch.setattr(fixtures, "_demographic_arguments", status_demographics) + generator = fixtures.property_host_source.__wrapped__(tmp_path_factory) + case = next(generator) + try: + # The student pins are patched on a context nested inside the + # generator's lifetime. Recorded on the outer context instead, the + # patch would be undone after ``generator.close()`` and restore + # the generator-era pins over the value the module's earlier + # fixture installed, so that fixture's retained student receipts + # would fail ``STUDENT_CONTENT_CHANGED`` at its own teardown. + with pytest.MonkeyPatch.context() as student_patch: + pins = runner._property_module().sources.routing.coverage._MEMBER_PINS + student_patch.setattr(graph.source.student, "_MEMBER_PINS", pins) + options = replace(case.options, completion_routing=True) + call = { + **case.call, + "property_income": options, + "rebase_property_taxes": request.param, + "person_status": True, + } + cold = runner.run_atomic_survey_financial(**call) + warm = runner.run_atomic_survey_financial(**call, resume="require") + yield SimpleNamespace(cold=cold, warm=warm, tax=request.param) + cold.checked_view() + warm.checked_view() + finally: + generator.close() + + +def test_optional_status_preserved_through_property_and_tax_hosts(extended_status_host): + case = extended_status_host + assert ( + len(case.cold.compiled.order) + == len(case.warm.compiled.order) + == (42 if case.tax else 39) + ) + assert case.cold.manifest.key == case.warm.manifest.key + assert all(record.hit for record in case.warm.manifest.nodes.values()) + for run in (case.cold, case.warm): + state = runner._run_entry(run)[2] + boundary = state.person_status_boundary + columns = list(graph.BIND_COLUMNS) + pd.testing.assert_frame_equal( + state.legacy_financial_population.frame.person[columns], + run.financial_population.frame.person[columns], + check_exact=True, + ) + status_parent = ( + state.property_population if case.tax else run.financial_population + ) + assert boundary.complement(status_parent).version == status_parent.version + assert all( + status_parent.owners["person", c] == graph.BIND_NODE for c in columns + ) + current_owner = ( + runner._tax_module().RECEIVING_NODE if case.tax else graph.BIND_NODE + ) + assert all( + run.financial_population.owners["person", c] == current_owner + for c in columns + ) + if case.tax: + assert run.financial_population.version == current_owner + assert status_parent is not run.financial_population + # A post-FILTER population must not masquerade as the original + # writer's version merely because all status values were carried. + with pytest.raises(ValueError, match="PERSON_STATUS_WRITER"): + boundary.complement(run.financial_population) + run.checked_view() + # The pure check runs both during issuance and after final source I/O. + # A genuine FILTER owner is accepted only with the retained complete + # population seal; neither stale ownership nor status-only drift passes. + populations = ( + (status_parent, run.financial_population) + if case.tax + else (run.financial_population,) + ) + column = "survey_status_original_age" + for population in populations: + people = population.frame.person + saved_values = people[column].copy(deep=True) + saved_owners = population.owners + for defect in ("value", "owner"): + try: + if defect == "value": + people.loc[people.index[0], column] = ( + int(saved_values.iloc[0]) + 1 + ) + else: + changed = dict(saved_owners) + changed["person", column] = ( + graph.BIND_NODE + if saved_owners["person", column] != graph.BIND_NODE + else "unissued_status_writer" + ) + object.__setattr__(population, "owners", changed) + with pytest.raises(ValueError): + runner._pure_run(run, runner._run_entry(run)) + finally: + people[column] = saved_values.copy(deep=True) + object.__setattr__(population, "owners", saved_owners) + runner._pure_run(run, runner._run_entry(run)) diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py b/packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py new file mode 100644 index 000000000..fc574a459 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py @@ -0,0 +1,319 @@ +"""Issued 19/35-node financial graphs over invented source files and blocks.""" + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments +from test_us_current_property_income_sources import source_arguments +from test_us_graph_atomic_survey_population import _support_payload +from test_us_survey_social_security import _social_security_arguments + +from microcosm.build.us_runtime import graph_atomic_survey_financial as runner +from microcosm.build.us_runtime import graph_current_survey_property as property_graph +from microcosm.build.us_runtime import graph_puf55_survey_recipients as puf_recipient +from microcosm.graph import compile_graph + + +@pytest.fixture(scope="module") +def property_host_source(tmp_path_factory): + import test_us_current_asec_demographics as demographic_fixture_module + import test_us_current_asec_income_routing as routing_fixture + import test_us_survey_population_preparation as survey_fixture_module + + from microcosm.build.us_runtime import current_asec_demographics as demographics + + root = tmp_path_factory.mktemp("atomic-property-financial") + with pytest.MonkeyPatch.context() as patch: + # Compose complete invented member bytes before the real source issuer + # runs. Only fixture constructors and their registry pins are replaced. + # The nesting order is original source -> Social Security -> demographics + # -> routing -> property. Each extension preserves prior columns and + # rebuilds the retained attachment; preparation is issued only afterward. + def social_security_fixture(path, monkeypatch, *, zero): + assert zero is False + return _social_security_arguments(path, monkeypatch, ambiguous=False) + + def demographic_fixture(path, monkeypatch): + return _demographic_arguments(path, monkeypatch, unknown=False, zero=False) + + patch.setattr(demographic_fixture_module, "fixture", social_security_fixture) + patch.setattr(routing_fixture, "fixture", demographic_fixture) + original_person = survey_fixture_module._person + + def person(*args, **kwargs): + row = original_person(*args, **kwargs) + # The property fixture makes this person age 14. Its other income + # questions must also be blank before the source archive is built. + if row["SERIALNO"] == "2024GQ0000001": + row.update(SSP="", SSIP="") + return row + + patch.setattr(survey_fixture_module, "_person", person) + arguments = source_arguments(root, patch) + patch.setattr( + demographics.demographic, + "_MEMBER_PINS", + property_graph.sources.routing.coverage._MEMBER_PINS, + ) + payload, source_ids = _support_payload() + support = root / "invented-block-support.npz" + support.write_bytes(payload) + config = runner.reconstruction.AtomicSurveyReconstruction( + support_path=str(support), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + options = property_graph.PropertyIncomeOptions( + scales=(1.0, 1.0, 1.0, 1.0), atol=1e-10, rtol=1e-12, n_estimators=2 + ) + call = { + **arguments, + "geography_config": config, + "demographic_conditioning": True, + "n_estimators": 2, + "return_values": True, + } + yield SimpleNamespace(root=root, call=call, options=options) + + +@pytest.fixture(scope="module") +def property_host_base(property_host_source): + case = property_host_source + run = runner.run_atomic_survey_financial( + **case.call, property_income=None, resume="auto" + ) + yield run + runner.values.source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture(scope="module") +def property_host_cold(property_host_source): + case = property_host_source + run = runner.run_atomic_survey_financial( + **case.call, property_income=case.options, resume="auto" + ) + yield run + runner.values.source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture(scope="module") +def property_host_warm(property_host_source, property_host_cold): + case = property_host_source + run = runner.run_atomic_survey_financial( + **case.call, property_income=case.options, resume="require" + ) + yield run + runner.values.source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture(scope="module") +def actual_property_host(request, property_host_source): + # Explicit ordering demonstrates reuse of the unchanged 19-node prefix. + # A targeted PUF test requests only the cold host and need not rerun these + # separately accepted baseline/replay checks. + base = request.getfixturevalue("property_host_base") + cold = request.getfixturevalue("property_host_cold") + warm = request.getfixturevalue("property_host_warm") + (property_host_source.root / "host-acceptance.json").write_text( + json.dumps( + { + "scope": "invented source-issued financial host; no native or engine", + "base_nodes": len(base.compiled.order), + "extended_nodes": len(cold.compiled.order), + "cold_hits": sum(n.hit for n in cold.manifest.nodes.values()), + "required_hits": sum(n.hit for n in warm.manifest.nodes.values()), + "persons": cold.financial_population.frame.n("person"), + "households": cold.financial_population.frame.n("household"), + "owned_property_columns": [ + o.column for o in property_graph.owned_columns() + ], + "tax_split_rebased": False, + }, + indent=2, + ) + + "\n" + ) + return SimpleNamespace( + base=base, cold=cold, warm=warm, options=property_host_source.options + ) + + +def test_default_graph_and_payload_remain_nineteen_nodes(actual_property_host): + run = actual_property_host.base + document = runner.codec.decode_json(run.checked_view().payload) + assert len(run.compiled.order) == 19 + assert runner.financial_output_node(run) == runner.financial.ATTACH_NODE + assert document["owned_columns"] == list(runner.values.OUTPUTS) + assert ( + not {"property_income", "property_node_count", "tax_split_rebased"} + & document.keys() + ) + assert not any( + n.startswith(property_graph.PREFIX + ".") for n in run.compiled.order + ) + + +def test_actual_extended_host_cold_required_and_checked_issuance(actual_property_host): + case = actual_property_host + assert len(case.cold.compiled.order) == len(case.warm.compiled.order) == 35 + assert case.cold.manifest.key == case.warm.manifest.key + assert sum(n.hit for n in case.cold.manifest.nodes.values()) == 19 + assert all(n.hit for n in case.warm.manifest.nodes.values()) + runner.atomic.same_replayed_population( + case.cold.financial_population, case.warm.financial_population + ) + checked = [run.checked_view() for run in (case.cold, case.warm)] + assert checked[0].payload == checked[1].payload + document = runner.codec.decode_json(checked[0].payload) + assert document["property_income"] == runner.codec.decode_json( + case.options.to_bytes() + ) + assert document["property_node_count"] == 16 + assert document["tax_split_rebased"] is False + assert document["capital_gains_conditioning"] == property_graph.CAP_LIMITATION + assert document["release_eligible"] is False + assert runner.financial_output_node(case.cold) == property_graph.ATTACH_NODE + + +def test_extended_host_retains_complete_legacy_population(actual_property_host): + base, extended = actual_property_host.base, actual_property_host.cold + state = runner._run_entry(extended)[2] + runner.atomic.same_replayed_population( + base.financial_population, state.legacy_financial_population + ) + before, after = base.financial_population, extended.financial_population + for entity in before.frame.entities: + pd.testing.assert_frame_equal( + after.frame.table(entity)[before.frame.table(entity).columns], + before.frame.table(entity), + check_exact=True, + ) + assert after.version == before.version + assert after.frame.schema == before.frame.schema + assert after.frame.links == before.frame.links + assert after.frame.metadata == before.frame.metadata + assert after.mass_ledger == before.mass_ledger + assert after.frame.mass_log == before.frame.mass_log + pd.testing.assert_series_equal(after.frame.strata, before.frame.strata) + for entity in before.design_weights: + np.testing.assert_array_equal( + after.design_weights[entity], before.design_weights[entity] + ) + np.testing.assert_array_equal( + after.frame.weights_for(entity).values, + before.frame.weights_for(entity).values, + ) + assert dict(after.owners) == { + **before.owners, + **{ + ("person", o.column): property_graph.ATTACH_NODE + for o in property_graph.owned_columns() + }, + } + + +def test_property_values_and_unknowns_follow_original_clone_identity( + actual_property_host, +): + run = actual_property_host.cold + people = run.financial_population.frame.person + source_id = runner.values.provenance.support_source_id_column("person") + columns = [o.column for o in property_graph.owned_columns()] + for _, pair in people.groupby(source_id, sort=False): + assert len(pair) == 2 + pd.testing.assert_series_equal( + pair.iloc[0][columns], pair.iloc[1][columns], check_names=False + ) + assert (~people.property_anchor_known).any() + assert ( + people.loc[ + ~people.property_anchor_known, property_graph.PROPERTY_REPORTED_TOTAL + ] + .isna() + .all() + ) + known = people.property_components_known + assert known.any() and (~known).any() + channel = runner.values.provenance.support_channel_column("person") + modeled = known & people[channel].astype(str).eq("acs") + assert modeled.any() + values = people.loc[modeled, list(property_graph.PROPERTY_COMPONENTS)] + np.testing.assert_allclose( + values.sum(axis=1), + people.loc[modeled, property_graph.PROPERTY_REPORTED_TOTAL], + atol=1e-10, + rtol=1e-12, + ) + assert (values.iloc[:, :3] >= 0).all().all() + assert (people.property_reported_total < 0).any() + + +@pytest.mark.parametrize("defect", ("options", "legacy", "final")) +def test_retained_host_lifetime_rejects_mutation(actual_property_host, defect): + run = actual_property_host.cold + state = runner._run_entry(run)[2] + if defect == "options": + original = state.property_income.atol + object.__setattr__(state.property_income, "atol", original + 1.0) + try: + with pytest.raises( + ValueError, match="PROPERTY_OPTIONS_OR_LEGACY_POPULATION_CHANGED" + ): + run.checked_view() + finally: + object.__setattr__(state.property_income, "atol", original) + else: + population = ( + state.legacy_financial_population + if defect == "legacy" + else run.financial_population + ) + table = population.frame.person + column = ( + runner.values.OUTPUTS[0] + if defect == "legacy" + else property_graph.PROPERTY_REPORTED_TOTAL + ) + index = table.index[table[column].notna()][0] + original = table.loc[index, column] + table.loc[index, column] = original + 1.0 + try: + with pytest.raises( + ValueError, + match=( + "PROPERTY_OPTIONS_OR_LEGACY_POPULATION_CHANGED" + "|FINANCIAL_NODE_POPULATION_CHANGED" + "|FINANCIAL_RUN_POPULATION_CHANGED" + "|FINANCIAL_RUN_ATTACHED_POPULATION_CHANGED" + ), + ): + run.checked_view() + finally: + table.loc[index, column] = original + runner._pure_run(run, runner._run_entry(run)) + + +def test_detached_run_copy_has_no_issuer_authority(actual_property_host): + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN"): + replace(actual_property_host.cold).checked_view() + + +def test_puf_recipient_reads_complete_final_property_writer(property_host_cold): + run = property_host_cold + qualified = puf_recipient.values.qualify_puf55_survey_recipients(run) + nodes = puf_recipient.puf55_survey_recipient_nodes(qualified) + compiled = compile_graph( + replace(run.compiled.graph, nodes=(*run.compiled.graph.nodes, *nodes)) + ) + assert len(compiled.order) == 37 + for node in nodes: + assert property_graph.ATTACH_NODE in compiled.predecessors[node.id] + inputs = next(s.columns for s in node.inputs if s.entity == "person") + assert {o.column for o in property_graph.owned_columns()}.issubset(inputs) + assert qualified.financial_run is run diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py b/packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py new file mode 100644 index 000000000..1f9d7330b --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py @@ -0,0 +1,254 @@ +"""Optional 38-node financial host over actual invented source owners.""" + +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_atomic_property_financial import property_host_source # noqa: F401 + +from microcosm.build.us_runtime import graph_atomic_survey_financial as runner +from microcosm.build.us_runtime import graph_current_survey_property as property_graph +from microcosm.build.us_runtime import graph_property_tax_leaves as tax +from microcosm.build.us_runtime import graph_puf55_survey_recipients as recipients +from microcosm.graph import describe, explain_html +from microcosm.graph.store import StoreCorrupt + + +@pytest.fixture(scope="module") +def tax_host_cold(request): + case = request.getfixturevalue("property_host_source") + return runner.run_atomic_survey_financial( + **case.call, + property_income=replace(case.options, completion_routing=True), + rebase_property_taxes=True, + resume="auto", + ) + + +@pytest.fixture(scope="module") +def tax_host_warm(request, tax_host_cold): + case = request.getfixturevalue("property_host_source") + return runner.run_atomic_survey_financial( + **case.call, + property_income=replace(case.options, completion_routing=True), + rebase_property_taxes=True, + resume="require", + ) + + +def test_opt_in_cold_required_issued_output(tax_host_cold, tax_host_warm): + cold, warm = tax_host_cold, tax_host_warm + assert len(cold.compiled.order) == len(warm.compiled.order) == 38 + assert cold.manifest.key == warm.manifest.key + assert all(record.hit for record in warm.manifest.nodes.values()) + runner.atomic.same_replayed_population( + cold.financial_population, warm.financial_population + ) + checked = cold.checked_view() + assert checked.payload == warm.checked_view().payload + document = runner.codec.decode_json(checked.payload) + assert document["property_income"]["completion_routing"] is True + assert document["tax_split_rebased"] is True + assert document["tax_rebase_node_count"] == 3 + assert document["tax_leaf_complete"] is False + assert document["release_eligible"] is False + assert runner.financial_output_node(cold) == tax.GATE_NODE + assert cold.financial_population.version == tax.RECEIVING_NODE + + +def test_opt_in_preserves_full_prior_populations_and_unknowns(tax_host_cold): + run = tax_host_cold + state = runner._run_entry(run)[2] + legacy, before, after = ( + state.legacy_financial_population, + state.property_population, + run.financial_population, + ) + assert before.version == legacy.version != after.version + for entity in before.frame.entities: + kept = [ + name + for name in before.frame.table(entity) + if name not in tax.TAX_LEAF_COLUMNS + ] + pd.testing.assert_frame_equal( + before.frame.table(entity)[kept], + after.frame.table(entity)[kept], + check_exact=True, + ) + assert before.frame.schema == after.frame.schema + assert before.frame.metadata == after.frame.metadata + assert before.frame.mass_log == after.frame.mass_log + assert after.mass_ledger[:-1] == before.mass_ledger + assert after.mass_ledger[-1].node_id == tax.RECEIVING_NODE + pd.testing.assert_series_equal(before.frame.strata, after.frame.strata) + for entity in before.design_weights: + np.testing.assert_array_equal( + before.design_weights[entity], after.design_weights[entity] + ) + expected = tax.split_property_tax_leaves(before.frame.person) + for name in tax.TAX_LEAF_COLUMNS: + np.testing.assert_array_equal( + after.frame.person[name].to_numpy().view("uint64"), + expected[name].to_numpy().view("uint64"), + ) + assert after.frame.person[list(tax.TAX_LEAF_COLUMNS)].isna().any().any() + assert legacy.frame.person[list(tax.TAX_LEAF_COLUMNS)].notna().all().all() + + +def test_incomplete_development_run_refuses_puf_before_source_qualification( + tax_host_cold, +): + run = tax_host_cold + assert ( + runner.codec.decode_json(runner._run_entry(run)[2].tax_verification)["complete"] + is False + ) + with pytest.raises(ValueError, match="PROPERTY_TAX_INPUTS_INCOMPLETE"): + recipients.values.qualify_puf55_survey_recipients(run) + edges = recipients._edges(run) + edge = next(e for e in edges if e.name == "property_tax_verification") + assert edge.producer == tax.GATE_NODE and edge.type == tax.VERIFICATION_TYPE + assert recipients._pins(run)[edge.name]["payload_sha256"] == runner.codec.sha( + runner._run_entry(run)[2].tax_verification + ) + + +@pytest.mark.parametrize("target", ["property", "leaf", "unknown"]) +def test_retained_tax_lifetime_refuses_changes(tax_host_cold, target): + run = tax_host_cold + state = runner._run_entry(run)[2] + population = ( + state.property_population if target == "property" else run.financial_population + ) + table = population.frame.person + column = ( + tax.PROPERTY_COMPONENTS[0] if target == "property" else tax.TAX_LEAF_COLUMNS[0] + ) + row = table.index[ + table[column].isna() if target == "unknown" else table[column].notna() + ][0] + old = table.loc[row, column] + table.loc[row, column] = 17.0 + try: + with pytest.raises(ValueError, match="POPULATION_CHANGED"): + run.checked_view() + finally: + table.loc[row, column] = old + runner._pure_run(run, runner._run_entry(run)) + + +def test_rebased_dataclass_copy_remains_unissued(tax_host_cold): + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN"): + replace(tax_host_cold).checked_view() + + +@pytest.mark.parametrize("flag", [None, 1, "yes", True]) +def test_invalid_or_unconfigured_rebase_refuses_before_source_io(flag): + with pytest.raises( + ValueError, match="PROPERTY_TAX_FLAG|PROPERTY_TAX_REQUIRES_PROPERTY" + ): + runner.run_atomic_survey_financial( + None, + snapshot_root=None, + store_root=None, + fraction=1.0, + seed=17, + geography_config=None, + rebase_property_taxes=flag, + ) + + +def test_completion_private_artifact_and_aggregate_receipt_replay( + tax_host_cold, tax_host_warm +): + """Actual enabled host/required path, with no additional source qualification.""" + private_payloads, receipts = [], [] + for run in (tax_host_cold, tax_host_warm): + state = runner._run_entry(run) + record = run.manifest.node(property_graph.PROJECTION_NODE) + assert set(record.opaque_artifacts) == { + "projection", + "donor_matrix", + "recipient_matrix", + "completion_routing", + } + payload = run.store.load_bytes(record.opaque_artifacts["completion_routing"]) + runner.survey._final_artifact( + run.manifest, + run.store, + node_id=property_graph.PROJECTION_NODE, + name="completion_routing", + type_=property_graph.completion.PROPERTY_COMPLETION_TYPE, + payload=payload, + capabilities=run.kernels.get( + property_graph.CurrentSurveyPropertyProjectionKernel.ref + ).capabilities, + ) + assert ( + property_graph.PROJECTION_NODE, + "completion_routing", + runner.codec.sha(payload), + ) in state[2].artifact_hashes + private = json.loads(payload) + assert private["stage"] == "before_property_models" + assert set(private["tables"]) == { + "person", + "components", + "reasons", + "clones", + "summary", + } + assert len( + private["tables"]["clones"]["rows"] + ) == run.financial_population.frame.n("person") + receipt = record.receipt["completion_routing"] + assert set(receipt) == { + "summary", + "private_artifact", + "private_artifact_sha256", + } + assert receipt["private_artifact"] == "completion_routing" + assert receipt["private_artifact_sha256"] == runner.codec.sha(payload) + summary = receipt["summary"] + assert ( + summary["source_authority"] is False + and summary["amounts_assigned"] is False + ) + assert summary["model_executed"] is False + assert summary["routes_mutually_exclusive"] is True + assert summary["reasons_overlap"] is True + assert all( + set(row) == {"selection", *property_graph.completion._PUBLIC_MEASURES} + for row in summary["summary"] + ) + public = explain_html(run.compiled, run.manifest) + describe( + run.compiled, property_graph.PROJECTION_NODE, run.manifest + ) + assert "completion_routing" in public and "carry_known_components" in public + assert payload.decode() not in public + private_payloads.append(payload) + receipts.append(receipt) + runner._pure_run(run, state) + assert private_payloads[0] == private_payloads[1] + assert receipts[0] == receipts[1] + + +def test_completion_artifact_retained_lifetime_refuses_store_tampering(tax_host_cold): + """Keep this destructive artifact control last; restore only test-owned bytes.""" + run = tax_host_cold + record = run.manifest.node(property_graph.PROJECTION_NODE) + path = ( + run.store.object_path(record.opaque_artifacts["completion_routing"]) + / "payload.bin" + ) + original = path.read_bytes() + try: + path.write_bytes(original + b" ") + with pytest.raises(StoreCorrupt): + run.checked_view() + finally: + path.write_bytes(original) + runner._pure_run(run, runner._run_entry(run)) diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py b/packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py new file mode 100644 index 000000000..6491c2576 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py @@ -0,0 +1,334 @@ +"""Nineteen actual graph nodes over invented, source-issued survey fixtures.""" + +import hashlib +import sys +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments +from test_us_graph_atomic_survey_population import _support_payload +from test_us_native_verify_once_epoch import _Counter + +from microcosm.build.us_runtime import graph_atomic_survey_financial as runner +from microcosm.build.us_runtime import survey_population_preparation as preparation +from microcosm.graph import ArtifactType, ArtifactValue, NumericScope +from microcosm.graph.keys import opaque_artifact_key + +financial = runner.financial +values = financial.values + + +@pytest.fixture(scope="module") +def known_financial_run(tmp_path_factory): + root = tmp_path_factory.mktemp("atomic-survey-financial") + with pytest.MonkeyPatch.context() as patch: + arguments = _demographic_arguments(root, patch, unknown=False, zero=False) + payload, source_ids = _support_payload() + support_path = root / "invented-block-support.npz" + support_path.write_bytes(payload) + config = runner.reconstruction.AtomicSurveyReconstruction( + support_path=str(support_path), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + call = { + **arguments, + "store_root": root / "store", + "geography_config": config, + "demographic_conditioning": True, + "n_estimators": 2, + "return_values": True, + } + cold = runner.run_atomic_survey_financial(**call, resume="auto") + warm = runner.run_atomic_survey_financial(**call, resume="require") + yield SimpleNamespace(call=call, cold=cold, warm=warm) + for run in (cold, warm): + values.source.verify_survey_population_preparation(run.prefix.preparation) + + +def test_post_run_check_fully_validates_once_and_at_close(known_financial_run): + run = known_financial_run.cold + with _Counter(preparation._source_files) as counter: + checked = run.checked_view() + assert counter.counts["_source_files"] == 2 + assert preparation.epoch_record() is None + assert preparation._MEMO == {} + with preparation.verification_epoch() as record: + run.prefix.preparation.checked_view() + with _Counter(preparation._source_files) as counter: + joined = run.checked_view() + assert joined.payload == checked.payload + assert joined.digest == checked.digest + assert joined.population is checked.population + assert counter.counts["_source_files"] == 0 + assert record["final_validations"] == 0 + assert record["final_validations"] == 1 + assert preparation._MEMO == {} + + +def test_nineteen_node_financial_cold_and_required_replay(known_financial_run): + case = known_financial_run + assert case.cold.manifest.key == case.warm.manifest.key + assert all(n.hit for n in case.warm.manifest.nodes.values()) + # Each run's verification-epoch record, read off the manifests it returns: + # this runner's own epoch on the outer manifest, and the nested nine-node + # population epoch's on the prefix's. Both epochs close before their runner + # returns, so what a caller reads here is the closed record. Every capsule + # either epoch memoised was re-validated in full as it closed + # (`final_validations == capsules`), which is what leaves the memo without + # authority; drop the `_verification_epoch=` argument from either runner and + # the corresponding record is empty here. `to_json` still cannot see it, so + # no receipt byte moves. The counts themselves are asserted over the + # nine-node runner in `test_us_native_verify_once_epoch.py` + # (`test_a_real_run_records_its_epoch_in_the_manifest`); this pins the same + # record on the nineteen-node runner, at no extra run cost. + for run in (case.cold, case.warm): + for manifest in (run.manifest, run.prefix.manifest): + record = manifest.verification_epoch + assert record["protocol"] == preparation.PROTOCOL + "/verification-epoch/1" + assert record["final_validations"] == record["capsules"] >= 1 + assert "verification_epoch" not in manifest.to_json() + for run in (case.cold, case.warm): + assert len(run.compiled.order) == 19 + edge = financial._geography_edge() + assert edge.producer in run.compiled.predecessors[financial.DONOR_NODE] + gate = run.manifest.node(edge.producer) + assert gate.opaque_artifacts[edge.artifact] == opaque_artifact_key( + gate.key, edge.artifact + ) + assert run.store.load_bytes( + gate.opaque_artifacts[edge.artifact] + ) == financial.codec.encode_json( + financial.codec.decode_json(run.projection)["atomic_geography"][ + "validation_receipt" + ] + ) + assert run.prefix.clone_population is run.prefix.geography_population + assert "census_block_geoid" not in run.prefix.expanded_population.frame.table( + "household" + ) + assert set(run.compiled.order) == { + *run.prefix.compiled.order, + financial.PROJECTION_NODE, + financial.DONOR_NODE, + financial.DONOR_COLUMNS_NODE, + *(f"{financial.FIT_PREFIX}.{i:03d}" for i in range(3)), + *(f"{financial.APPLY_PREFIX}.{i:03d}" for i in range(3)), + financial.ATTACH_NODE, + } + before, after = run.prefix.clone_population, run.financial_population + assert before is not after + assert "census_block_geoid" not in run.prefix.allocated_population.frame.table( + "household" + ) + assert "census_block_geoid" in before.frame.table("household") + assert ( + tuple(financial.model_input.decode_recipient_matrix(run.matrix).features) + == values.DEMOGRAPHIC_FEATURES + ) + evidence = financial.codec.decode_json(run.projection) + assert ( + evidence["model_judgments"]["demographic_conditioning"][ + "asec_sex_and_state_observation_year" + ] + == 2025 + ) + assert evidence["atomic_geography"]["config_sha256"] == financial.codec.sha( + run.prefix.geography_config.to_bytes() + ) + assert evidence["release_eligible"] is False + assert not any("prior" in c for c in values.DEMOGRAPHIC_FEATURES) + assert len(values.OUTPUTS) == 8 + for entity in before.frame.entities: + retained = [ + c + for c in before.frame.table(entity) + if entity != "person" or c not in values.OUTPUTS + ] + pd.testing.assert_frame_equal( + after.frame.table(entity)[retained], + before.frame.table(entity)[retained], + check_exact=True, + ) + assert after.version == before.version + assert after.weight_kind == before.weight_kind + assert after.mass_ledger == before.mass_ledger + assert after.frame.metadata == before.frame.metadata + assert after.frame.mass_log == before.frame.mass_log + assert after.frame.schema == before.frame.schema + assert after.frame.links == before.frame.links + pd.testing.assert_series_equal( + after.frame.strata, before.frame.strata, check_exact=True + ) + assert dict(after.owners) == { + **before.owners, + **{("person", c): financial.ATTACH_NODE for c in values.OUTPUTS}, + } + for entity in before.design_weights: + np.testing.assert_array_equal( + after.design_weights[entity], before.design_weights[entity] + ) + np.testing.assert_array_equal( + after.frame.weights_for(entity).values, + before.frame.weights_for(entity).values, + ) + # Source-origin pairing must carry each modeled ACS draw to both arms. + person = after.frame.person + source_id = values.provenance.support_source_id_column("person") + for _, rows in person.groupby(source_id, sort=False): + assert len(rows) == 2 + for column in values.OUTPUTS: + assert rows[column].notna().all() + assert rows[column].nunique() == 1 + runner.survey._same_frame(after.frame, run.manifest.population(after.version)) + runner.atomic.same_replayed_population( + case.cold.financial_population, case.warm.financial_population + ) + + +def test_geography_option_preserves_default_three_features(known_financial_run): + run = known_financial_run.cold + prefix = run.prefix + default = values.qualify_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + geography_config=prefix.geography_config, + ) + opted = values.qualify_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + geography_config=prefix.geography_config, + demographic_conditioning=True, + ) + assert default.demographic_conditioning is False + assert ( + tuple(financial.model_input.decode_recipient_matrix(default.matrix).features) + == values.FEATURES + ) + assert len(values.FEATURES) == 3 + assert len(values.DEMOGRAPHIC_FEATURES) == 5 + pd.testing.assert_frame_equal( + default.native_money, opted.native_money, check_exact=True + ) + assert ( + default.geography_config_payload + == opted.geography_config_payload + == prefix.geography_config.to_bytes() + ) + # A geography-enriched clone cannot be admitted through the historical arm. + with pytest.raises(ValueError, match="CLONE_FRAME_VALUES"): + values.qualify_current_survey_predictors( + prefix.preparation, prefix.allocated_population, prefix.clone_population + ) + + +def test_final_support_return_cannot_mutate_detached_money(known_financial_run): + case, fired = known_financial_run, [] + + def profile(frame, event, arg): + caller = frame.f_back + if ( + event == "return" + and frame.f_code is runner.reconstruction._read_support.__code__ + and caller is not None + and caller.f_code is values.qualify_current_survey_predictors.__code__ + and "result" in caller.f_locals + and not fired + ): + fired.append(True) + result = caller.f_locals["result"] + index = result.origins.index[result.origins.source.eq("asec")][0] + result.native_money.loc[index, "WSAL_VAL"] += 1 + + previous = sys.getprofile() + sys.setprofile(profile) + try: + prefix = case.cold.prefix + with pytest.raises(ValueError, match="FINAL_DERIVED_VALUES"): + values.qualify_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + geography_config=prefix.geography_config, + demographic_conditioning=True, + ) + finally: + sys.setprofile(previous) + assert fired == [True] + + +def test_final_owner_return_cannot_mutate_materialized_geography(known_financial_run): + case, fired = known_financial_run, [] + + def wrapper_profile(frame, event, arg): + caller = frame.f_back + if ( + event == "return" + and frame.f_code + is financial.verify_materialized_current_survey_predictors.__code__ + and caller is not None + and caller.f_code is runner.run_atomic_survey_financial.__code__ + and "result" in caller.f_locals + and not fired + ): + fired.append(True) + table = caller.f_locals["result"].financial_population.frame.table( + "household" + ) + table.loc[table.index[0], "survey_observed_state"] = "99" + + previous = sys.getprofile() + sys.setprofile(wrapper_profile) + try: + with pytest.raises(ValueError, match="ATOMIC_FINAL_POPULATION_MUTATION"): + runner.run_atomic_survey_financial(**case.call, resume="require") + finally: + sys.setprofile(previous) + assert fired == [True] + + +@pytest.mark.parametrize( + "defect,reason", + ( + ("payload", "GEOGRAPHY_VALIDATION_BINDING"), + ("producer", "GEOGRAPHY_VALIDATION_BINDING"), + ("type", "DETAIL_ARTIFACT_IDENTITY"), + ), +) +def test_financial_donor_requires_actual_geography_gate( + known_financial_run, defect, reason +): + run = known_financial_run.cold + node = run.compiled.graph.node(financial.DONOR_NODE) + artifacts = {} + for edge in node.artifact_inputs: + producer = run.manifest.node(edge.producer) + key = producer.opaque_artifacts[edge.artifact] + artifacts[edge.name] = ArtifactValue( + run.store.load_bytes(key), edge.type, key, producer.key, NumericScope() + ) + # A detached consumer-context negative, using actual source-owned run keys. + # This does not pretend that constructing ArtifactValue issues an authority. + edge = financial._geography_edge() + original = artifacts[edge.name] + if defect == "payload": + artifacts[edge.name] = replace(original, payload=original.payload + b" ") + elif defect == "producer": + other = "0" * 64 if original.producer_key != "0" * 64 else "1" * 64 + artifacts[edge.name] = replace( + original, producer_key=other, key=opaque_artifact_key(other, edge.artifact) + ) + else: + artifacts[edge.name] = replace( + original, type=ArtifactType("invented.wrong_gate", 1) + ) + context = SimpleNamespace(node=node, sources={}, artifacts=artifacts) + kernel = run.kernels.get(financial.CurrentSurveyPredictorDonorFilterKernel.ref) + with pytest.raises(ValueError, match=reason): + kernel._qualified(context) diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py b/packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py new file mode 100644 index 000000000..43dc09b50 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py @@ -0,0 +1,396 @@ +"""Actual survey/geography/clone execution over pinned invented originals. + +Normalized invented support proves no publisher provenance or native acceptance. +Invented registry pins bind source fixtures; real issuers and kernels execute. +""" + +import hashlib +import json +import sys +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments + +from microcosm.build import atomic_geography as atomic +from microcosm.build.us_runtime import atomic_block_support as blocks +from microcosm.build.us_runtime import current_survey_geography as observed +from microcosm.build.us_runtime import graph_atomic_survey_population as runner +from microcosm.build.us_runtime import graph_combined_clone as clone +from microcosm.build.us_runtime import graph_current_survey_geography as projection +from microcosm.build.us_runtime import graph_survey_population as survey +from microcosm.build.us_runtime import puf_support +from microcosm.build.us_runtime import survey_atomic_geography as reconstruction +from microcosm.build.us_runtime import survey_population_preparation as source +from microcosm.build.us_runtime import survey_population_replay as replay +from microcosm.frame import WeightKind + + +def _support_payload(*, first_population=2): + areas = tuple( + int(value) + for value in ( + "060010201001000", + "060010201001001", + "060010202001000", + "360010001001000", + ) + ) + source_ids = { + name: "invented-" + name for name in ("district", "population", "puma") + } + payload = blocks.assemble_atomic_block_support( + block_population=dict(zip(areas, (first_population, 6, 3, 5), strict=True)), + cd_by_block=dict(zip(areas, (601, 602, 603, 3601), strict=True)), + puma_by_tract={ + areas[0] // 10000: 612345, + areas[2] // 10000: 699999, + areas[3] // 10000: 3600001, + }, + source_ids=source_ids, + ) + return payload, source_ids + + +@pytest.fixture(scope="module") +def known_atomic_run(tmp_path_factory): + root = tmp_path_factory.mktemp("atomic-survey-population") + with pytest.MonkeyPatch.context() as patch: + arguments = _demographic_arguments(root, patch, unknown=False) + payload, source_ids = _support_payload() + support_path = root / "invented-block-support.npz" + support_path.write_bytes(payload) + config = reconstruction.AtomicSurveyReconstruction( + support_path=str(support_path), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + store_root = root / "store" + runs = tuple( + runner.run_atomic_survey_population( + **arguments, + store_root=store_root, + geography_config=config, + resume=resume, + return_values=True, + ) + for resume in ("auto", "require") + ) + yield SimpleNamespace( + arguments=arguments, + payload=payload, + config=config, + store_root=store_root, + cold=runs[0], + warm=runs[1], + ) + for run in runs: + source.verify_survey_population_preparation(run.preparation) + + +def _assert_complete_clone(before, after): + expected = puf_support.clone_us_frame_for_puf_support( + before.frame, clone_attachment_fraction=1.0, clone_attachment_seed=0 + ) + replay.same_replayed_frame(expected, after.frame) + # Independently pair both roles by assembly source ID, so inheritance is + # checked independently of the clone operator's row ordering/remapping. + for entity in before.frame.entities: + source_id = puf_support.support_source_id_column(entity) + clone_index = puf_support.support_clone_index_column(entity) + original = before.frame.table(entity).set_index( + source_id, verify_integrity=True + ) + expanded = after.frame.table(entity) + assert len(expanded) == 2 * len(original) + assert set(expanded[clone_index]) == {0, 1} + remapped = {before.frame.schema.entity_id_column(entity), clone_index} + if entity == before.frame.schema.person_entity: + remapped.update( + before.frame.schema.membership_column(group) + for group in before.frame.schema.group_entities + ) + inherited = [column for column in original if column not in remapped] + for role in (0, 1): + arm = expanded.loc[expanded[clone_index].eq(role)].set_index( + source_id, verify_integrity=True + ) + pd.testing.assert_index_equal(arm.index, original.index, exact=True) + pd.testing.assert_frame_equal( + arm.loc[original.index, inherited], + original.loc[:, inherited], + check_exact=True, + ) + assert after.version == clone.COMBINED_CLONE_NODE + assert after.mass_ledger[:-1] == before.mass_ledger + assert ( + after.weight_kind == before.weight_kind == {"household": WeightKind.IMPORTANCE} + ) + np.testing.assert_array_equal( + after.frame.weights_for("household").values, + np.tile(before.frame.weights_for("household").values / 2, 2), + ) + assert set(after.design_weights) == set(before.design_weights) == {"household"} + np.testing.assert_array_equal( + after.design_weights["household"], + np.tile(before.design_weights["household"], 2), + ) + assert dict(after.owners) == { + (entity, column): ( + clone.COMBINED_CLONE_CLAIM_NODE + if column == puf_support.support_clone_index_column(entity) + else clone.COMBINED_CLONE_NODE + ) + for entity in after.frame.entities + for column in after.frame.table(entity) + } + + +def test_cold_and_required_replay_assign_after_complete_clone(known_atomic_run): + case = known_atomic_run + assert case.cold.manifest.key == case.warm.manifest.key + assert all(record.hit for record in case.warm.manifest.nodes.values()) + assert all( + not record.hit + for name, record in case.cold.manifest.nodes.items() + if name not in {survey.CREATE_NODE, survey.ALLOCATION_NODE} + ) + for field in ( + "allocated_population", + "observed_population", + "expanded_population", + "geography_population", + "clone_population", + ): + replay.same_replayed_population( + getattr(case.cold, field), getattr(case.warm, field) + ) + support = atomic.decode_atomic_support(case.payload) + for run in (case.cold, case.warm): + raw, observed_population = run.allocated_population, run.observed_population + expanded, geography = run.expanded_population, run.geography_population + assert run.clone_population is geography + assert raw.version == observed_population.version == survey.ALLOCATION_NODE + assert expanded.version == geography.version == clone.COMBINED_CLONE_NODE + assert len(run.compiled.order) == 9 + stages = ( + survey.ALLOCATION_NODE, + projection.NODE, + clone.COMBINED_CLONE_NODE, + clone.COMBINED_CLONE_CLAIM_NODE, + "geography.assign", + "geography.derive", + "geography.gate", + ) + positions = tuple(run.compiled.order.index(name) for name in stages) + assert positions == tuple(sorted(positions)) + assert ( + clone.COMBINED_CLONE_CLAIM_NODE + in run.compiled.predecessors["geography.assign"] + ) + rebuilt = reconstruction.reconstruct_atomic_survey_geography( + run.preparation, raw, case.config + ) + for expected, actual in ( + (rebuilt.observed_population, observed_population), + (rebuilt.expanded_population, expanded), + (rebuilt.population, geography), + ): + replay.same_replayed_population(expected, actual) + receipt, definition = ( + json.loads(rebuilt.receipt), + json.loads(rebuilt.definition), + ) + assert receipt["protocol"] == "microcosm.us.atomic-survey-reconstruction.v2" + assert receipt["assignment_identity"] == list( + reconstruction.composition.ASSIGNMENT_IDENTITY + ) + assert rebuilt.support_payload == case.payload + assert receipt["support_sha256"] == case.config.support_sha256 + for name in ( + "publisher_provenance_established", + "source_admission_issued", + "population_admission_issued", + "release_eligible", + ): + assert receipt[name] is False + assigned = tuple(definition["outputs"].values()) + derived = tuple(layer["output"] for layer in definition["systems"][0]["layers"]) + assert not set((*observed.COLUMNS, *assigned, *derived)) & set( + raw.frame.table("household") + ) + assert not set((*assigned, *derived)) & set(expanded.frame.table("household")) + _assert_complete_clone(observed_population, expanded) + # Both ordinary projection steps preserve every incumbent cell, entity, + # membership, axis, weight, design anchor and context field exactly. + for before, after, added in ( + (raw, observed_population, observed.COLUMNS), + (expanded, geography, (*assigned, *derived)), + ): + assert set(after.frame.table("household")) - set( + before.frame.table("household") + ) == set(added) + for entity in before.frame.entities: + pd.testing.assert_frame_equal( + after.frame.table(entity).loc[ + :, before.frame.table(entity).columns + ], + before.frame.table(entity), + check_exact=True, + ) + assert after.frame.schema == before.frame.schema + assert after.frame.entities == before.frame.entities + assert after.frame.links == before.frame.links == () + assert after.frame.metadata == before.frame.metadata + assert after.frame.mass_log == before.frame.mass_log + assert after.mass_ledger == before.mass_ledger + assert after.weight_kind == before.weight_kind + assert after.frame.weighted_entities == before.frame.weighted_entities + pd.testing.assert_series_equal( + after.frame.strata, before.frame.strata, check_exact=True + ) + for entity in before.design_weights: + np.testing.assert_array_equal( + after.design_weights[entity], before.design_weights[entity] + ) + np.testing.assert_array_equal( + after.frame.weights_for(entity).values, + before.frame.weights_for(entity).values, + ) + assert dict(observed_population.owners) == { + **raw.owners, + **{("household", name): projection.NODE for name in observed.COLUMNS}, + } + assert dict(geography.owners) == { + **expanded.owners, + **{("household", name): "geography.assign" for name in assigned}, + **{("household", name): "geography.derive" for name in derived}, + } + replay.same_replayed_frame( + observed_population.frame, run.manifest.population(survey.ALLOCATION_NODE) + ) + replay.same_replayed_frame( + geography.frame, run.manifest.population(clone.COMBINED_CLONE_NODE) + ) + households = geography.frame.table("household") + assert len(households) == 12 + assert not households.duplicated( + list(reconstruction.composition.ASSIGNMENT_IDENTITY) + ).any() + keys = households[observed.COLUMNS[0]].map(json.loads) + acs = keys.map(lambda key: key[0] == "acs") + assert acs.sum() == 8 + assert households.loc[acs, "survey_observed_puma"].eq("0612345").all() + assert households.loc[acs, "assigned_puma_geoid"].eq("0612345").all() + assert households.loc[~acs, "survey_observed_puma"].isna().all() + for native, state in (("00007", "06"), ("00008", "36")): + selected = keys.map( + lambda key, native=native: key[0] == "asec" and key[-1] == native + ) + assert selected.sum() == 2 + assert households.loc[selected, "survey_observed_state"].tolist() == [ + state, + state, + ] + assert households.loc[selected, "assigned_state_fips"].tolist() == [ + state, + state, + ] + assert households.loc[acs, "assigned_state_fips"].eq("06").all() + assert set(households.census_block_geoid) <= set(support.arrays["area"]) + atomic.validate_geography(households, definition, {blocks.SYSTEM: support}) + gate = run.manifest.node("geography.gate") + assert gate.receipt["outcome"] == "pass" + assert ( + run.store.load_bytes(gate.opaque_artifacts["validation"]) + == rebuilt.stages[-1].receipt + ) + + +def test_changed_normalized_support_is_refused_by_exact_digest( + known_atomic_run, tmp_path +): + case = known_atomic_run + path = tmp_path / "changed-invented-support.npz" + changed, _ = _support_payload(first_population=3) + assert changed != case.payload + atomic.decode_atomic_support(changed) + path.write_bytes(changed) + config = replace(case.config, support_path=str(path)) + with pytest.raises( + ValueError, match="ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_CHANGED" + ): + reconstruction.reconstruct_atomic_survey_geography( + case.cold.preparation, case.cold.allocated_population, config + ) + + +def test_enriched_population_cannot_substitute_for_raw_allocation(known_atomic_run): + run = known_atomic_run.cold + with pytest.raises(ValueError, match="MATERIALIZED_FRAME_VALUES"): + reconstruction.reconstruct_atomic_survey_geography( + run.preparation, run.geography_population, known_atomic_run.config + ) + + +@pytest.mark.parametrize( + "change,reason", + ( + ("observed_cell", "SURVEY_POPULATION_REPLAY_"), + ("observed_snapshot", "SURVEY_POPULATION_REPLAY_"), + ("expanded_snapshot", "SURVEY_POPULATION_REPLAY_"), + ("source_container", "ATOMIC_FINAL_RECONSTRUCTION"), + ), +) +def test_late_returned_population_or_sources_mutation_refuses( + known_atomic_run, change, reason +): + case, fired = known_atomic_run, [] + + def trace(frame, event, arg): + caller = frame.f_back + if ( + event == "return" + and frame.f_code + is reconstruction.reconstruct_atomic_survey_geography.__code__ + and caller is not None + and caller.f_code is runner.run_atomic_survey_population.__code__ + and "result" in caller.f_locals + and not fired + ): + fired.append(True) + result = caller.f_locals["result"] + if change == "observed_cell": + households = result.geography_population.frame.table("household") + households.loc[households.index[0], "survey_observed_state"] = "99" + elif change in {"observed_snapshot", "expanded_snapshot"}: + value = ( + result.observed_population + if change == "observed_snapshot" + else result.expanded_population + ) + households = value.frame.table("household") + households.loc[households.index[0], "survey_observed_state"] = "99" + else: + result.sources[blocks.SOURCE] = ( + str(result.sources[blocks.SOURCE]) + ".changed" + ) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError, match=reason): + runner.run_atomic_survey_population( + **case.arguments, + store_root=case.store_root, + geography_config=case.config, + resume="require", + return_values=True, + ) + finally: + sys.setprofile(previous) + assert fired == [True] diff --git a/packages/microcosm-build/tests/test_us_graph_compact_retention.py b/packages/microcosm-build/tests/test_us_graph_compact_retention.py new file mode 100644 index 000000000..b59cfcbed --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_compact_retention.py @@ -0,0 +1,395 @@ +"""Actual compiler/store/owners over invented financial and completion inputs.""" + +import gc +import hashlib +import json +import sys +import weakref +from dataclasses import replace +from types import SimpleNamespace + +import pytest +from test_us_graph_atomic_completion_host import ( + _assert_all_row_lineage, + _source_arguments, +) +from test_us_graph_atomic_survey_financial import ( + known_financial_run as known_financial_run, +) + +from microcosm.build.us_runtime import graph_atomic_survey_financial as host +from microcosm.build.us_runtime import graph_survey_completion_host as completion +from microcosm.build.us_runtime.survey_population_replay import same_replayed_population + + +def _same_run(expected, actual): + assert expected.manifest.key == actual.manifest.key + assert expected.checked_view().payload == actual.checked_view().payload + left, right = host._run_entry(expected)[2], host._run_entry(actual)[2] + assert left.artifact_hashes == right.artifact_hashes + assert left.node_states == right.node_states + for name in expected.compiled.order: + a, b = expected.manifest.node(name), actual.manifest.node(name) + assert a.key == b.key and a.receipt == b.receipt + same_replayed_population(expected.financial_population, actual.financial_population) + + +def _record_snapshots(patch): + recorder = SimpleNamespace(references=[]) + original = host.run_graph + + def run_graph(compiled, **arguments): + observer = arguments["_population_observer"] + + def observe(node_id, population): + recorder.references.append( + (compiled.order, node_id, weakref.ref(population)) + ) + observer(node_id, population) + + return original(compiled, **{**arguments, "_population_observer": observe}) + + patch.setattr(host, "run_graph", run_graph) + return recorder + + +def _assert_retention(recorder, runs): + retained_by_order = {} + for run in runs: + state = host._run_entry(run)[2] + assert state.retention_profile == "compact" + assert tuple(name for name, _ in state.node_witnesses) == run.compiled.order + assert len(state.node_populations) == len(state.retained_node_ids) + retained_by_order[run.compiled.order] = set(state.retained_node_ids) + if state.completion_boundary is not None: + boundary = state.completion_boundary + assert not (set(boundary.observed) & set(boundary.base.compiled.order)) + base_state = host._run_entry(boundary.base)[2] + assert ( + base_state.retention_profile == "compact" + ) # recursion did not fall back + retained_by_order[boundary.base.compiled.order] = set( + base_state.retained_node_ids + ) + gc.collect() + discarded = 0 + for order, node, reference in recorder.references: + assert order in retained_by_order + if node not in retained_by_order[order]: + assert reference() is None + discarded += 1 + else: + assert reference() is not None + assert discarded > 0 + + +def test_financial_compact_cold_required_parity_and_nonretention(known_financial_run): + case = known_financial_run + baseline_payload = case.cold.checked_view().payload + with pytest.MonkeyPatch.context() as patch: + recorder = _record_snapshots(patch) + call = { + **case.call, + "store_root": case.call["store_root"].parent / "compact-store", + "_population_retention": "compact", + } + cold = host.run_atomic_survey_financial(**call) + warm = host.run_atomic_survey_financial(**call, resume="require") + # The wrapper is a live implementation binding; compare while still installed. + _same_run(cold, warm) + assert cold.manifest.key == case.cold.manifest.key + assert cold.checked_view().payload == baseline_payload + assert all(node.hit for node in warm.manifest.nodes.values()) + _assert_retention(recorder, (cold, warm)) + state = host._run_entry(warm)[2] + assert state.retained_node_ids == (host.financial.ATTACH_NODE,) + key = warm.manifest.node(host.financial.APPLY_PREFIX + ".000").opaque_artifacts[ + "raw_draw" + ] + directory = warm.store.object_path(key) + payload_path, metadata_path = directory / "payload.bin", directory / "meta.json" + payload, metadata = payload_path.read_bytes(), metadata_path.read_bytes() + changed = payload + b"invented_changed" + document = json.loads(metadata) + document["payloads"]["payload.bin"] = { + "sha256": hashlib.sha256(changed).hexdigest(), + "size": len(changed), + } + try: + payload_path.write_bytes(changed) + metadata_path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + ) + assert warm.store.load_bytes(key) == changed + with pytest.raises(ValueError, match="FINANCIAL_RUN_ARTIFACT_CHANGED$"): + warm.checked_view() + finally: + payload_path.write_bytes(payload) + metadata_path.write_bytes(metadata) + original = state.node_witnesses + object.__setattr__( + state, + "node_witnesses", + ((original[0][0], original[0][1] + b" "), *original[1:]), + ) + with pytest.raises(ValueError, match="COMPACT_CAPSULE_CHANGED$"): + warm.checked_view() + object.__setattr__(state, "node_witnesses", original) + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN$"): + replace(warm).checked_view() + + +@pytest.mark.parametrize( + "person_status,household_roles,count", + [(False, False, 45), (True, False, 49), (True, True, 51)], +) +def test_compact_completion_cold_required_parity_and_live_custody( + tmp_path, person_status, household_roles, count +): + with pytest.MonkeyPatch.context() as patch: + call, _ = _source_arguments( + tmp_path, + patch, + person_status=person_status, + household_roles=household_roles, + ) + recorder = _record_snapshots(patch) + all_cold = host.run_atomic_survey_financial(**call) + all_warm = host.run_atomic_survey_financial(**call, resume="require") + # Ignore intentional all-profile retention when checking compact lifetimes. + recorder.references.clear() + compact = { + **call, + "store_root": tmp_path / "compact-store", + "_population_retention": "compact", + } + cold = host.run_atomic_survey_financial(**compact) + warm = host.run_atomic_survey_financial(**compact, resume="require") + for run in (all_warm, cold, warm): + _same_run(all_cold, run) + assert len(run.compiled.order) == count + _assert_all_row_lineage( + run, household_roles=household_roles, person_status=person_status + ) + assert all(node.hit for node in warm.manifest.nodes.values()) + _assert_retention(recorder, (cold, warm)) + state = host._run_entry(warm)[2] + boundary = state.completion_boundary + assert len(boundary.base.compiled.order) == (39 if person_status else 35) + assert all( + warm.manifest.node(node).hit for node in boundary.base.compiled.order + ) + for node in ( + completion.child.DONOR, + completion.child.RECIPIENT, + completion.child.FIT, + completion.child.DRAW, + ): + population = boundary.observed[node] + assert ( + completion._population_stamp(boundary, warm.compiled, node, population) + == dict(boundary.observed_stamps)[node] + ) + node = completion.child.DONOR + boundary.observed[node] = replace(boundary.observed[node]) + with pytest.raises(ValueError, match="COMPLETION_CUSTODY_CHANGED$"): + warm.checked_view() + assert boundary.revoked and boundary.child.revoked + + +@pytest.mark.parametrize("profile", [True, None, "unknown"]) +def test_unknown_retention_profile_refuses_before_source_work(profile): + with pytest.raises(ValueError, match="RETENTION_PROFILE$"): + host.run_atomic_survey_financial( + "never-read", + snapshot_root="never-created", + store_root="never-created", + fraction=None, + seed=1, + geography_config=None, + _population_retention=profile, + ) + + +@pytest.mark.parametrize( + "change", ["source", "implementation", "typed_contract", "capabilities", "writers"] +) +def test_frozen_population_inputs_rederive_current_union_obligations( + known_financial_run, monkeypatch, change +): + run = known_financial_run.cold + state = host._run_entry(run)[2] + populations = dict( + zip(run.compiled.order, (p for p, _ in state.node_populations), strict=True) + ) + inputs = { + node: host.atomic._state_population_inputs( + run.compiled, node, populations[node] + ) + for node in run.compiled.order + } + receipts = {node: value["receipt"] for node, value in state.node_states.items()} + + def derive(): + return host.atomic._states( + run.compiled, + run.kernels, + dict(state.source_keys), + {}, + receipts, + _retained_inputs=inputs, + ) + + baseline = derive() + host.survey._check_node_states(run.manifest, baseline) + if change in ("source", "implementation"): + original = host.atomic._all_node_keys + + def changed(*args): + keys, implementations = original(*args) + keys, implementations = dict(keys), dict(implementations) + target = keys if change == "source" else implementations + node = run.compiled.order[0] + target[node] = "a" * 64 if target[node] != "a" * 64 else "b" * 64 + return keys, implementations + + monkeypatch.setattr(host.atomic, "_all_node_keys", changed) + elif change == "typed_contract": + monkeypatch.setattr( + host.atomic, + "typed_contracts", + lambda *args: {"invented_changed_scope": True}, + ) + elif change == "capabilities": + original = host.atomic._capabilities_projection + monkeypatch.setattr( + host.atomic, + "_capabilities_projection", + lambda value: {**original(value), "invented_changed_scope": True}, + ) + else: + monkeypatch.setattr( + host.atomic, + "_input_writers", + lambda *args, **kwargs: { + ("person", "invented"): ("invented_union_writer",) + }, + ) + current = derive() + assert current != baseline + with pytest.raises(ValueError, match="MANIFEST_NODE_STATE$"): + host.survey._check_node_states(run.manifest, current) + + +@pytest.mark.parametrize("change", ["node", "version", "mass_partition"]) +def test_frozen_population_inputs_refuse_different_compiled_scope( + known_financial_run, change +): + from microcosm.graph.canonical import canonical_json + + run = known_financial_run.cold + state = host._run_entry(run)[2] + node = run.compiled.order[0] + population = state.node_populations[0][0] + payload = json.loads( + host.atomic._state_population_inputs(run.compiled, node, population) + ) + payload[change] = "invented_foreign_scope" + with pytest.raises(ValueError, match="RETAINED_STATE_SCOPE$"): + host.atomic._states( + run.compiled, + run.kernels, + dict(state.source_keys), + {}, + {}, + _retained_inputs={node: canonical_json(payload)}, + ) + + +@pytest.mark.parametrize( + "defect,reason", + [ + ("duplicate", "ATOMIC_OBSERVER_DUPLICATE"), + ("missing", "ATOMIC_OBSERVER_ROSTER"), + ("reordered", "ATOMIC_OBSERVER_ROSTER"), + ("changed", "SURVEY_POPULATION_REPLAY_"), + ], +) +def test_compact_discarded_observation_refusals( + known_financial_run, monkeypatch, defect, reason +): + original = host.run_graph + fired = [] + + def run_graph(compiled, **arguments): + observer = arguments["_population_observer"] + first = [] + + def observe(node_id, population): + if node_id == compiled.order[0]: + fired.append(True) + if defect == "missing": + return + if defect == "reordered": + first.append((node_id, population)) + return + if defect == "duplicate": + observer(node_id, population) + if defect == "changed": + table = population.frame.person + column = population.frame.schema.entity_id_column( + population.frame.schema.person_entity + ) + table.loc[table.index[0], column] += 1 + observer(node_id, population) + if first: + observer(*first.pop()) + + return original(compiled, **{**arguments, "_population_observer": observe}) + + monkeypatch.setattr(host, "run_graph", run_graph) + with pytest.raises(ValueError, match=reason): + host.run_atomic_survey_financial( + **known_financial_run.call, + resume="require", + _population_retention="compact", + ) + assert fired == [True] + + +def test_compact_expected_population_still_sealed_between_checkpoints( + known_financial_run, +): + fired = [] + + def profile(frame, event, arg): + caller = frame.f_back + if ( + event == "return" + and frame.f_code + is host.financial.verify_materialized_current_survey_predictors.__code__ + and caller is not None + and caller.f_code is host.run_atomic_survey_financial.__code__ + and "result" in caller.f_locals + and not fired + ): + fired.append(True) + population = caller.f_locals["expected"][host.financial.DONOR_COLUMNS_NODE] + table = population.frame.person + # An independently reconstructed donor output, not the prepared + # source Frame (whose earlier source seal would correctly refuse). + column = host.values.TARGETS[0] + table.loc[table.index[0], column] += 1 + + previous = sys.getprofile() + sys.setprofile(profile) + try: + with pytest.raises(ValueError, match="ATOMIC_FINAL_POPULATION_MUTATION$"): + host.run_atomic_survey_financial( + **known_financial_run.call, + resume="require", + _population_retention="compact", + ) + finally: + sys.setprofile(previous) + assert fired == [True] diff --git a/packages/microcosm-build/tests/test_us_graph_current_asec_development_inputs.py b/packages/microcosm-build/tests/test_us_graph_current_asec_development_inputs.py new file mode 100644 index 000000000..1dc8dc00f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_current_asec_development_inputs.py @@ -0,0 +1,480 @@ +"""Invented source/receiving components, never full financial-run issuance.""" + +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_asec_income_routing import routing_arguments +from test_us_puf55_observed_recipients import _invented_values +from test_us_puf55_survey_recipients import _frames + +from microcosm.build.us_runtime import graph_atomic_survey_financial as host +from microcosm.build.us_runtime import graph_current_asec_development_inputs as graph +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys +from microcosm.graph.population import Population + + +def _pure_fixture(): + fixed, frame = _invented_values(rules=graph.fixed.DEVELOPMENT_RULES) + original, _, _ = _frames() + frame.person.drop(columns=list(graph.OUTPUTS), inplace=True) + qualified = graph.routing.CurrentAsecIncomeRoutingValues( + fixed.source_basis, pd.DataFrame(), {"invented_component_only": True} + ) + origins = pd.DataFrame( + {"native_person_id": original.person.person_id.to_numpy(), "source": "asec"}, + index=pd.Index(original.person.person_id.to_numpy(), name="person_id"), + ) + return qualified, origins, frame + + +def test_keyed_signed_values_unknowns_and_unchanged_frame(): + qualified, originals, receiving = _pure_fixture() + before = graph.source._frame_identity(receiving) + receiving.person.iloc[:] = receiving.person.iloc[::-1].to_numpy() + values = graph.columns(qualified, originals, receiving.person) + assert values["person", "rental_income"].loc[1] == -40 + assert values["person", "farm_operations_income"].loc[1001] == -90 + assert np.isnan(values["person", "taxable_private_pension_income"].loc[2]) + assert values["person", "farm_operations_income"].loc[2] == -90 + for name in graph.OUTPUTS: + a, b = ( + values["person", name].loc[range(1, 11)], + values["person", name].loc[range(1001, 1011)], + ) + np.testing.assert_array_equal(a, b) + receiving.person.iloc[:] = receiving.person.iloc[::-1].to_numpy() + assert graph.source._frame_identity(receiving) == before + assert not graph.codec.decode_json(graph.qualification_payload(qualified))[ + "rule_metadata" + ]["observed_taxable_amount_claim"] + + +@pytest.mark.parametrize( + "change", ("incumbent", "origin", "clone", "basis", "nonfinite") +) +def test_refuses_unqualified_join_or_rewrite(change): + qualified, originals, receiving = _pure_fixture() + if change == "incumbent": + receiving.person[graph.OUTPUTS[0]] = 123.0 + elif change == "origin": + receiving.person.loc[0, graph.provenance.spine_source_id_column("person")] = 999 + elif change == "clone": + receiving.person.loc[ + 0, graph.provenance.support_clone_index_column("person") + ] = 1 + elif change == "basis": + qualified.person.loc[1, "native_person_id"] = 999 + else: + qualified.person.loc[1, "farm_known_amount"] = np.inf + with pytest.raises(ValueError): + graph.columns(qualified, originals, receiving.person) + + +def _clone_component(frame): + """Explicit fixture receiving support; this function issues no graph owner.""" + tables = {} + for entity in frame.entities: + original = frame.table(entity) + arms = [] + for arm in (0, 1): + value = original.copy(deep=True) + value[graph.provenance.support_source_id_column(entity)] = value[ + entity + "_id" + ] + value[graph.provenance.support_clone_index_column(entity)] = np.full( + len(value), arm, dtype=np.int64 + ) + value[entity + "_id"] += arm * 100000 + if entity == "person": + for group in frame.schema.group_entities: + value["person_" + group + "_id"] += arm * 100000 + arms.append(value) + tables[entity] = pd.concat(arms, ignore_index=True) + return Frame( + tables, + frame.schema, + { + "household": Weights( + np.tile(frame.weights_for("household").values / 2, 2), + WeightKind.IMPORTANCE, + ) + }, + pd.concat((frame.strata, frame.strata), ignore_index=True), + metadata=frame.metadata, + ) + + +def test_genuine_routing_two_real_nodes_cold_required_and_reconstruction( + tmp_path, monkeypatch +): + (tmp_path / "sources").mkdir() + arguments = routing_arguments(tmp_path / "sources", monkeypatch) + prepared = graph.source.prepare_authenticated_survey_population(**arguments) + entry = prepared._checked() + receiving = _clone_component(entry[2].frame) + original_stamp = graph.source._frame_identity(receiving) + source_node = Node( + "invented.receiving", + "invented.development_receiving@1", + structural=StructuralDelta.CREATE, + sources=("invented",), + outputs=tuple( + Owned( + e, c, graph.population_ops.token_for_dtype(receiving.table(e)[c].dtype) + ) + for e in receiving.entities + for c in receiving.table(e) + if c != receiving.schema.entity_id_column(e) + and not ( + e == "person" + and c + in { + receiving.schema.membership_column(g) + for g in receiving.schema.group_entities + } + ) + ), + ) + + class Receiving(KernelBase): + ref = source_node.kernel + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return "a" * 64 + + def run(self, context): + return KernelResult(frame=receiving) + + boundary = host._DevelopmentBoundary( + prepared, entry, Population.from_frame(receiving, source_node.id), {} + ) + nodes = graph.nodes( + boundary.qualified, + receiving, + population=source_node.id, + host_edges=(), + host_pins={}, + ) + boundary.nodes = nodes + compiled = compile_graph( + Graph("us", (SourceRef("invented", "raw-bytes-v1"),), (source_node, *nodes)) + ) + registry = KernelRegistry() + registry.register(Receiving()) + for node in nodes: + registry.register(graph.DevelopmentKernel(boundary, node)) + path = tmp_path / "component-source.txt" + path.write_text("Invented receiving support; genuine routing owns source values.") + store = ContentStore(tmp_path / "store") + sources = {"invented": path} + _, source_keys = _source_paths_and_keys(compiled, sources, store) + keys, _ = _all_node_keys(compiled, registry, source_keys) + boundary.keys = tuple(sorted(keys.items())) + observed = {} + cold = run_graph( + compiled, + sources=sources, + kernels=registry, + store=store, + _population_observer=lambda n, p: observed.update({n: p}), + ) + warm = run_graph( + compiled, sources=sources, kernels=registry, store=store, resume="require" + ) + assert warm.key == cold.key and all(n.hit for n in warm.nodes.values()) + expected = Population.from_frame(receiving, source_node.id) + for node in nodes: + result = graph.result( + boundary.qualified, boundary.originals, node, expected.frame.person + ) + for name, payload in result.artifacts.items(): + assert payload == store.load_bytes( + warm.node(node.id).opaque_artifacts[name] + ) + expected = graph.population_ops.patch(expected, node, result) + host.atomic.same_replayed_population(expected, observed[node.id]) + host.atomic.same_replayed_population( + boundary.complement(expected), Population.from_frame(receiving, source_node.id) + ) + assert graph.source._frame_identity(receiving) == original_stamp + asec = expected.frame.person[graph.provenance.support_channel_column("person")].eq( + "asec" + ) + assert expected.frame.person.loc[~asec, list(graph.OUTPUTS)].isna().all().all() + native = expected.frame.person[graph.provenance.spine_source_id_column("person")] + assert (asec & native.eq(105)).sum() == 2 + assert (asec & native.eq(107)).sum() == 2 + assert ( + expected.frame.person.loc[asec & native.eq(105), "rental_income"].eq(-400).all() + ) + assert ( + expected.frame.person.loc[asec & native.eq(107), "farm_operations_income"] + .eq(-9000) + .all() + ) + boundary.requalify() + expected.frame.person.loc[0, graph.OUTPUTS[0]] = 123.0 + with pytest.raises(ValueError, match="DEVELOPMENT_VALUE_CHANGED"): + boundary.complement(expected) + basis = boundary.qualified.person + basis.loc[basis.native_person_id.eq(105), "net_property_known_amount"] = -399.0 + with pytest.raises(ValueError, match="DEVELOPMENT_SOURCE_OR_RECEIVING_CHANGED"): + boundary.pure() + + +@pytest.mark.parametrize( + "binding", + ( + "targets", + "node", + "type", + "capability", + "method", + "fraction", + "mapper", + "origins", + ), +) +def test_live_contract_refuses_rebinding_before_source_io(monkeypatch, binding): + if binding == "targets": + monkeypatch.setattr( + graph.fixed, "DEVELOPMENT_TARGETS", graph.fixed.DEVELOPMENT_TARGETS[::-1] + ) + elif binding == "node": + monkeypatch.setattr(graph, "ATTACH_NODE", "other.attach") + elif binding == "type": + monkeypatch.setattr( + graph, "PROJECTION_TYPE", replace(graph.PROJECTION_TYPE, schema_version=2) + ) + elif binding == "capability": + monkeypatch.setattr( + graph.DevelopmentKernel, + "capabilities", + replace(graph.DevelopmentKernel.capabilities, dependencies=()), + ) + elif binding == "method": + monkeypatch.setattr(graph.DevelopmentKernel, "run", lambda *_: None) + elif binding == "mapper": + monkeypatch.setattr(graph.attachment, "attach_columns", lambda *_: {}) + elif binding == "origins": + monkeypatch.setattr(graph.original_source, "_origins", lambda *_: None) + else: + monkeypatch.setattr(graph.fixed.leaves, "TAXABLE_PENSION_FRACTION", 0.111) + assert graph._live() != graph._LIVE + with pytest.raises(ValueError, match="DEVELOPMENT_IMPLEMENTATION_CHANGED"): + host._DevelopmentBoundary(None, None, None, {}) + + +@pytest.mark.parametrize("development", (False, True)) +@pytest.mark.parametrize("completion", (False, True)) +def test_puf_receives_actual_terminal_after_completion(development, completion): + # Selector contract only. No fabricated issuance or execution receipt. + actual = host._financial_terminal( + property_income=object(), + rebase_property_taxes=completion, + completion_boundary=object() if completion else None, + development_boundary=object() if development else None, + ) + assert actual == ( + host._tax_module().GATE_NODE + if completion + else graph.ATTACH_NODE + if development + else host._property_module().ATTACH_NODE + ) + + +def test_additions_compose_with_actual_child_roles_tax_declarations_and_compact_roster(): + """Real fragment factories/compiler; invented values, no full51 execution.""" + from types import SimpleNamespace + + from test_us_child_property_income_graph import ( + _options, + _qualified, + _receiving_frame, + ) + from test_us_current_asec_income_routing import _pure_rows + from test_us_current_survey_household_roles import _qualified as role_values + from test_us_current_survey_household_roles import _table as role_table + + from microcosm.build.us_runtime import graph_child_property_income as child + from microcosm.build.us_runtime import graph_current_survey_household_roles as roles + from microcosm.build.us_runtime import graph_property_tax_leaves as tax + from microcosm.build.us_runtime import graph_survey_completion as receiving_graph + from microcosm.build.us_runtime import graph_survey_completion_host as completion + from microcosm.graph import ArtifactInput, ArtifactOutput, ArtifactType + + children, origins = _qualified() + frame = _receiving_frame(children, origins) + for name in (*host.values.OUTPUTS, *tax.PROPERTY_COMPONENTS): + if name not in frame.person: + frame.person[name] = np.zeros(len(frame.person), dtype=np.float64) + ordering_type = ArtifactType("invented.development_parent", 1) + create = Node( + host.financial.ATTACH_NODE, + "invented.property_parent@1", + structural=StructuralDelta.CREATE, + sources=(child.SOURCE_NAME,), + outputs=tuple( + Owned(e, c, graph.population_ops.token_for_dtype(frame.table(e)[c].dtype)) + for e in frame.entities + for c in frame.table(e) + if c != frame.schema.entity_id_column(e) + and not ( + e == "person" + and c + in { + frame.schema.membership_column(g) + for g in frame.schema.group_entities + } + ) + ), + artifact_outputs=(ArtifactOutput("parent", ordering_type),), + ) + parent_edge = ArtifactInput("parent", create.id, "parent", ordering_type) + subset = origins.loc[origins.source.eq("asec")] + raw, ages = _pure_rows( + len(subset), children.recipients.loc[subset.index, "source_age"].tolist() + ) + basis = graph.routing.project_income_routing(raw, ages) + basis.index = pd.Index(subset.index, name="person_id") + basis["native_person_id"] = subset.selected_receiving_person_id.to_numpy() + qualified = graph.routing.CurrentAsecIncomeRoutingValues( + basis, pd.DataFrame(), {"invented_declaration_only": True} + ) + additions = graph.nodes( + qualified, + frame, + population=create.id, + host_edges=(parent_edge,), + host_pins={}, + after_columns=host.values.OUTPUTS, + ) + base = compile_graph( + Graph( + "us", (SourceRef(child.SOURCE_NAME, "raw-bytes-v1"),), (create, *additions) + ) + ) + assert graph.ATTACH_NODE in host._compact_retained_roster(base, None, False) + original_axis = origins[["source"]].copy() + original_axis["native_person_id"] = origins.selected_receiving_person_id + augmented = graph.source._copy_source(frame) + for (_, name), vector in graph.columns( + qualified, original_axis, frame.person + ).items(): + augmented.person[name] = vector.to_numpy(copy=True) + after = ArtifactInput( + "development_binding", graph.ATTACH_NODE, "binding", graph.BINDING_TYPE + ) + receiving_node = receiving_graph.completion_receiving_node( + augmented, population=create.id, ordering=(after,) + ) + qualified_roles = role_values( + role_table( + tuple( + ( + r.Index, + r.selected_receiving_person_id, + r.source, + 1 if r.source == "asec" else 20, + 1, + 0, + ) + for r in origins.itertuples() + ) + ) + ) + role_nodes = roles.current_survey_household_roles_nodes( + qualified_roles, + augmented, + receiving_version=receiving_node.id, + host_pins={}, + after=after, + host_edges=(), + ) + with_roles = graph.source._copy_source(augmented) + for (_, name), vector in roles.roles.household_role_columns_for_population( + qualified_roles, with_roles + ).items(): + with_roles.person[name] = vector.array.copy() + parent = Population.from_frame(with_roles, receiving_node.id) + pins = { + after.name: { + "producer_key": "a" * 64, + "artifact_key": "b" * 64, + "payload_sha256": "c" * 64, + } + } + child_nodes = child.child_property_nodes( + children, + origins, + parent, + options=_options(), + host_edges=(after,), + host_pins=pins, + ) + finished = graph.source._copy_source(with_roles) + finished.person[child.STATUS] = pd.array( + ["invented"] * len(finished.person), dtype="string" + ) + finished.person[child.IMPUTED] = False + tax_nodes = tax.property_tax_leaf_nodes( + finished, + population=receiving_node.id, + projection=replace(parent_edge, name="projection"), + reconciliation=replace(parent_edge, name="reconciliation"), + atol=0.0, + rtol=0.0, + completion=ArtifactInput( + "child_verification", child.VERIFY, "verification", child.VERIFICATION_TYPE + ), + ) + boundary = SimpleNamespace( + household_roles=True, + receiving_node=receiving_node, + role_nodes=role_nodes, + child=SimpleNamespace(nodes=child_nodes), + base=SimpleNamespace(compiled=base), + ) + extensions = completion._completion_nodes(boundary, tax_nodes) + compiled = compile_graph( + replace(base.graph, nodes=(*base.graph.nodes, *extensions)) + ) + order = compiled.order + assert ( + order.index(graph.ATTACH_NODE) + < order.index(receiving_node.id) + < order.index(roles.BIND_NODE) + < order.index(child.ATTACH) + < order.index(tax.GATE_NODE) + ) + assert set(graph.OUTPUTS).isdisjoint( + o.column for n in extensions for o in n.outputs + ) + assert host._compact_retained_roster(compiled, None, False, boundary) == tuple( + n for n in order if n not in base.order + ) + assert set(graph.OUTPUTS) <= set( + c for selection in tax_nodes[0].inputs for c in selection.columns + ) diff --git a/packages/microcosm-build/tests/test_us_graph_current_survey_geography.py b/packages/microcosm-build/tests/test_us_graph_current_survey_geography.py new file mode 100644 index 000000000..eadbedd74 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_current_survey_geography.py @@ -0,0 +1,353 @@ +"""Observed-geography graph contracts over actual issuers and invented originals.""" + +import json +import sys +from dataclasses import replace +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments + +from microcosm.build.us_runtime import current_survey_geography as qualifier +from microcosm.build.us_runtime import graph_current_survey_geography as graph +from microcosm.build.us_runtime import graph_survey_population as population +from microcosm.build.us_runtime import survey_population_preparation as source +from microcosm.graph import ArtifactType, StructuralDelta, compile_graph, run_graph + + +def _executed(tmp_path, monkeypatch, *, unknown=False): + """Construct privately pinned invented originals for a complete graph run.""" + arguments = _demographic_arguments(tmp_path, monkeypatch, unknown=unknown) + return _execute(arguments, store_root=tmp_path / "store") + + +def _execute(arguments, *, store_root): + """Keep real preparation/artifact owners and capture a real kernel context.""" + prefix = population.run_authenticated_survey_population( + **arguments, + store_root=store_root, + clones=False, + return_values=True, + ) + qualified = qualifier.qualify_current_survey_geography(prefix.preparation) + node = graph.current_survey_geography_node( + preparation_sha256=population._sha(prefix.preparation.payload), + projection_receipt_sha256=population._sha(qualified.receipt), + population=population.ALLOCATION_NODE, + ) + compiled = compile_graph( + replace(prefix.compiled.graph, nodes=(*prefix.compiled.graph.nodes, node)) + ) + kernel = graph.CurrentSurveyGeographyKernel(prefix.preparation) + prefix.kernels.register(kernel) + contexts = [] + + def trace(frame, event, arg): + if event == "call" and frame.f_code is kernel.run.__code__: + contexts.append(frame.f_locals["context"]) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + manifest = run_graph( + compiled, + sources=prefix.sources, + store=prefix.store, + kernels=prefix.kernels, + ) + finally: + sys.setprofile(previous) + assert len(contexts) == 1 + return SimpleNamespace( + arguments=arguments, + prefix=prefix, + qualified=qualified, + node=node, + compiled=compiled, + kernel=kernel, + context=contexts[0], + manifest=manifest, + ) + + +@pytest.fixture(scope="module") +def known_run(tmp_path_factory): + """Share one real execution while its invented registry pins remain active.""" + with pytest.MonkeyPatch.context() as patch: + run = _executed(tmp_path_factory.mktemp("survey-geography-known"), patch) + yield run + # All consumers of this fixture must leave its issued source untouched. + source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture +def fresh_source_run(known_run, tmp_path): + """Reissue independent mutable Frames over the shared immutable originals.""" + run = _execute(known_run.arguments, store_root=tmp_path / "fresh-source-store") + assert run.prefix.preparation is not known_run.prefix.preparation + assert run.prefix.preparation.frame is not known_run.prefix.preparation.frame + yield run + source.verify_survey_population_preparation(known_run.prefix.preparation) + + +def _detached_context(context): + """Each negative gets independent tables, weights and container mappings.""" + return replace( + context, + tables={name: table.copy(deep=True) for name, table in context.tables.items()}, + weights={ + name: replace(value, values=value.values.copy()) + for name, value in context.weights.items() + }, + strata=context.strata.copy(deep=True), + params=dict(context.params), + sources=dict(context.sources), + artifacts=dict(context.artifacts), + tolerances=dict(context.tolerances), + numerics=dict(context.numerics), + ) + + +@pytest.mark.parametrize("unknown", (False, True)) +def test_real_projection_graph_execution_and_replay_preserve_inputs( + tmp_path, monkeypatch, unknown +): + run = _executed(tmp_path, monkeypatch, unknown=unknown) + assert run.compiled.order == ( + population.CREATE_NODE, + population.ALLOCATION_NODE, + graph.NODE, + ) + assert run.node.structural is StructuralDelta.NONE + assert run.node.sources == run.node.artifact_outputs == () + assert run.node.weights is None + assert tuple(o.column for o in run.node.outputs) == qualifier.COLUMNS + assert all( + o.entity == "household" and o.dtype == "string" and not o.rewrite + for o in run.node.outputs + ) + assert run.node.artifact_inputs[0].producer == population.CREATE_NODE + assert run.node.artifact_inputs[0].type == population.PREPARATION_TYPE + assert not run.manifest.node(graph.NODE).hit + original = run.prefix.allocated_population.frame + version = run.compiled.versions[graph.NODE] + projected = run.manifest.population(version) + for entity in original.entities: + pd.testing.assert_frame_equal( + projected.table(entity).loc[:, original.table(entity).columns], + original.table(entity), + check_exact=True, + ) + assert ( + projected.weights_for("household").kind + is original.weights_for("household").kind + ) + assert ( + projected.weights_for("household").values.tobytes() + == original.weights_for("household").values.tobytes() + ) + assert ( + run.manifest.mass_ledger(version) == run.prefix.allocated_population.mass_ledger + ) + table = projected.table("household").set_index("household_id") + actual = table.loc[:, list(graph.OUTPUT_COLUMNS)] + pd.testing.assert_frame_equal(actual, run.qualified.household, check_exact=True) + assert len(actual) == 6 + assert actual.survey_observed_state.isna().sum() == int(unknown) + assert actual.survey_observed_puma.notna().sum() == 4 + unknown_asec = actual.survey_geography_origin_key.eq('["asec",2024,2025,"00008"]') + assert unknown_asec.sum() == 1 + if unknown: + assert actual.loc[unknown_asec, "survey_observed_state"].isna().all() + else: + assert actual.loc[unknown_asec, "survey_observed_state"].tolist() == ["36"] + receipt = run.manifest.node(graph.NODE).receipt + assert receipt["projection_receipt_sha256"] == population._sha( + run.qualified.receipt + ) + assert receipt["source_projection"]["households"] == 6 + assert ( + not receipt["population_admission_issued"] and not receipt["release_eligible"] + ) + + warm_calls = [] + + def trace(frame, event, arg): + if event == "call" and frame.f_code is run.kernel.run.__code__: + warm_calls.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + warm = run_graph( + run.compiled, + sources=run.prefix.sources, + store=run.prefix.store, + kernels=run.prefix.kernels, + resume="require", + ) + finally: + sys.setprofile(previous) + assert warm_calls == [] + assert all(node.hit for node in warm.nodes.values()) + assert warm.key == run.manifest.key + population._same_frame(projected, warm.population(version)) + # Cached graph output is not source admission. Requalify the retained + # preparation independently, as the country runner must on every replay. + current = qualifier.qualify_current_survey_geography(run.prefix.preparation) + pd.testing.assert_frame_equal(current.household, actual, check_exact=True) + assert current.receipt == run.qualified.receipt + + +@pytest.mark.parametrize( + "change,reason", + ( + ("preparation_digest", "DECLARATION"), + ("projection_digest", "PROJECTION_RECEIPT"), + ("artifact_payload", "ARTIFACT_BINDING"), + ("artifact_type", "ARTIFACT_BINDING"), + ("rewrite_declaration", "DECLARATION"), + ("source_roster", "CONTEXT_ROSTER"), + ), +) +def test_mismatched_declaration_or_typed_evidence_refuses( + known_run, tmp_path, change, reason +): + run = known_run + context = _detached_context(run.context) + if change in {"preparation_digest", "projection_digest"}: + field = ( + "preparation_sha256" + if change == "preparation_digest" + else "projection_receipt_sha256" + ) + node = replace(context.node, params={**context.node.params, field: "f" * 64}) + context = replace(context, node=node, params=node.params) + elif change in {"artifact_payload", "artifact_type"}: + artifact = context.artifacts["preparation"] + artifact = ( + replace(artifact, payload=artifact.payload + b" ") + if change == "artifact_payload" + else replace( + artifact, type=ArtifactType(population.PREPARATION_TYPE.name, 99) + ) + ) + context = replace(context, artifacts={"preparation": artifact}) + elif change == "rewrite_declaration": + first, *rest = context.node.outputs + context = replace( + context, + node=replace(context.node, outputs=(replace(first, rewrite=True), *rest)), + ) + else: + context = replace(context, sources={"undeclared": tmp_path}) + with pytest.raises(ValueError, match=reason): + run.kernel.run(context) + + +def test_same_household_ids_with_wrong_native_origin_refuse(known_run): + run = known_run + context = _detached_context(run.context) + table = context.tables["household"] + column = population.spine_source_id_column("household") + table.loc[table.index[0], column] += 1 + with pytest.raises(ValueError, match="HOUSEHOLD_ORIGIN"): + run.kernel.run(context) + + +def test_returned_series_are_detached_and_do_not_authorize_reuse(known_run): + run = known_run + before = source._frame_identity(run.prefix.preparation.frame) + result = run.kernel.run(_detached_context(run.context)) + column = ("household", "survey_observed_state") + result.columns[column].iloc[0] = "99" + result.receipt["source_projection"]["households"] = 999 + fresh = run.kernel.run(_detached_context(run.context)) + assert fresh.receipt["source_projection"]["households"] == 6 + for name in graph.OUTPUT_COLUMNS: + pd.testing.assert_series_equal( + fresh.columns[("household", name)], + run.qualified.household[name], + check_exact=True, + ) + assert source._frame_identity(run.prefix.preparation.frame) == before + assert ( + fresh.frame + is fresh.keep + is fresh.expand + is fresh.weights + is fresh.strata + is None + ) + assert not fresh.artifacts + + +def _assert_final_mutation_refuses(run, change): + context = _detached_context(run.context) + state = source._ISSUED[id(run.prefix.preparation)][2] + fired = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_code + is source.AuthenticatedSurveyPopulationPreparation._checked.__code__ + and frame.f_back.f_code is run.kernel.run.__code__ + and "result" in frame.f_back.f_locals + and not fired + ): + fired.append(True) + result = frame.f_back.f_locals["result"] + if change == "source_frame": + state.frame.table("household")["state_fips"] = 99 + elif change == "returned_column": + result.columns[("household", "survey_observed_state")].iloc[0] = "99" + else: + result.receipt["phase"] = "changed" + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError): + run.kernel.run(context) + finally: + sys.setprofile(previous) + assert fired == [True] + + +@pytest.mark.parametrize("change", ("returned_column", "receipt")) +def test_mutation_at_final_owner_return_refuses(known_run, change): + _assert_final_mutation_refuses(known_run, change) + + +def test_source_frame_mutation_at_final_owner_return_refuses(fresh_source_run): + _assert_final_mutation_refuses(fresh_source_run, "source_frame") + + +def test_changed_qualifier_return_cannot_hide_behind_unchanged_receipt( + known_run, +): + run = known_run + context = _detached_context(run.context) + fired = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_code is qualifier.qualify_current_survey_geography.__code__ + and frame.f_back.f_code is run.kernel.run.__code__ + and not fired + ): + fired.append(True) + assert isinstance(json.loads(arg.receipt)["projection_sha256"], str) + arg.household.loc[arg.household.index[0], "survey_observed_state"] = "99" + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError, match="QUALIFIED_PROJECTION"): + run.kernel.run(context) + finally: + sys.setprofile(previous) + assert fired == [True] diff --git a/packages/microcosm-build/tests/test_us_graph_current_survey_hours.py b/packages/microcosm-build/tests/test_us_graph_current_survey_hours.py new file mode 100644 index 000000000..3243a7b96 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_current_survey_hours.py @@ -0,0 +1,325 @@ +"""Real hours source owner and tiny invented receiving graph, without PUF fits.""" + +import sys +from dataclasses import replace +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_current_survey_health_coverage import _frame +from test_us_current_survey_hours_source import actual, receiving_people # noqa: F401 + +from microcosm.build.us_runtime import graph_current_survey_hours as graph +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + artifact_edges, + compile_graph, + run_graph, +) +from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys + + +@pytest.fixture(scope="module") +def case(actual, tmp_path_factory): # noqa: F811 + root = tmp_path_factory.mktemp("hours-fragment") + qualified = actual.values + people = receiving_people(qualified) + people["unrelated_value"] = 17.25 + receiving = _frame(people) + before = graph.source.source._frame_identity(receiving) + attachment_type = ArtifactType("invented.hours_parent", 1) + + class Receiving(KernelBase): + ref = "invented.hours_receiving@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return "a" * 64 + + def run(self, context): + return KernelResult(frame=receiving, artifacts={"attachment": b"parent"}) + + create = Node( + "invented.receiving", + Receiving.ref, + structural=StructuralDelta.CREATE, + sources=(graph.SOURCE_NAME,), + outputs=tuple( + Owned("person", c, graph.population_ops.token_for_dtype(people[c].dtype)) + for c in people + if c != "person_id" + ), + artifact_outputs=(ArtifactOutput("attachment", attachment_type),), + ) + after = ArtifactInput( + "housing_attachment", create.id, "attachment", attachment_type + ) + nodes = graph.hours_nodes( + qualified, receiving, receiving_version=create.id, after=after + ) + compiled = compile_graph( + Graph("us", (SourceRef(graph.SOURCE_NAME, "raw-bytes-v1"),), (create, *nodes)) + ) + + def current(): + assert graph.source.source._frame_identity(receiving) == before + + kernels = KernelRegistry() + kernels.register(Receiving()) + for kernel in graph.hours_kernels( + qualified, + receiving, + receiving_version=create.id, + after=after, + require_current=current, + ): + kernels.register(kernel) + path = root / "invented-source.txt" + path.write_bytes(b"invented receiving; hours use actual retained source owner") + store = ContentStore(root / "store") + args = dict(sources={graph.SOURCE_NAME: path}, store=store, kernels=kernels) + qualified.validate() + cold = run_graph(compiled, **args) + warm = run_graph(compiled, **args, resume="require") + qualified.validate() + current() + return SimpleNamespace( + qualified=qualified, + receiving=receiving, + nodes=nodes, + compiled=compiled, + cold=cold, + warm=warm, + store=store, + args=args, + current=current, + after=after, + version=create.id, + ) + + +def test_real_fragment_cold_required_preserves_clones_and_nonowned_values(case): + assert case.cold.key == case.warm.key + assert all(n.hit for n in case.warm.nodes.values()) + output = case.warm.population(case.version) + expected = graph.source.borrow_cloned_hours_columns( + case.qualified, case.receiving.person + ) + for (_entity, name), values in expected.items(): + pd.testing.assert_series_equal( + output.person.set_index("person_id")[name], values + ) + for entity in case.receiving.entities: + pd.testing.assert_frame_equal( + output.table(entity)[case.receiving.table(entity).columns], + case.receiving.table(entity), + check_exact=True, + ) + assert ( + output.weights_for("household").values.tolist() + == case.receiving.weights_for("household").values.tolist() + ) + + +def test_graph_hours_and_nullable_provenance_survive_development_storage( + case, tmp_path +): + from microcosm.build.us_runtime import native_survey_handoff as handoff + + frame = case.warm.population(case.version) + inventory = handoff.native_survey_input_inventory(frame) + hours = next(row for row in inventory if row["variable"] == graph.hours.TARGET) + assert hours["status"] == "present" and hours["missing_values"] == 0 + assert not hours["source_signal_verified"] and not hours["applicability_verified"] + report = { + "protocol": handoff.PROTOCOL, + "input_inventory": inventory, + "invented_storage_test": True, + "release_eligible": False, + } + output = tmp_path / "hours-checkpoint" + # A genuine hours fragment is insufficient to issue an enrichment owner. + # Exercise only storage; public handoff still requires the retained full host. + handoff._write_checkpoint(frame, report, output) + restored = handoff.load_native_survey_development_checkpoint(output) + assert not restored.owner_live_verified + assert restored.report == report + handoff.same_replayed_frame(frame, restored.frame) + for name in (graph.hours.TARGET, "hours_provenance", "hours_policy"): + pd.testing.assert_series_equal(frame.person[name], restored.frame.person[name]) + + +def artifacts_for(case, node): + inputs = case.warm.node(node.id).typed_artifacts["inputs"] + return { + edge.name: artifact_edges.value_from_descriptor( + case.store.load_bytes(inputs[edge.name]["key"]), inputs[edge.name] + ) + for edge in node.artifact_inputs + } + + +def test_replay_reconstructs_every_artifact_and_column_from_independent_owner(case): + current = graph.population_ops.Population.from_frame(case.receiving, case.version) + for node in case.nodes: + result = graph.hours_result( + case.qualified, node, artifacts_for(case, node), current.frame.person + ) + record = case.warm.node(node.id) + for name, expected in result.artifacts.items(): + descriptor = record.typed_artifacts["outputs"][name] + assert case.store.load_bytes(descriptor["key"]) == expected + current = graph.population_ops.patch(current, node, result) + pd.testing.assert_frame_equal( + current.frame.person, + case.warm.population(case.version).person, + check_exact=True, + ) + assert current.mass_ledger == case.warm.mass_ledger(case.version) + params = str([node.params for node in case.nodes]) + assert "2024HU" not in params and "PERIDNUM" not in params + assert graph.hours.AGE15_POLICY in params + assert graph.hours.UNDER15_POLICY in params + assert len(case.nodes[-1].inputs[0].columns) == 4 + assert set(case.nodes[-1].inputs[0].columns).isdisjoint( + {"unrelated_value", graph.hours.TARGET} + ) + + +@pytest.mark.parametrize("alias", ["hours_projection", "hours_asec", "hours_acs"]) +def test_attachment_refuses_altered_upstream_payload(case, alias): + node = case.nodes[-1] + artifacts = artifacts_for(case, node) + artifacts[alias] = replace(artifacts[alias], payload=b"{}") + with pytest.raises(ValueError, match="ARTIFACT_PAYLOAD"): + graph.hours_result(case.qualified, node, artifacts, case.receiving.person) + + +def kernels_for(case, qualified=None, callback=None): + return graph.hours_kernels( + case.qualified if qualified is None else qualified, + case.receiving, + receiving_version=case.version, + after=case.after, + require_current=case.current if callback is None else callback, + ) + + +def test_copied_or_revoked_owner_cannot_issue_kernels(case): + with pytest.raises(ValueError, match="PROJECTION_OBJECT_CHANGED"): + kernels_for(case, replace(case.qualified)) + callback = case.qualified._revalidate + try: + object.__setattr__(case.qualified, "_revalidate", None) + with pytest.raises(ValueError, match="RETAINED_OWNER_REQUIRED"): + kernels_for(case) + finally: + object.__setattr__(case.qualified, "_revalidate", callback) + case.qualified.validate() + + +def test_policy_identity_changes_source_and_every_downstream_key(case): + _, source_keys = _source_paths_and_keys( + case.compiled, case.args["sources"], case.store + ) + before, _ = _all_node_keys(case.compiled, case.args["kernels"], source_keys) + changed = replace( + case.nodes[0], + params={**case.nodes[0].params, "under15_policy": "different-explicit-policy"}, + ) + compiled = compile_graph( + replace( + case.compiled.graph, + nodes=(case.compiled.graph.nodes[0], changed, *case.nodes[1:]), + ) + ) + after, _ = _all_node_keys(compiled, case.args["kernels"], source_keys) + assert before[case.version] == after[case.version] + assert all(before[node.id] != after[node.id] for node in case.nodes) + + +def test_existing_hours_owner_refuses_before_attachment(case): + people = case.receiving.person.copy(deep=True) + people[graph.hours.TARGET] = 40.0 + with pytest.raises(ValueError, match="ATTACH_OWNERSHIP_COLLISION"): + graph.hours_nodes( + case.qualified, + _frame(people), + receiving_version=case.version, + after=case.after, + ) + + +def test_final_host_callback_cannot_change_hours_before_kernel_returns(case): + table = case.qualified.person_hours + pid = table.index[0] + previous = table.at[pid, graph.hours.TARGET] + calls = 0 + + def callback(): + nonlocal calls + calls += 1 + if calls == 2: + table.at[pid, graph.hours.TARGET] = previous + 1 + + kernel = kernels_for(case, callback=callback)[0] + try: + with pytest.raises(ValueError, match="FINAL_HOURS_CHANGED"): + kernel.run( + SimpleNamespace( + node=case.nodes[0], + artifacts=artifacts_for(case, case.nodes[0]), + tables={}, + ) + ) + finally: + table.at[pid, graph.hours.TARGET] = previous + assert calls == 2 + case.qualified.validate() + + +def test_final_source_io_cannot_mutate_receiving_owner_before_kernel_returns(case): + kernel = kernels_for(case)[0] + table = case.receiving.person + previous = table.loc[table.index[0], "unrelated_value"] + previous_profile = sys.getprofile() + calls = 0 + + def profile(frame, event, argument): + nonlocal calls + if event == "return" and frame.f_code is type(case.qualified).validate.__code__: + calls += 1 + if calls == 2: + table.loc[table.index[0], "unrelated_value"] = previous + 1 + + try: + sys.setprofile(profile) + with pytest.raises(AssertionError): + kernel.run( + SimpleNamespace( + node=case.nodes[0], + artifacts=artifacts_for(case, case.nodes[0]), + tables={}, + ) + ) + finally: + sys.setprofile(previous_profile) + table.loc[table.index[0], "unrelated_value"] = previous + assert calls == 2 + case.qualified.validate() + case.current() diff --git a/packages/microcosm-build/tests/test_us_graph_current_survey_immigration.py b/packages/microcosm-build/tests/test_us_graph_current_survey_immigration.py new file mode 100644 index 000000000..8195d3eaf --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_current_survey_immigration.py @@ -0,0 +1,267 @@ +"""Typed pair encoding and clone mapping; these pure tests issue no authority.""" + +import importlib.util +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_current_survey_immigration as graph +from microcosm.build.us_runtime import graph_us_survey_enrichment as host + + +def pairs(): + return pd.DataFrame( + { + "ssn_card_type": ["CITIZEN", "NONE"], + "immigration_status_str": ["CITIZEN", "UNDOCUMENTED"], + }, + index=pd.Index([2**53 + 1, 2**63 - 1], dtype="int64", name="person_id"), + dtype=object, + ) + + +def mapping_case(): + table = pairs() + originals = table.index.to_numpy() + origins = pd.DataFrame( + { + "source": ["asec", "acs"], + # Overlapping native IDs must stay distinct by combined original ID. + "native_person_id": np.array([2**53 + 7, 2**53 + 7], dtype="int64"), + "source_year": [2024, 2024], + "survey_year": [2025, 2024], + }, + index=table.index.copy(), + ) + provenance = graph.attachment.provenance + people = pd.DataFrame( + { + "person_id": np.array([201, 107, 301, 99], dtype="int64"), + provenance.support_source_id_column("person"): originals[[1, 0, 0, 1]], + provenance.support_clone_index_column("person"): np.array( + [0, 1, 0, 1], dtype="int64" + ), + provenance.spine_source_id_column("person"): np.repeat(2**53 + 7, 4).astype( + "int64" + ), + provenance.support_channel_column("person"): pd.array( + ["acs", "asec", "asec", "acs"], dtype="string" + ), + "unrelated": [1.5, 2.5, 3.5, 4.5], + } + ) + return origins, SimpleNamespace(person=people), table + + +def test_private_pair_encoding_is_exact_lossless_and_defensive(): + original = pairs() + before = original.copy(deep=True) + canonical = graph.canonical_pairs(original) + assert canonical is not original + assert all(dtype == graph.STRING for dtype in canonical.dtypes) + payload = graph.pair_bytes(original) + assert json.loads(payload)["person_id"] == [2**53 + 1, 2**63 - 1] + restored = graph.read_pairs(payload) + pd.testing.assert_frame_equal(restored, canonical) + pd.testing.assert_frame_equal(original, before) + canonical.iloc[0, 0] = "NONE" + pd.testing.assert_frame_equal(original, before) + + +@pytest.mark.parametrize("value", [None, pd.NA, 1.0, True, "UNKNOWN"]) +def test_invalid_pair_values_do_not_become_strings(value): + table = pairs() + table.iloc[0, 0] = value + with pytest.raises(ValueError, match="PAIR_DOMAIN"): + graph.pair_bytes(table) + + +@pytest.mark.parametrize( + "change", ["float", "duplicate", "unnamed", "reversed", "extra"] +) +def test_pair_axis_and_target_order_refuse(change): + table = pairs() + if change == "float": + table.index = table.index.astype("float64") + elif change == "duplicate": + table.index = pd.Index([1, 1], name="person_id") + elif change == "unnamed": + table.index.name = None + elif change == "reversed": + table = table.loc[:, list(reversed(graph.owner.OUTPUTS))] + else: + table["years_in_us"] = 5 + with pytest.raises(ValueError, match="PAIR_AXIS"): + graph.pair_bytes(table) + + +@pytest.mark.parametrize("value", [True, 1.0, 2**63, -(2**63) - 1, "001"]) +def test_private_decoder_does_not_coerce_coordinate_types(value): + document = json.loads(graph.pair_bytes(pairs())) + document["person_id"][0] = value + with pytest.raises(ValueError, match="PAIR_ENCODING"): + graph.read_pairs(graph.owner.source._encode(document)) + + +def test_clone_mapping_preserves_indivisible_pair_and_exact_source_coordinates(): + origins, receiving, table = mapping_case() + before = receiving.person.copy(deep=True) + original = table.copy(deep=True) + mapped = graph.attachment.attach_columns( + origins, receiving, graph.canonical_pairs(table) + ) + support = graph.attachment.provenance.support_source_id_column("person") + for name in graph.owner.OUTPUTS: + values = mapped["person", name] + assert values.dtype == graph.STRING + assert values.index.tolist() == before.person_id.tolist() + assert values.tolist() == table[name].reindex(before[support]).tolist() + pd.testing.assert_frame_equal(receiving.person, before) + pd.testing.assert_frame_equal(table, original) + + +@pytest.mark.parametrize( + "change", + [ + "missing", + "extra", + "duplicate_id", + "float_id", + "role", + "duplicate_role", + "original", + "native", + "channel", + "ssn_collision", + "status_collision", + "reordered_pairs", + ], +) +def test_clone_mapping_refuses_invalid_receiving_identity(change): + origins, receiving, table = mapping_case() + people = receiving.person + provenance = graph.attachment.provenance + if change == "missing": + receiving.person = people.iloc[:-1].copy() + elif change == "extra": + receiving.person = pd.concat([people, people.iloc[:1]]) + elif change == "duplicate_id": + people.loc[0, "person_id"] = people.loc[1, "person_id"] + elif change == "float_id": + people["person_id"] = people.person_id.astype("float64") + elif change == "role": + people.loc[0, provenance.support_clone_index_column("person")] = 2 + elif change == "duplicate_role": + people.loc[0, provenance.support_clone_index_column("person")] = 1 + elif change == "original": + people.loc[0, provenance.support_source_id_column("person")] = 42 + elif change == "native": + people.loc[0, provenance.spine_source_id_column("person")] = 17 + elif change == "channel": + people.loc[0, provenance.support_channel_column("person")] = "asec" + elif change == "ssn_collision": + people[graph.owner.OUTPUTS[0]] = "CITIZEN" + elif change == "status_collision": + people[graph.owner.OUTPUTS[1]] = "CITIZEN" + else: + table = table.iloc[::-1] + with pytest.raises(ValueError, match="ATTACH_|CLONE_"): + graph.attachment.attach_columns( + origins, receiving, graph.canonical_pairs(table) + ) + + +@pytest.mark.parametrize( + "value", [object(), graph.owner.CurrentSurveyImmigrationTransfer(None, None, b"{}")] +) +def test_unissued_transfer_refuses_without_source_io(value): + with pytest.raises(ValueError, match="ISSUED_TRANSFER_REQUIRED"): + graph.retained_entry(value) + with pytest.raises(ValueError, match="ISSUED_TRANSFER_REQUIRED"): + host.run_us_survey_enrichment(object(), immigration_transfer=value) + + +def test_copied_descriptive_owner_cannot_declare_nodes(): + value = graph.owner.CurrentSurveyImmigrationTransfer(None, None, b"{}") + with pytest.raises(ValueError, match="ISSUED_TRANSFER_REQUIRED"): + graph.immigration_nodes( + replace(value), object(), receiving_version="x", after=object() + ) + + +@pytest.mark.parametrize("name", ["_pure", "_live", "_modules", "_require", "validate"]) +def test_rebound_owner_entrypoint_refuses_before_invocation(monkeypatch, name): + container = ( + graph.owner.CurrentSurveyImmigrationTransfer + if name == "validate" + else graph.owner + ) + before = host._live() + invoked = [] + + def replacement(*args, **kwargs): + invoked.append(True) + raise AssertionError("A replaced validation body must not run") + + monkeypatch.setattr(container, name, replacement) + assert host._live() != before + with pytest.raises(ValueError, match="OWNER_IMPLEMENTATION_CHANGED"): + graph.retained_entry(object()) + with pytest.raises(ValueError, match="OWNER_IMPLEMENTATION_CHANGED"): + host.run_us_survey_enrichment(object(), immigration_transfer=object()) + assert not invoked + + +@pytest.mark.parametrize("name", ["_pure", "_live", "validate"]) +def test_mutated_owner_entrypoint_code_refuses_before_invocation(monkeypatch, name): + container = ( + graph.owner.CurrentSurveyImmigrationTransfer + if name == "validate" + else graph.owner + ) + + def replacement(*args, **kwargs): + raise AssertionError("An altered validation body must not run") + + monkeypatch.setattr(getattr(container, name), "__code__", replacement.__code__) + with pytest.raises(ValueError, match="OWNER_IMPLEMENTATION_CHANGED"): + graph.retained_entry(object()) + + +@pytest.mark.parametrize("name", ["_pure", "_live", "_modules", "_require", "validate"]) +def test_first_fragment_import_cannot_bless_changed_owner(monkeypatch, name): + container = ( + graph.owner.CurrentSurveyImmigrationTransfer + if name == "validate" + else graph.owner + ) + + def replacement(*args, **kwargs): + raise AssertionError("A replaced validation body must not run") + + monkeypatch.setattr(container, name, replacement) + # Load the maintained fragment fresh after changing an already-imported + # original owner. No registry entry or source authority is manufactured. + spec = importlib.util.spec_from_file_location( + graph.__name__ + "_guard_probe", graph.__file__ + ) + fresh = importlib.util.module_from_spec(spec) + spec.loader.exec_module(fresh) + with pytest.raises(ValueError, match="OWNER_IMPLEMENTATION_CHANGED"): + fresh.retained_entry(object()) + + +@pytest.mark.parametrize("spm", [False, True]) +def test_attachment_follows_existing_optional_terminal(spm): + edge = host._immigration_after_edge(spm) + assert edge.producer == ( + host.spm_graph.ATTACH_NODE if spm else host.hours_graph.ATTACH_NODE + ) + assert edge.artifact == "attachment" + assert edge.type == ( + host.spm_graph.ATTACHMENT_TYPE if spm else host.hours_graph.ATTACHMENT_TYPE + ) diff --git a/packages/microcosm-build/tests/test_us_graph_current_survey_spm.py b/packages/microcosm-build/tests/test_us_graph_current_survey_spm.py new file mode 100644 index 000000000..87bff3b0e --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_current_survey_spm.py @@ -0,0 +1,344 @@ +"""Pure invented SPM fragment checks; no source issuance, engines or PUF fits.""" + +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_survey_spm_projection import fixture as projection_fixture + +from microcosm.build.spm_input_contract import ROLE_INPUT, UNIVERSE_INPUT +from microcosm.build.us_runtime import graph_current_survey_spm as graph +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ArtifactInput, ArtifactType, ArtifactValue, NumericScope + + +def frame(view): + people = view.person.copy(deep=True) + tables = {"person": people} + for entity in US_SCHEMA.group_entities: + if hasattr(view, entity): + tables[entity] = getattr(view, entity).copy(deep=True) + else: + people[US_SCHEMA.membership_column(entity)] = people.person_id + tables[entity] = pd.DataFrame({entity + "_id": people.person_id}) + tables[entity] = ( + tables[entity].sort_values(entity + "_id").reset_index(drop=True) + ) + return Frame( + tables, + US_SCHEMA, + {"household": Weights(np.ones(len(tables["household"])), WeightKind.DESIGN)}, + ) + + +@pytest.fixture +def case(): + original, origins, roles, statuses, receiving = projection_fixture() + qualified = graph.source.QualifiedNativeSpmInputs( + frame(original), + origins, + roles, + statuses, + statuses.rename("status").to_frame(), + pd.DataFrame({"literal": ["01"]}, index=pd.Index([20], name="person_id")), + graph.source.source._encode( + { + "income_year": 2024, + "acs_survey_year": 2024, + "asec_survey_year": 2025, + "acs_profile": {"fixture": "invented; no source admission"}, + "asec_scope_policy": None, + } + ), + ) + receiving.person["unrelated_value"] = 17.25 + receiving = frame(receiving) + after = ArtifactInput( + "hours_attachment", + "invented.hours", + "attachment", + ArtifactType("test.hours", 1), + ) + nodes = graph.spm_nodes( + qualified, + receiving, + receiving_version="invented.receiving", + after=after, + outside_role_placeholder=False, + ) + return SimpleNamespace( + qualified=qualified, receiving=receiving, after=after, nodes=nodes + ) + + +def artifacts(case, node, produced): + return { + edge.name: ArtifactValue( + produced[edge.producer, edge.artifact], + edge.type, + "b" * 64, + "a" * 64, + NumericScope(), + ) + for edge in node.artifact_inputs + } + + +def results(case): + produced = {(case.after.producer, case.after.artifact): b"invented hours"} + values = [] + for node in case.nodes: + inputs = artifacts(case, node, produced) + result = graph.spm_result(case.qualified, node, inputs, case.receiving) + produced.update( + {(node.id, key): value for key, value in result.artifacts.items()} + ) + values.append((node, inputs, result)) + return values + + +def test_three_nodes_own_only_person_role_and_unit_scope(case): + assert [n.id for n in case.nodes] == [ + graph.SOURCE_NODE, + graph.PROJECT_NODE, + graph.ATTACH_NODE, + ] + assert all(n.population == "invented.receiving" for n in case.nodes) + assert not case.nodes[0].outputs and not case.nodes[1].outputs + assert {(o.entity, o.column, o.dtype) for o in case.nodes[2].outputs} == { + ("person", ROLE_INPUT, "bool"), + ("spm_unit", UNIVERSE_INPUT, "string"), + } + for node in case.nodes[1:]: + assert {s.entity for s in node.inputs} == {"person", "spm_unit", "household"} + assert all(len(s.columns) == 4 for s in node.inputs) + assert all("unrelated_value" not in s.columns for s in node.inputs) + assert all(2024 == n.params["income_year"] for n in case.nodes) + + +def test_mixed_grain_artifacts_and_patch_preserve_complete_population(case): + before = graph.source.source._frame_identity(case.receiving) + computed = results(case) + mapped = json.loads(computed[1][2].artifacts["projection"]) + assert set(mapped["columns"]) == {"person", "spm_unit"} + assert len(mapped["columns"]["person"]["data"]) == 10 + assert len(mapped["columns"]["spm_unit"]["data"]) == 6 + assert mapped["placeholder_person_ids"] == [20030, 10030] + source = json.loads(computed[0][2].artifacts["source"]) + assert source["roles"]["data"][-1]["role"] is None + expected = graph.projection.project_spm_inputs( + case.qualified.source_frame, + case.qualified.origins, + case.qualified.roles, + case.qualified.unit_status, + case.receiving, + source_year=2024, + year=2024, + outside_role_placeholder=False, + ) + attached = computed[-1][2] + for key, values in expected.columns.items(): + pd.testing.assert_series_equal(attached.columns[key], values) + population = graph.population_ops.Population.from_frame( + case.receiving, "invented.receiving" + ) + for node, _, result in computed: + population = graph.population_ops.patch(population, node, result) + for entity in case.receiving.entities: + pd.testing.assert_frame_equal( + population.frame.table(entity)[case.receiving.table(entity).columns], + case.receiving.table(entity), + check_exact=True, + ) + assert graph.source.source._frame_identity(case.receiving) == before + assert not attached.receipt["release_eligible"] + assert not attached.receipt["weights_changed"] + + +@pytest.mark.parametrize("alias", ["spm_source", "spm_projection"]) +def test_attachment_refuses_stale_or_foreign_payload(case, alias): + node, inputs, _ = results(case)[-1] + inputs[alias] = replace(inputs[alias], payload=b"{}") + with pytest.raises(ValueError, match="ARTIFACT_PAYLOAD"): + graph.spm_result(case.qualified, node, inputs, case.receiving) + + +def test_artifact_roster_and_nominal_types_are_checked(case): + node, inputs, _ = results(case)[-1] + with pytest.raises(ValueError, match="ARTIFACT_ROSTER"): + graph.spm_result(case.qualified, node, {}, case.receiving) + inputs["spm_source"] = replace( + inputs["spm_source"], type=ArtifactType("test.foreign", 1) + ) + with pytest.raises(ValueError, match="ARTIFACT_TYPE"): + graph.spm_result(case.qualified, node, inputs, case.receiving) + + +@pytest.mark.parametrize("entity", ["spm_unit", "household"]) +def test_membership_changes_refuse_before_attachment(case, entity): + node, inputs, _ = results(case)[-1] + people = case.receiving.person + link = "person_" + entity + "_id" + people.loc[people.index[0], link] = people[link].iloc[1] + with pytest.raises(ValueError, match="GROUP_MEMBERSHIP"): + graph.spm_result(case.qualified, node, inputs, case.receiving) + + +@pytest.mark.parametrize("name", [ROLE_INPUT, UNIVERSE_INPUT]) +def test_existing_columns_refuse_without_rewriting(case, name): + entity = "person" if name == ROLE_INPUT else "spm_unit" + case.receiving.table(entity)[name] = False if entity == "person" else "OUTSIDE" + with pytest.raises(ValueError, match="OWNERSHIP_COLLISION"): + graph.spm_nodes( + case.qualified, + case.receiving, + receiving_version="invented.receiving", + after=case.after, + outside_role_placeholder=False, + ) + + +@pytest.mark.parametrize( + "parameter,value", [("income_year", 2025), ("outside_role_placeholder", 0)] +) +def test_year_and_explicit_representation_are_bound(case, parameter, value): + node, inputs, _ = results(case)[-1] + node = replace(node, params={**node.params, parameter: value}) + with pytest.raises(ValueError, match="NODE_PARAMETERS"): + graph.spm_result(case.qualified, node, inputs, case.receiving) + + +def test_policy_receipt_and_each_source_grain_are_sealed(case): + node, inputs, _ = results(case)[-1] + original = graph.spm_seal(case.qualified) + case.qualified.unit_status.iloc[0] = "OUTSIDE" + assert graph.spm_seal(case.qualified) != original + object.__setattr__(case.qualified, "receipt", b"{}") + with pytest.raises(ValueError, match="SOURCE_PERIOD|NODE_PARAMETERS"): + graph.spm_result(case.qualified, node, inputs, case.receiving) + + +def test_unissued_values_cannot_create_production_kernels(case): + with pytest.raises(ValueError, match="RETAINED_OWNER_REQUIRED"): + graph.spm_kernels( + case.qualified, + case.receiving, + receiving_version="invented.receiving", + after=case.after, + outside_role_placeholder=False, + require_current=lambda: None, + require_context=lambda context: None, + ) + + +@pytest.mark.parametrize( + "changed", ["person", "spm_unit", "artifact", "receipt", "mutable_artifact"] +) +def test_final_validation_detects_output_mutation(case, changed): + # This is the pure final-output fence used by the kernel. No source-owner + # validation or issuance is replaced to exercise it. + result = results(case)[-1][2] + + def validate(): + if changed == "person": + values = result.columns["person", ROLE_INPUT] + values.iloc[0] = not values.iloc[0] + elif changed == "spm_unit": + result.columns["spm_unit", UNIVERSE_INPUT].iloc[0] = "OUTSIDE" + elif changed == "artifact": + result.artifacts["attachment"] = b"changed" + elif changed == "mutable_artifact": + result.artifacts["attachment"] = bytearray(result.artifacts["attachment"]) + else: + result.receipt["release_eligible"] = True + + with pytest.raises(ValueError, match="FINAL_RESULT_CHANGED"): + graph._validate_result(result, validate) + + +def test_final_validation_accepts_unchanged_result_after_callback(case): + result = results(case)[-1][2] + calls = [] + assert graph._validate_result(result, lambda: calls.append("validated")) is result + assert calls == ["validated"] + + +def test_receiving_context_preserves_all_three_declared_entity_tables(case): + node, inputs, _ = results(case)[-1] + context = SimpleNamespace( + node=node, + artifacts=inputs, + tables={ + e: case.receiving.table(e) for e in ("person", "spm_unit", "household") + }, + ) + calls = [] + view = graph._receiving_context( + context, case.nodes, node.kernel, lambda value: calls.append(value) + ) + assert calls == [context] + assert all(getattr(view, e) is table for e, table in context.tables.items()) + + +def test_host_callback_authenticates_producer_keys_before_result(case): + def require_context(context): + assert all(v.producer_key == "a" * 64 for v in context.artifacts.values()), ( + "FOREIGN_PRODUCER" + ) + + node, inputs, _ = results(case)[-1] + inputs["spm_source"] = replace(inputs["spm_source"], producer_key="c" * 64) + with pytest.raises(AssertionError, match="FOREIGN_PRODUCER"): + graph._receiving_context( + SimpleNamespace(node=node, artifacts=inputs, tables={}), + case.nodes, + node.kernel, + require_context, + ) + + +@pytest.mark.parametrize( + "name", ["ROLE_INPUT", "UNIVERSE_INPUT", "PROTOCOL", "SOURCE_NODE"] +) +def test_graph_global_declarations_are_part_of_retained_seal(case, monkeypatch, name): + before = graph.spm_seal(case.qualified) + monkeypatch.setattr(graph, name, "changed") + assert graph.spm_seal(case.qualified) != before + + +@pytest.mark.parametrize("name", ["source", "projection", "population_ops"]) +def test_imported_module_aliases_are_part_of_retained_seal(case, monkeypatch, name): + before = graph.spm_seal(case.qualified) + proxy = SimpleNamespace(**vars(getattr(graph, name))) + if name == "projection": + proxy.project_spm_inputs = lambda *args, **kwargs: None + monkeypatch.setattr(graph, name, proxy) + assert graph.spm_seal(case.qualified) != before + + +@pytest.mark.parametrize("name", ["SOURCE_TYPE", "PROJECTION_TYPE", "ATTACHMENT_TYPE"]) +def test_artifact_type_seal_is_detached_from_in_place_mutation(case, name): + before = graph.spm_seal(case.qualified) + artifact_type = getattr(graph, name) + version = artifact_type.schema_version + try: + object.__setattr__(artifact_type, "schema_version", version + 1) + assert graph.spm_seal(case.qualified) != before + finally: + object.__setattr__(artifact_type, "schema_version", version) + assert graph.spm_seal(case.qualified) == before + + +@pytest.mark.parametrize("name", ["SOURCE_TYPE", "PROJECTION_TYPE", "ATTACHMENT_TYPE"]) +def test_artifact_type_seal_refuses_boolean_schema_version(case, name): + artifact_type = getattr(graph, name) + version = artifact_type.schema_version + try: + object.__setattr__(artifact_type, "schema_version", True) + with pytest.raises(ValueError, match="ARTIFACT_TYPE_CONFIGURATION"): + graph.spm_seal(case.qualified) + finally: + object.__setattr__(artifact_type, "schema_version", version) diff --git a/packages/microcosm-build/tests/test_us_graph_current_survey_ss_completion.py b/packages/microcosm-build/tests/test_us_graph_current_survey_ss_completion.py new file mode 100644 index 000000000..14f8674d5 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_current_survey_ss_completion.py @@ -0,0 +1,409 @@ +"""Genuine invented source -> categorical artifact -> report graph and replay. + +No country model, full financial graph, actual survey records, canonical +beneficiary columns or claimed scientific/release qualification. +""" + +import hashlib +import sys +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from ss_report_source_fixture import ss_report_source_arguments + +from microcosm.build.us_runtime import graph_current_survey_ss_completion as graph +from microcosm.fit import categorical +from microcosm.graph import ( + ArtifactValue, + ContentStore, + Graph, + KernelContext, + Numeric, + NumericScope, + SourceRef, + compile_graph, + platform_fingerprint, + run_graph, +) +from microcosm.graph.keys import opaque_artifact_key + +values = graph.values +CONFIG = categorical.CategoricalConfig(max_iter=3, min_samples_leaf=1) + + +def compiled(boundary): + return compile_graph( + Graph( + "us", + sources=(SourceRef(graph.SOURCE_NAME, "frame-store"),), + nodes=boundary.nodes, + ) + ) + + +def probability_binding(manifest, store, q): + fit = manifest.node(graph.FIT_NODE) + metadata = graph.codec.decode_json( + store.load_bytes(fit.opaque_artifacts["model_metadata"]) + ) + probability = manifest.node(graph.PROBABILITY_NODE) + payload = store.load_bytes(probability.opaque_artifacts["probabilities"]) + return { + "model_sha256": hashlib.sha256( + store.load_bytes(fit.opaque_artifacts["model"]) + ).hexdigest(), + "model_producer_key": fit.key, + "training_id": metadata["training"]["training_id"], + "donor_source_sha256": graph.codec.sha(q.donor_projection), + "matrix_sha256": graph.codec.sha(q.matrix), + "matrix_producer_key": manifest.node(graph.MATRIX_NODE).key, + "source_sha256": graph.codec.sha(q.recipient_projection), + "source_producer_key": manifest.node(graph.SOURCE_NODE).key, + "classes": list(values.basis.COMPONENTS), + "probability_producer_key": probability.key, + "probability_sha256": graph.codec.sha(payload), + } + + +@pytest.fixture(scope="module") +def genuine_graph(tmp_path_factory): + root = tmp_path_factory.mktemp("ss-report-graph") + with pytest.MonkeyPatch.context() as patch: + arguments = ss_report_source_arguments(root, patch) + preparation = values.source.prepare_authenticated_survey_population(**arguments) + boundary = graph._ReportBoundary(preparation, seed=29, config=CONFIG) + yield root, boundary + + +def test_genuine_seven_node_report_cold_required_and_exact_source_preservation( + genuine_graph, +): + root, boundary = genuine_graph + boundary.validate() + q = boundary.qualified + pd.testing.assert_series_equal( + q.source_frame.person.age, q.full_source.asec_frame.person.age + ) + assert [(output.entity, output.column) for output in boundary.nodes[0].outputs] == [ + ("person", "age") + ] + seal = values.qualified_seal(q) + before = values.source._frame_identity(boundary.entry[2].frame) + store = ContentStore(root / "graph-store") + refs = { + graph.SOURCE_NAME: store.put_frame( + graph.codec.sha(q.donor_projection), q.source_frame + ) + } + registry = graph.ss_report_kernel_registry(boundary) + observed = {} + cold = run_graph( + compiled(boundary), + sources=refs, + store=store, + kernels=registry, + _population_observer=lambda name, population: observed.setdefault( + name, population + ), + ) + assert len(cold.nodes) == 7 and not any(n.hit for n in cold.nodes.values()) + donor = observed[graph.DONOR_NODE].frame + assert donor.person.person_id.tolist() == [105, 109, 110, 111] + assert tuple(sorted(donor.person[values.LABEL])) == tuple( + sorted(values.basis.COMPONENTS) + ) + assert donor.resolve_weights("person").kind.value == "design" + np.testing.assert_array_equal( + donor.resolve_weights("person").values, [2552.12] * 3 + [100.0] + ) + assert ( + 105 + not in q.originals.loc[ + q.originals.source.eq("asec"), "native_person_id" + ].tolist() + ) + payload = store.load_bytes(cold.node(graph.REPORT_NODE).opaque_artifacts["report"]) + expected = probability_binding(cold, store, q) + completed, probabilities = graph.read_report( + payload, qualified=q, expected_model_binding=expected + ) + _, requested = values._report_selection(q.originals) + assert set(q.originals.loc[requested, "source"]) == {"asec", "acs"} + assert len(probabilities) == requested.sum() > 1 + assert np.isfinite(probabilities.to_numpy()).all() + np.testing.assert_allclose(probabilities.sum(axis=1), 1, rtol=0, atol=1e-12) + known = q.originals.source_reporting_universe.to_numpy() + np.testing.assert_array_equal( + completed.loc[known].sum(axis=1), + q.originals.loc[known, "social_security_source_total"], + ) + assert values._bits(completed.loc[~requested].to_numpy()) == values._bits( + q.originals.loc[~requested, list(values.basis.COMPONENTS)].to_numpy() + ) + ambiguous_id = q.originals.index[ + q.originals.source.eq("asec") & q.originals.native_person_id.eq(107) + ][0] + assert probabilities.loc[ambiguous_id, "social_security_retirement"] > 0 + assert completed.loc[ambiguous_id, "social_security_retirement"] == 0 + np.testing.assert_allclose( + completed.loc[ambiguous_id, list(values.basis.COMPONENTS[1:])].to_numpy(), + probabilities.loc[ambiguous_id, list(values.basis.COMPONENTS[1:])].to_numpy() + / probabilities.loc[ambiguous_id, list(values.basis.COMPONENTS[1:])].sum() + * 3000, + rtol=0, + atol=1e-10, + ) + # No graph node writes canonical beneficiary fields or changes receiving data. + assert not any( + output.column in values.basis.COMPONENTS + for node in boundary.nodes + for output in node.outputs + ) + assert values.source._frame_identity(boundary.entry[2].frame) == before + boundary.validate() + # A hit-only run must not call the estimator; retain any supervising profile. + previous = sys.getprofile() + fits = [] + + def no_fit(frame, event, result): + if previous is not None: + previous(frame, event, result) + if event == "call" and frame.f_code is categorical.fit_categorical.__code__: + fits.append(True) + raise AssertionError("required replay entered a fit") + + try: + sys.setprofile(no_fit) + warm = run_graph( + compiled(boundary), + sources=refs, + store=store, + kernels=registry, + resume="require", + ) + finally: + sys.setprofile(previous) + assert not fits and len(warm.nodes) == 7 and all(n.hit for n in warm.nodes.values()) + assert cold.key == warm.key + assert ( + cold.node(graph.REPORT_NODE).opaque_artifacts + == warm.node(graph.REPORT_NODE).opaque_artifacts + ) + assert ( + store.load_bytes(warm.node(graph.REPORT_NODE).opaque_artifacts["report"]) + == payload + ) + assert values.qualified_seal(q) == seal + boundary.validate() + boundary.pure() + + +@pytest.mark.parametrize("change", ["basis", "matrix", "config"]) +def test_borrowed_boundary_refuses_mutation_before_cached_replay(genuine_graph, change): + _, boundary = genuine_graph + q, old_config = boundary.qualified, boundary.config + basis = q.originals.copy(deep=True) + matrix = q.matrix + try: + if change == "basis": + q.originals.iloc[ + 0, q.originals.columns.get_loc("social_security_source_total") + ] = 777 + elif change == "matrix": + boundary.qualified = replace(q, matrix=matrix + b" ") + else: + boundary.config = categorical.CategoricalConfig( + max_iter=4, min_samples_leaf=1 + ) + with pytest.raises(ValueError, match="QUALIFIED_OR_CONFIGURATION_CHANGED"): + boundary.pure() + finally: + q.originals.loc[:, :] = basis + boundary.qualified = q + boundary.config = old_config + boundary.pure() + + +@pytest.mark.parametrize("change", ["body", "binding", "trailing"]) +def test_report_decoder_refuses_changed_report_or_expected_binding( + genuine_graph, change +): + _, boundary = genuine_graph + q = boundary.qualified + p = pd.DataFrame( + 0.25, + index=q.recipient_features.index, + columns=values.basis.COMPONENTS, + dtype="float64", + ) + payload = graph.encode_report(q, p, {"fixture": "numeric-codec-only"}) + expected = {"fixture": "numeric-codec-only"} + if change == "binding": + expected = {"fixture": "changed"} + elif change == "trailing": + payload += b"x" + else: + payload = payload[:-1] + bytes([payload[-1] ^ 1]) + with pytest.raises(ValueError): + graph.read_report(payload, qualified=q, expected_model_binding=expected) + + +def test_empty_source_branch_needs_no_class_support_or_model(tmp_path): + with pytest.MonkeyPatch.context() as patch: + arguments = ss_report_source_arguments(tmp_path, patch, no_recipients=True) + preparation = values.source.prepare_authenticated_survey_population(**arguments) + boundary = graph._ReportBoundary(preparation, seed=29, config=CONFIG) + q = boundary.qualified + assert q.matrix is None and len(q.recipient_features) == 0 + assert q.evidence["resolved_positive_donors"] == 0 + assert set(q.evidence["class_design_weight_mass"].values()) == {0.0} + assert [node.id for node in boundary.nodes] == [ + graph.SOURCE_NODE, + graph.REPORT_NODE, + ] + previous = sys.getprofile() + fits = [] + + def no_fit(frame, event, result): + if previous is not None: + previous(frame, event, result) + if event == "call" and frame.f_code is categorical.fit_categorical.__code__: + fits.append(True) + raise AssertionError("empty branch entered a fit") + + boundary.validate() + store = ContentStore(tmp_path / "empty-store") + refs = { + graph.SOURCE_NAME: store.put_frame( + graph.codec.sha(q.donor_projection), q.source_frame + ) + } + try: + sys.setprofile(no_fit) + cold = run_graph( + compiled(boundary), + sources=refs, + store=store, + kernels=graph.ss_report_kernel_registry(boundary), + ) + warm = run_graph( + compiled(boundary), + sources=refs, + store=store, + kernels=graph.ss_report_kernel_registry(boundary), + resume="require", + ) + finally: + sys.setprofile(previous) + assert not fits and len(cold.nodes) == len(warm.nodes) == 2 + assert not any(n.hit for n in cold.nodes.values()) and all( + n.hit for n in warm.nodes.values() + ) + assert cold.key == warm.key + payload = store.load_bytes( + cold.node(graph.REPORT_NODE).opaque_artifacts["report"] + ) + assert payload == store.load_bytes( + warm.node(graph.REPORT_NODE).opaque_artifacts["report"] + ) + completed, probabilities = graph.read_report( + payload, qualified=q, expected_model_binding={} + ) + assert probabilities.empty + assert values._bits(completed.to_numpy()) == values._bits( + q.originals.loc[:, list(values.basis.COMPONENTS)].to_numpy() + ) + assert q.originals.social_security_source_total.isna().any() + assert q.originals.social_security_source_total.eq(0).any() + boundary.validate() + boundary.pure() + + +@pytest.mark.parametrize("mutate", [False, True]) +def test_late_boundary_callback_cannot_change_detached_kernel_columns( + genuine_graph, mutate +): + _, boundary = genuine_graph + q = boundary.qualified + context = columns_context(boundary) + previous = sys.getprofile() + visited = [] + + def callback(frame, event, result): + if previous is not None: + previous(frame, event, result) + caller = frame.f_back + if ( + event == "return" + and frame.f_code is graph._ReportBoundary.pure.__code__ + and caller is not None + and caller.f_code is graph._Kernel.run.__code__ + and "result" in caller.f_locals + and not visited + ): + visited.append(True) + if mutate: + caller.f_locals["result"].columns[("person", values.FEATURES[0])].iloc[ + 0 + ] += 1 + + try: + sys.setprofile(callback) + if mutate: + with pytest.raises(ValueError, match="FINAL_RESULT_CHANGED"): + graph.ColumnsKernel(boundary).run(context) + else: + result = graph.ColumnsKernel(boundary).run(context) + pd.testing.assert_series_equal( + result.columns[("person", values.FEATURES[0])], + q.donor_columns[values.FEATURES[0]], + ) + finally: + sys.setprofile(previous) + assert visited == [True] + boundary.pure() + + +def columns_context(boundary): + node = next(n for n in boundary.nodes if n.id == graph.COLUMNS_NODE) + q = boundary.qualified + # Direct kernel refusal probe. The real graph/replay test separately binds + # actual producer keys; this typed context confers no source authority. + producer = "a" * 64 + return KernelContext( + node, + {"person": q.source_frame.person[["person_id", "age"]].copy(deep=True)}, + {}, + pd.Series(dtype=str), + node.params, + np.random.default_rng(0), + artifacts={ + "donor_projection": ArtifactValue( + q.donor_projection, + graph.PROJECTION_TYPE, + opaque_artifact_key(producer, "donor_projection"), + producer, + NumericScope(Numeric.PLATFORM_BITWISE, platform=platform_fingerprint()), + ) + }, + ) + + +@pytest.mark.parametrize("defect", ["value", "dtype", "absent"]) +def test_model_columns_refuse_changed_declared_source_age(genuine_graph, defect): + _, boundary = genuine_graph + context = columns_context(boundary) + person = context.tables["person"].copy(deep=True) + if defect == "value": + person.loc[person.index[0], "age"] += 1 + elif defect == "dtype": + person["age"] = person.age.astype( + "float32" if person.age.dtype != np.dtype("float32") else "float64" + ) + else: + person = person.drop(columns="age") + with pytest.raises(ValueError, match="CONTEXT_SOURCE_AGE"): + graph.ColumnsKernel(boundary).run(replace(context, tables={"person": person})) + boundary.pure() diff --git a/packages/microcosm-build/tests/test_us_graph_current_survey_state.py b/packages/microcosm-build/tests/test_us_graph_current_survey_state.py new file mode 100644 index 000000000..3965eb47a --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_current_survey_state.py @@ -0,0 +1,382 @@ +"""Canonical state binding over genuine invented post-clone geography. + +Fixture legacy writers deliberately supply unqualified carried state values. +They exercise replacement only; the actual survey issuers and atomic geography +operators establish the original observed/derived inputs. No release is tested. +""" + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments +from test_us_graph_atomic_survey_population import _support_payload + +from microcosm.build.us_runtime import current_survey_geography as observed +from microcosm.build.us_runtime import graph_atomic_survey_population as parent +from microcosm.build.us_runtime import graph_current_survey_state as binding +from microcosm.calibrate.geography_constants import US_STATE_FIPS_TO_POSTAL +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactType, + ArtifactValue, + Capabilities, + Determinism, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + NumericScope, + Owned, + Slice, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import dtype_for_token + + +def _table(code="06", *, existing=None, dtype="int64"): + table = pd.DataFrame( + { + "household_id": np.array([10, 20], dtype="int64"), + "assigned_state_fips": pd.array( + [code, code], dtype=dtype_for_token("string") + ), + "survey_observed_state": pd.array( + [code, code], dtype=dtype_for_token("string") + ), + } + ) + if existing is not None: + table["state_fips"] = pd.array(existing, dtype=dtype) + return table + + +def _view(table): + return SimpleNamespace(table=lambda entity: table) + + +def _artifacts(node): + gate = canonical_json( + {"outcome": "pass", "scope": "atomic_geography_mapping_integrity"} + ) + return { + edge.name: ArtifactValue( + type=edge.type, + payload=gate if edge.name == "geography_validation" else b"host", + key="a" * 64, + producer_key="b" * 64, + numerics=NumericScope(Numeric.BITWISE), + ) + for edge in node.artifact_inputs + } + + +@pytest.mark.parametrize("code", tuple(US_STATE_FIPS_TO_POSTAL)) +def test_exact_closed_conversion_preserves_axis_and_inputs(code): + table = _table(code) + before = table.copy(deep=True) + result = binding.bind_state_fips(table) + assert result.tolist() == [int(code), int(code)] + assert result.dtype == np.dtype("int64") + assert result.index.tolist() == [10, 20] + assert result.index.name == "household_id" + pd.testing.assert_frame_equal(table, before, check_exact=True) + + +@pytest.mark.parametrize( + "token", [None, "", "6", "006", "6.0", " 06", "06 ", "00", "03", "72", "99", "０６"] +) +def test_invalid_or_unknown_states_never_default(token): + with pytest.raises(ValueError, match="STATE_DOMAIN"): + binding.bind_state_fips(_table(token)) + + +@pytest.mark.parametrize( + "dtype,value", [("int64", 6), ("float64", 6.0), ("bool", True), ("object", "06")] +) +def test_noncanonical_assigned_storage_refuses(dtype, value): + table = _table() + table["assigned_state_fips"] = pd.Series([value, value], dtype=dtype) + with pytest.raises(ValueError, match="STATE_DOMAIN"): + binding.bind_state_fips(table) + + +@pytest.mark.parametrize( + "dtype,existing", [("int64", [6, 6]), ("int64", [99, 99]), ("Int64", [None, 99])] +) +def test_carried_canonical_values_are_replaced_not_qualified(dtype, existing): + table = _table(existing=existing, dtype=dtype) + before = table.copy(deep=True) + result = binding.bind_state_fips(table) + assert result.tolist() == [6, 6] + assert str(result.dtype) == dtype + node = binding.state_binding_node(_view(table), population="existing-clones") + assert node.outputs == (Owned("household", "state_fips", dtype, rewrite=True),) + pd.testing.assert_frame_equal(table, before, check_exact=True) + + +@pytest.mark.parametrize("dtype", ["float64", "string", "bool"]) +def test_unsupported_incumbent_dtype_refuses(dtype): + table = _table(existing=[6, 6], dtype=dtype) + with pytest.raises(ValueError, match="INCUMBENT_DTYPE"): + binding.bind_state_fips(table) + + +def test_observed_assigned_mismatch_and_axis_mutation_refuse(): + table = _table() + table.loc[1, "survey_observed_state"] = "36" + with pytest.raises(ValueError, match="OBSERVED_STATE_MISMATCH"): + binding.bind_state_fips(table) + table = _table() + table.loc[1, "household_id"] = 10 + with pytest.raises(ValueError, match="HOUSEHOLD_AXIS"): + binding.bind_state_fips(table) + + +@pytest.mark.parametrize( + "change", + [ + "gate_fail", + "gate_type", + "gate_scope", + "extra_artifact", + "output", + "missing_edge", + ], +) +def test_declared_output_and_typed_gate_mutations_refuse(change): + table = _table() + node = binding.state_binding_node(_view(table), population="existing-clones") + artifacts = _artifacts(node) + if change in ("gate_fail", "gate_scope"): + document = json.loads(artifacts["geography_validation"].payload) + document["outcome" if change == "gate_fail" else "scope"] = "wrong" + artifacts["geography_validation"] = replace( + artifacts["geography_validation"], payload=canonical_json(document) + ) + elif change == "gate_type": + artifacts["geography_validation"] = replace( + artifacts["geography_validation"], type=ArtifactType("wrong", 1) + ) + elif change == "extra_artifact": + artifacts["extra"] = artifacts["geography_validation"] + elif change == "output": + node = replace(node, outputs=(Owned("household", "county_fips", "int64"),)) + else: + node = replace(node, artifact_inputs=()) + with pytest.raises(ValueError): + binding.state_binding_result(node, table, artifacts) + + +def test_optional_host_edge_is_visible_and_no_authority_is_issued(): + table = _table() + edge = ArtifactInput( + "previous", "host.attach", "attachment", ArtifactType("host", 1) + ) + node = binding.state_binding_node( + _view(table), population="existing-clones", after=edge + ) + result = binding.state_binding_result(node, table, _artifacts(node)) + assert node.structural is StructuralDelta.NONE and node.weights is None + assert node.inputs == (Slice("household", binding.INPUTS),) + assert node.artifact_inputs[1] == edge + assert result.frame is result.keep is result.weights is None + assert set(result.columns) == {("household", "state_fips")} + assert result.receipt["new_geography_assignment"] is False + assert result.receipt["source_admission_issued"] is False + assert result.receipt["release_eligible"] is False + + +@pytest.fixture(scope="module") +def genuine_atomic(tmp_path_factory): + root = tmp_path_factory.mktemp("canonical-state-invented") + with pytest.MonkeyPatch.context() as patch: + arguments = _demographic_arguments(root, patch, unknown=False) + payload, identities = _support_payload() + support = root / "invented-support.npz" + support.write_bytes(payload) + config = parent.reconstruction.AtomicSurveyReconstruction( + support_path=str(support), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(identities.items())), + seed=17, + ) + run = parent.run_atomic_survey_population( + **arguments, + store_root=root / "store", + geography_config=config, + return_values=True, + ) + yield SimpleNamespace(run=run, arguments=arguments) + parent.survey._source_owner().verify_survey_population_preparation( + run.preparation + ) + + +class _FixtureLegacyState(KernelBase): + ref = "fixture.legacy_state@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.PLATFORM_BITWISE + ) + + def run(self, context): + table = context.tables["household"] + values = binding.bind_state_fips(table) + if context.params["kind"] != "same": + values[:] = 99 + values = values.astype(context.node.outputs[0].dtype) + if context.params["kind"] == "nullable": + values.iloc[0] = pd.NA + return KernelResult(columns={("household", "state_fips"): values}) + + +class _FixtureKeepAll(KernelBase): + ref = "fixture.state_keep_all@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.FILTER + ) + + def run(self, context): + return KernelResult( + keep=pd.Series(True, index=context.tables["person"].person_id, dtype="bool") + ) + + +def _with_households(frame, table): + return Frame( + { + entity: table if entity == "household" else frame.table(entity) + for entity in frame.entities + }, + frame.schema, + dict(frame._weights), + frame.strata, + metadata=frame.metadata, + mass_log=frame.mass_log, + ) + + +@pytest.mark.parametrize("incumbent", [None, "same", "different"]) +def test_genuine_atomic_clone_binding_and_required_replay(genuine_atomic, incumbent): + run = genuine_atomic.run + before = run.geography_population + original_stamp = parent.reconstruction._population_stamp(before) + observed_before = observed.qualify_current_survey_geography(run.preparation) + registry = KernelRegistry() + for kernel in run.kernels.as_mapping().values(): + registry.register(kernel) + kernel = binding.CurrentSurveyStateKernel() + registry.register(kernel) + receiving = before.frame + population = before.version + added = () + if incumbent is not None: + table = receiving.table("household").copy(deep=True) + dtype = "Int64" if incumbent == "nullable" else "int64" + table["state_fips"] = pd.array([99] * len(table), dtype=dtype) + receiving = _with_households(receiving, table) + legacy = Node( + "fixture.legacy_state", + _FixtureLegacyState.ref, + population=population, + inputs=(Slice("household", binding.INPUTS),), + outputs=( + Owned( + "household", + "state_fips", + dtype, + rewrite="state_fips" in before.frame.table("household"), + ), + ), + params={"kind": incumbent}, + ) + carrier = Node( + "fixture.state_carrier", + _FixtureKeepAll.ref, + structural=StructuralDelta.FILTER, + base=population, + inputs=(Slice("person", ("person_support_clone_index",)),), + ) + added = (legacy, carrier) + population = carrier.id + registry.register(_FixtureLegacyState()) + registry.register(_FixtureKeepAll()) + node = binding.state_binding_node(receiving, population=population) + compiled = compile_graph( + replace(run.compiled.graph, nodes=(*run.compiled.graph.nodes, *added, node)) + ) + snapshots = {} + manifest = run_graph( + compiled, + sources=run.sources, + store=run.store, + kernels=registry, + _population_observer=lambda name, value: snapshots.setdefault(name, value), + ) + base = snapshots["geography.gate" if incumbent is None else population] + completed = manifest.population(population) + expected = binding.bind_state_fips(base.frame.table("household")) + actual = completed.table("household").set_index("household_id").state_fips + pd.testing.assert_series_equal(actual, expected, check_exact=True) + for entity in completed.entities: + columns = [ + name + for name in base.frame.table(entity) + if not (entity == "household" and name == "state_fips") + ] + pd.testing.assert_frame_equal( + completed.table(entity)[columns], + base.frame.table(entity)[columns], + check_exact=True, + ) + np.testing.assert_array_equal( + completed.weights_for("household").values, + base.frame.weights_for("household").values, + ) + assert ( + completed.weights_for("household").kind + == base.frame.weights_for("household").kind + ) + assert manifest.mass_ledger(population) == base.mass_ledger + assert len(completed.table("household")) == 2 * len( + run.allocated_population.frame.table("household") + ) + assert "geography.gate" in compiled.predecessors[node.id] + assert manifest.node(node.id).receipt["release_eligible"] is False + warm = run_graph( + compiled, + sources=run.sources, + store=run.store, + kernels=registry, + resume="require", + ) + assert all(record.hit for record in warm.nodes.values()) + assert warm.key == manifest.key + parent.survey._same_frame(completed, warm.population(population)) + fresh = observed.qualify_current_survey_geography(run.preparation) + pd.testing.assert_frame_equal( + fresh.household, observed_before.household, check_exact=True + ) + assert fresh.receipt == observed_before.receipt + assert parent.reconstruction._population_stamp(before) == original_stamp + + +def test_source_member_mutation_remains_a_host_refusal(tmp_path, monkeypatch): + """Pure binding never replaces the host's live source requalification.""" + arguments = _demographic_arguments(tmp_path, monkeypatch, unknown=False) + preparation = parent.survey._source_owner().prepare_authenticated_survey_population( + **arguments + ) + path = arguments["source_dir"] / "asec" / "hhpub25.csv" + path.write_bytes(path.read_bytes() + b"\n") + with pytest.raises(ValueError): + observed.qualify_current_survey_geography(preparation) diff --git a/packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py b/packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py new file mode 100644 index 000000000..7f69dec3f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py @@ -0,0 +1,473 @@ +"""Invented common-frame fiscal solve through existing grouped Adam and graph.""" + +import hashlib +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_fiscal_measurement import InventedSource, context, declaration, frame + +from microcosm.build.us_runtime import graph_fiscal_dense_calibration as stage +from microcosm.build.us_runtime import graph_fiscal_measurement as measurement +from microcosm.calibrate import TargetRegistry, group_bounds +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactOutput, + ArtifactValue, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + NodeRejectedError, + Numeric, + NumericScope, + Owned, + Slice, + SourceRef, + StoreMissError, + StructuralDelta, + WeightTransition, + compile_graph, + run_graph, +) +from microcosm.graph.keys import opaque_artifact_key + + +def setup(*, incoming=None, row_upper=None, ids=None): + value = frame() + if ids is not None: + old = value.table("household").household_id.to_numpy().copy() + mapping = dict(zip(old.tolist(), ids.tolist(), strict=True)) + value.table("household")["household_id"] = ids + value.table("person")["person_household_id"] = np.array( + [mapping[v] for v in value.person.person_household_id], dtype=ids.dtype + ) + incoming = np.array([1.0, 2.0, 3.0] if incoming is None else incoming) + value = Frame( + {e: value.table(e) for e in value.entities}, + value.schema, + {"household": Weights(incoming, WeightKind.IMPORTANCE)}, + ) + args = declaration() + # Exact common-frame targets correspond to desired weights [2, 2, 4]. + args["registry"] = TargetRegistry( + [ + replace(s, value=v) + for s, v in zip(args["registry"], [10, 6, 4, 4, 2, 4], strict=True) + ], + country="us", + ) + if incoming[1] == 0: + args["registry"] = TargetRegistry( + [ + replace(s, value=v) + for s, v in zip(args["registry"], [8, 4, 4, 4, 0, 4], strict=True) + ], + country="us", + ) + mc = context(args, value) + measured = measurement.FiscalMeasurementKernel().run(mc).artifacts["measurement"] + hh_ids = value.table("household").household_id.to_numpy() + groups = group_bounds.GroupedUpperBounds(hh_ids, np.array([0, 0, 1]), [6.0, 5.0]) + bounds = stage.encode_fiscal_calibration_bounds( + population=mc.node.population, + household_ids=hh_ids, + original_design_weights=Weights(np.array([8.0, 9.0, 10.0]), WeightKind.DESIGN), + incoming_weights=value.weights_for("household"), + grouped_upper_bounds=groups, + row_upper=np.array([6.0, 6.0, 5.0] if row_upper is None else row_upper), + budget_sha256=hashlib.sha256(b"invented budget; not an issuer").hexdigest(), + ) + node = stage.fiscal_dense_calibration_node( + measurement_node=mc.node, + bounds_node="invented.bounds", + epochs=120, + learning_rate=0.03, + ) + artifacts = {} + for name, payload, type_, output in ( + ("measurement", measured, measurement.MEASUREMENT_TYPE, "measurement"), + ("bounds", bounds, stage.BOUNDS_TYPE, "numeric_bounds"), + ): + key = hashlib.sha256(name.encode()).hexdigest() + artifacts[name] = ArtifactValue( + payload, + type_, + opaque_artifact_key(key, output), + key, + NumericScope(Numeric.BITWISE), + ) + return value, mc, replace(mc, node=node, params=node.params, artifacts=artifacts) + + +def test_real_grouped_solver_rebuilds_exact_matrix_and_residuals(monkeypatch): + value, mc, c = setup() + captured = [] + original_calibrate = stage.calibrate + + def record(*args, **kwargs): + result = original_calibrate(*args, **kwargs) + captured.append(result) + return result + + monkeypatch.setattr(stage, "calibrate", record) + result = stage.FiscalDenseCalibrationKernel().run(c) + solved = captured[0] + expected = measurement.decode_fiscal_measurement(c.artifacts["measurement"].payload) + np.testing.assert_array_equal( + solved.problem.matrix.toarray(), expected.matrix.toarray() + ) + assert solved.problem.names == tuple( + s.to_target().row_name for s in expected.registry + ) + assert solved.problem.target_vector.tobytes() == expected.target_values.tobytes() + assert solved.final_loss < solved.initial_loss + assert result.weights.kind is WeightKind.CALIBRATED + assert result.frame is None and not result.columns + assert result.receipt["release_eligible"] is False + assert result.receipt["source_admission"] == "required_from_complete_parent_owner" + diagnostic = json.loads(result.artifacts["diagnostics"]) + assert diagnostic["schema_version"] == 8 + origin = json.loads(result.artifacts["origin_diagnostics"]) + achieved = expected.matrix @ result.weights.values + for row, actual in zip(diagnostic["targets"], achieved, strict=True): + assert row["final_estimate"] == actual + assert origin["accepted_weights"] == measurement._array(result.weights.values) + assert origin["group_totals"] == measurement._array( + np.array([result.weights.values[:2].sum(), result.weights.values[2]]) + ) + assert origin["fixed_zero_rows"] == 0 + assert diagnostic["build"]["matrix_matches_measurement"] is True + # The direct kernel receives observations only, never a writable parent. + for entity in value.entities: + if entity in mc.tables: + pd.testing.assert_frame_equal(c.tables[entity], mc.tables[entity]) + assert ( + value.weights_for("household").values.tobytes() + == np.array([1.0, 2.0, 3.0]).tobytes() + ) + + +def test_uint64_order_original_design_and_strict_zero_support(): + ids = np.array([2**63, 2**63 + 1, 2**64 - 1], dtype=np.uint64) + _, _, c = setup(incoming=[1.0, -0.0, 3.0], ids=ids) + bounds = stage.verify_fiscal_calibration_bounds( + c.artifacts["bounds"].payload, + household_ids=ids, + incoming_weights=c.weights["household"], + original_design_weights=Weights(np.array([8.0, 9.0, 10.0]), WeightKind.DESIGN), + ) + assert bounds.household_ids.dtype == ids.dtype + assert ( + bounds.original_design.values.tobytes() == np.array([8.0, 9.0, 10.0]).tobytes() + ) + output = stage.FiscalDenseCalibrationKernel().run(c) + assert output.weights.values[1:2].tobytes() == np.array([-0.0]).tobytes() + assert np.all(output.weights.values[[0, 2]] > 0) + stage.check_fiscal_calibration_weights(bounds, output.weights.values) + assert json.loads(output.artifacts["origin_diagnostics"])["fixed_zero_rows"] == 1 + with pytest.raises(ValueError, match="DESIGN_ANCHOR"): + stage.verify_fiscal_calibration_bounds( + c.artifacts["bounds"].payload, + household_ids=ids, + incoming_weights=c.weights["household"], + original_design_weights=Weights( + np.array([8.0, 8.0, 10.0]), WeightKind.DESIGN + ), + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("epochs", 0), + ("epochs", True), + ("epochs", 1.5), + ("learning_rate", float("nan")), + ("learning_rate", float("inf")), + ("learning_rate", 0.0), + ("learning_rate", True), + ], +) +def test_solver_settings_refuse_nonfinite_or_undeclared_modes(field, value): + _, mc, _ = setup() + with pytest.raises(ValueError): + stage.fiscal_dense_calibration_node( + measurement_node=mc.node, bounds_node="bounds", **{field: value} + ) + + +def test_misaligned_measurement_bounds_and_calibrated_parent_refuse_before_solve( + monkeypatch, +): + _, _, c = setup() + + def forbidden(*a, **kw): + pytest.fail("solver ran before admission") + + monkeypatch.setattr(stage, "calibrate", forbidden) + with pytest.raises(ValueError, match="IMPORTANCE"): + stage.FiscalDenseCalibrationKernel().run( + replace( + c, + weights={ + "household": Weights( + np.array([1.0, 2.0, 3.0]), WeightKind.CALIBRATED + ) + }, + ) + ) + changed = dict(c.tables) + changed["person"] = c.tables["person"].copy() + changed["person"]["amount"] += 1 + with pytest.raises(ValueError, match="PROJECTION_BINDING"): + stage.FiscalDenseCalibrationKernel().run(replace(c, tables=changed)) + raw = json.loads(c.artifacts["bounds"].payload) + raw["household_ids"] = measurement._array(np.array([30, 20, 10], dtype=np.int64)) + artifacts = dict(c.artifacts) + artifacts["bounds"] = replace( + artifacts["bounds"], payload=measurement.canonical_json(raw) + ) + with pytest.raises(ValueError, match="HOUSEHOLD_ALIGNMENT"): + stage.FiscalDenseCalibrationKernel().run(replace(c, artifacts=artifacts)) + + +def test_group_and_row_caps_refuse_without_clipping(): + _, _, c = setup(row_upper=[1.0, 6.0, 5.0]) + with pytest.raises(ValueError, match="ROW_REFERENCE_CAP"): + stage.FiscalDenseCalibrationKernel().run(c) + bounds = stage.decode_fiscal_calibration_bounds(c.artifacts["bounds"].payload) + with pytest.raises(ValueError, match="frozen absolute bounds"): + stage.check_fiscal_calibration_weights(bounds, np.array([5.0, 2.0, 3.0])) + + +@pytest.mark.parametrize( + "change,reason", + [ + ("names", "TARGET_ALIGNMENT"), + ("matrix", "MATRIX_IDENTITY"), + ("diagnostic", "DIAGNOSTIC_ESTIMATES"), + ], +) +def test_solver_return_with_wrong_matrix_or_target_order_is_refused( + monkeypatch, change, reason +): + _, _, c = setup() + actual = stage.calibrate + + def changed(*a, **kw): + r = actual(*a, **kw) + if change == "names": + return replace( + r, problem=replace(r.problem, names=tuple(reversed(r.problem.names))) + ) + if change == "matrix": + matrix = r.problem.matrix.copy() + matrix.data[0] += 1 + return replace(r, problem=replace(r.problem, matrix=matrix)) + return replace( + r, + diagnostics=( + replace(r.diagnostics[0], final_estimate=999.0), + *r.diagnostics[1:], + ), + ) + + monkeypatch.setattr(stage, "calibrate", changed) + with pytest.raises(ValueError, match=reason): + stage.FiscalDenseCalibrationKernel().run(c) + + +class InventedImportance(KernelBase): + ref = "test.fiscal_dense_importance@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def run(self, context): + return KernelResult( + weights=Weights(np.array(context.params["incoming"]), WeightKind.IMPORTANCE) + ) + + +class InventedBounds(KernelBase): + ref = "test.fiscal_dense_bounds@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context): + payload = context.sources["bounds"].read_bytes() + stage.decode_fiscal_calibration_bounds(payload) + return KernelResult(artifacts={"numeric_bounds": payload}) + + +def graph_fixture(tmp_path): + value, mc, c = setup(incoming=[1.0, 0.0, 3.0]) + value = Frame( + {e: value.table(e) for e in value.entities}, + value.schema, + {"household": Weights(np.array([8.0, 9.0, 10.0]), WeightKind.DESIGN)}, + ) + create = Node( + "invented.design", + InventedSource.ref, + sources=("fixture",), + structural=StructuralDelta.CREATE, + outputs=( + Owned("person", "amount", "float64"), + Owned("person", "keep", "bool"), + Owned("tax_unit", "tax_unit_amount", "float64"), + Owned("household", "state_fips", "int64"), + Owned("household", "congressional_district_geoid", "int64"), + ), + ) + importance = Node( + "invented", + InventedImportance.ref, + base=create.id, + structural=StructuralDelta.REWEIGHT, + inputs=(Slice("household", ("state_fips",)),), + weights=WeightTransition("household", "importance", mass="free"), + mass="free", + params={"incoming": (1.0, 0.0, 3.0)}, + ) + bounds = Node( + "invented.bounds", + InventedBounds.ref, + population=importance.id, + sources=("bounds",), + artifact_outputs=(ArtifactOutput("numeric_bounds", stage.BOUNDS_TYPE),), + ) + nodes = (create, importance, bounds, mc.node, c.node) + sources = (SourceRef("fixture", "frame-store"), SourceRef("bounds", "raw-bytes-v1")) + kernels = KernelRegistry() + for kernel in ( + InventedSource(), + InventedImportance(), + InventedBounds(), + measurement.FiscalMeasurementKernel(), + stage.FiscalDenseCalibrationKernel(), + ): + kernels.register(kernel) + store = ContentStore(tmp_path / "store") + path = store.put_frame( + hashlib.sha256(b"invented fiscal dense fixture").hexdigest(), value + ) + bound_path = tmp_path / "bounds.json" + bound_path.write_bytes(c.artifacts["bounds"].payload) + arguments = { + "store": store, + "kernels": kernels, + "sources": {"fixture": path, "bounds": bound_path}, + } + return value, sources, nodes, arguments + + +def test_real_graph_cold_required_replay_preserves_parent_and_original_anchors( + tmp_path, +): + original, sources, nodes, arguments = graph_fixture(tmp_path) + compiled = compile_graph(Graph("us", sources, nodes)) + anchors = {} + + def observe(node_id, population): + anchors[node_id] = population.design_weights["household"].tobytes() + + cold = run_graph(compiled, **arguments, _population_observer=observe) + cold_anchors = dict(anchors) + warm = run_graph( + compiled, **arguments, resume="require", _population_observer=observe + ) + assert cold.key == warm.key and all(n.store_hit for n in warm.nodes.values()) + before = warm.population("invented") + after = warm.population(nodes[-1].id) + # The graph performs precisely DESIGN -> IMPORTANCE -> CALIBRATED. + assert after.weights_for("household").kind is WeightKind.CALIBRATED + assert before.weights_for("household").kind is WeightKind.IMPORTANCE + for entity in original.entities: + pd.testing.assert_frame_equal(before.table(entity), original.table(entity)) + pd.testing.assert_frame_equal(after.table(entity), original.table(entity)) + assert after.weights_for("household").values[1] == 0 + assert ( + cold.population(nodes[-1].id).weights_for("household").values.tobytes() + == after.weights_for("household").values.tobytes() + ) + # Detached observer snapshots expose original anchors without owner authority. + assert anchors == cold_anchors + assert anchors[nodes[-1].id] == np.array([8.0, 9.0, 10.0]).tobytes() + payload = arguments["store"].load_bytes( + warm.node(nodes[-1].id).opaque_artifacts["origin_diagnostics"] + ) + assert json.loads(payload)["fixed_zero_rows"] == 1 + # A repeated CALIBRATED -> CALIBRATED stage is refused before another solve. + new_measure = replace(nodes[-2], id="repeat.measurement", population=nodes[-1].id) + repeated = stage.fiscal_dense_calibration_node( + measurement_node=new_measure, + bounds_node=nodes[2].id, + node_id="repeat.calibration", + ) + compiled_repeat = compile_graph( + Graph("us", sources, (*nodes, new_measure, repeated)) + ) + with pytest.raises( + NodeRejectedError, match="(?i)(importance|calibrated|transition|forward)" + ): + run_graph(compiled_repeat, **arguments) + + +def test_real_graph_replay_refuses_changed_bounds_and_target_declaration(tmp_path): + _, sources, nodes, arguments = graph_fixture(tmp_path) + compiled = compile_graph(Graph("us", sources, nodes)) + run_graph(compiled, **arguments) + changed_measure = measurement.fiscal_measurement_node(**declaration()) + changed_solve = stage.fiscal_dense_calibration_node( + measurement_node=changed_measure, bounds_node=nodes[2].id + ) + changed = compile_graph( + Graph("us", sources, (*nodes[:3], changed_measure, changed_solve)) + ) + with pytest.raises( + (NodeRejectedError, StoreMissError), match="(?i)(require|cache|missing)" + ): + run_graph(changed, **arguments, resume="require") + path = arguments["sources"]["bounds"] + raw = json.loads(path.read_bytes()) + raw["group_upper"] = measurement._array(np.array([5.5, 5.0])) + path.write_bytes(measurement.canonical_json(raw)) + with pytest.raises( + (NodeRejectedError, StoreMissError), match="(?i)(require|cache|missing)" + ): + run_graph(compiled, **arguments, resume="require") + + +def test_bound_codec_refuses_noncanonical_values_and_wrong_types(): + _, _, c = setup() + payload = c.artifacts["bounds"].payload + for field, value in ( + ("incoming", np.array([1.0, float("nan"), 3.0])), + ("group_indices", np.array([0, 0, 9], dtype=np.int64)), + ("original_design", np.array([8.0, 9.0])), + ("household_ids", np.array([10, 10, 30], dtype=np.int64)), + ): + raw = json.loads(payload) + raw[field] = measurement._array(value) + with pytest.raises(ValueError): + stage.decode_fiscal_calibration_bounds(measurement.canonical_json(raw)) + with pytest.raises(ValueError, match="CANONICAL"): + stage.decode_fiscal_calibration_bounds(payload + b"\n") + + +def test_implementation_identity_binds_existing_solver_and_target_math( + monkeypatch, tmp_path +): + # Point source identity at changed synthetic bytes without editing package source. + baseline = stage.FiscalDenseCalibrationKernel().implementation_hash() + changed = tmp_path / "changed_target.py" + changed.write_text("# invented changed target arithmetic\n") + monkeypatch.setattr(stage.target_module, "__file__", str(changed)) + assert stage.FiscalDenseCalibrationKernel().implementation_hash() != baseline diff --git a/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py b/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py new file mode 100644 index 000000000..3fe97facb --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py @@ -0,0 +1,650 @@ +"""Invented fiscal measures through the actual interpreter, CSR and graph store.""" + +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_fiscal_measurement as stage +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate import target as target_math +from microcosm.calibrate.hierarchy import ( + CalibrationHierarchy, + HierarchyCategory, + HierarchyGeography, + HierarchyNode, +) +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.frame import materialize as frame_materialize +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + NodeRejectedError, + Owned, + SourceRef, + StoreMissError, + StructuralDelta, + compile_graph, + load_source, + run_graph, + source_hash, +) +from microcosm.graph.canonical import canonical_json + + +def frame(): + tables = { + "person": pd.DataFrame( + { + "person_id": np.array([400, 100, 300, 200], dtype=np.int64), + **{ + f"person_{e}_id": np.array([30, 10, 20, 10], dtype=np.int64) + for e in US_SCHEMA.group_entities + }, + "amount": [40.0, 10.0, 30.0, 20.0], + "keep": [True, True, False, True], + } + ), + **{ + e: pd.DataFrame({f"{e}_id": np.array([10, 20, 30], dtype=np.int64)}) + for e in US_SCHEMA.group_entities + }, + } + tables["household"]["state_fips"] = np.array([6, 6, 36], dtype=np.int64) + tables["household"]["congressional_district_geoid"] = np.array( + [601, 602, 3601], dtype=np.int64 + ) + tables["tax_unit"]["tax_unit_amount"] = [100.0, 200.0, 300.0] + return Frame( + tables, + US_SCHEMA, + {"household": Weights(np.array([1.0, 2.0, 3.0]), WeightKind.DESIGN)}, + ) + + +def declaration(*, value_variable="person_count", entity="person", model_outputs=()): + definitions = ( + ("national", "0100000US", None, None), + ("state", "0400000US06", "state_fips", 6), + ("state", "0400000US36", "state_fips", 36), + ( + "congressional_district", + "5001900US0601", + "congressional_district_geoid", + 601, + ), + ( + "congressional_district", + "5001900US0602", + "congressional_district_geoid", + 602, + ), + ( + "congressional_district", + "5001900US3601", + "congressional_district_geoid", + 3601, + ), + ) + scopes = tuple( + stage.FiscalGeographyScope(HierarchyGeography(id_, id_, level), column, value) + for level, id_, column, value in definitions + ) + specs, bindings = [], {} + for i, scope in enumerate(scopes): + name = f"invented_{i}" + specs.append( + TargetSpec( + name, + entity, + 1.0, + f"measure_{i}", + period=2024, + source="Invented test; not an administrative observation", + family="invented", + metadata={"contract_target_id": name}, + hierarchy=CalibrationHierarchy( + HierarchyNode("test", "Test"), + HierarchyCategory("count", "Count", "test"), + scope.geography, + (), + HierarchyNode(name, name), + ), + ) + ) + bindings[name] = { + "bindings": {"policyengine": {"value_variable": value_variable}} + } + return { + "registry": TargetRegistry(specs, country="us"), + "population": "invented", + "input_columns": { + "person": ("amount", "keep"), + "tax_unit": ("tax_unit_amount",), + "household": ("state_fips", "congressional_district_geoid"), + }, + "contract_targets": bindings, + "advertised_scopes": scopes, + "geography_vintage": "invented CD119 mapping", + "period": 2024, + "model_outputs": model_outputs, + } + + +def context(arguments=None, value=None): + value = frame() if value is None else value + arguments = declaration() if arguments is None else arguments + node = stage.fiscal_measurement_node(**arguments) + tables = {} + for slice_ in node.inputs: + entity = slice_.entity + structural = [f"{entity}_id"] + if entity == "person": + structural += [f"person_{e}_id" for e in US_SCHEMA.group_entities] + tables[entity] = ( + value.table(entity) + .loc[:, list(dict.fromkeys([*structural, *slice_.columns]))] + .copy() + ) + return KernelContext( + node, + tables, + {"household": value.weights_for("household")}, + value.strata, + node.params, + np.random.default_rng(0), + ) + + +def test_actual_measurement_maps_person_counts_and_tax_units_to_households(): + original = frame() + c = context(value=original) + result = stage.FiscalMeasurementKernel().run(c) + payload = result.artifacts["measurement"] + measured = stage.verify_fiscal_measurement( + payload, context=c, expected_sha256=hashlib.sha256(payload).hexdigest() + ) + np.testing.assert_array_equal(measured.household_ids, [10, 20, 30]) + np.testing.assert_array_equal( + measured.matrix.toarray(), + [[2, 1, 1], [2, 1, 0], [0, 0, 1], [2, 0, 0], [0, 1, 0], [0, 0, 1]], + ) + np.testing.assert_array_equal( + measured.matrix.toarray()[3:].sum(axis=0), measured.matrix.toarray()[0] + ) + assert result.frame is None and result.weights is None and not result.columns + assert result.receipt["fit_performed"] is False + assert result.receipt["release_eligible"] is False + for entity in original.entities: + pd.testing.assert_frame_equal(original.table(entity), frame().table(entity)) + c = context(declaration(value_variable="tax_unit_amount", entity="tax_unit")) + values = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel().run(c).artifacts["measurement"] + ) + np.testing.assert_array_equal(values.matrix.toarray()[0], [100, 200, 300]) + + +def test_actual_shared_interpreter_sum_and_filter(): + arguments = declaration(value_variable="amount") + for target in arguments["contract_targets"].values(): + target["bindings"]["policyengine"] = { + "value_expression": "amount + amount", + "filters": [{"variable": "amount", "operator": ">", "value": 5}], + } + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel().run(context(arguments)).artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [60, 60, 80]) + + +def test_boolean_indicator_expression_is_an_arithmetic_sum(): + arguments = declaration() + for target in arguments["contract_targets"].values(): + target["bindings"]["policyengine"] = {"value_expression": "keep + keep"} + value = frame() + value.person["keep"] = [True, False, True, True] + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel() + .run(context(arguments, value)) + .artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [2, 2, 2]) + + +@pytest.mark.parametrize("kind", ["float", "nullable_boolean"]) +@pytest.mark.parametrize("location", ["binding", "target_filter"]) +def test_missing_consumed_predicate_is_refused(kind, location): + arguments, value = declaration(), frame() + variable = "amount" if kind == "float" else "keep" + if kind == "float": + value.person.loc[1, variable] = np.nan + predicate = {"variable": variable, "operator": ">", "value": 5} + else: + value.person[variable] = pd.Series([True, pd.NA, True, True], dtype="boolean") + predicate = {"variable": variable, "equals": True} + if location == "binding": + for binding in arguments["contract_targets"].values(): + binding["bindings"]["policyengine"]["filters"] = [predicate] + else: + arguments["registry"] = TargetRegistry( + [replace(s, filter=variable) for s in arguments["registry"]], country="us" + ) + with pytest.raises( + ValueError, match="MISSING_PREDICATE_INPUT:person\\." + variable + ): + stage.FiscalMeasurementKernel().run(context(arguments, value)) + + +@pytest.mark.parametrize("location", ["binding", "target_filter"]) +def test_known_false_predicates_and_unconsumed_missing_inputs_are_preserved(location): + arguments, value = declaration(), frame() + value.person["keep"] = [True, False, True, True] + value.person.loc[0, "amount"] = np.nan # This measure does not read amount. + if location == "binding": + for binding in arguments["contract_targets"].values(): + binding["bindings"]["policyengine"]["filters"] = [ + {"variable": "keep", "equals": True} + ] + else: + arguments["registry"] = TargetRegistry( + [replace(s, filter="keep") for s in arguments["registry"]], country="us" + ) + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel() + .run(context(arguments, value)) + .artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [1, 1, 1]) + + +def test_implementation_identity_includes_target_arithmetic_source( + tmp_path, monkeypatch +): + before = stage.FiscalMeasurementKernel().implementation_hash() + changed = tmp_path / "changed_target.py" + source = Path(target_math.__file__).read_text() + original = " return values * filter_mask\n" + assert source.count(original) == 1 + changed.write_text( + source.replace(original, " return 2.0 * values * filter_mask\n") + ) + monkeypatch.setattr(target_math, "__file__", str(changed)) + assert stage.FiscalMeasurementKernel().implementation_hash() != before + + +def test_implementation_identity_includes_engine_table_transform(tmp_path, monkeypatch): + assert stage.policyengine_us.engine_tables is frame_materialize.engine_tables + before = stage.FiscalMeasurementKernel().implementation_hash() + changed = tmp_path / "changed_engine_tables.py" + source = Path(frame_materialize.__file__).read_text() + original = " ).values\n" + assert source.count(original) == 1 + changed.write_text(source.replace(original, " ).values * 2.0\n")) + monkeypatch.setattr(frame_materialize, "__file__", str(changed)) + assert stage.FiscalMeasurementKernel().implementation_hash() != before + + +@pytest.mark.parametrize( + "change,reason", + [ + ("binding", "UNSUPPORTED_BINDING"), + ("coverage", "ADVERTISED_LEVELS"), + ("period", "TARGET_PERIOD_HIERARCHY"), + ("collision", "INPUT_MEASURE_COLLISION"), + ("unknown", "UNMATERIALIZED_TARGETS"), + ("empty_area", "EMPTY_ADVERTISED_GEOGRAPHY"), + ("zero_support", "UNSUPPORTED_TARGET"), + ("zero_weight", "UNSUPPORTED_TARGET"), + ], +) +def test_declared_surface_and_advertised_support_fail_closed(change, reason): + arguments, value = declaration(), frame() + if change == "binding": + arguments["contract_targets"]["invented_0"]["bindings"]["policyengine"][ + "kind" + ] = "input_substitution_counterfactual" + elif change == "coverage": + arguments["advertised_scopes"] = arguments["advertised_scopes"][:3] + elif change == "period": + arguments["period"] = 2025 + elif change == "collision": + arguments["input_columns"]["person"] += ("measure_0",) + elif change == "unknown": + arguments["contract_targets"]["invented_0"]["bindings"]["policyengine"][ + "value_variable" + ] = "not_produced" + elif change == "empty_area": + value.table("household").loc[2, "congressional_district_geoid"] = 3699 + elif change == "zero_support": + arguments["contract_targets"]["invented_5"]["bindings"]["policyengine"][ + "filters" + ] = [{"variable": "amount", "operator": ">", "value": 999}] + elif change == "zero_weight": + value = Frame( + {e: value.table(e) for e in value.entities}, + US_SCHEMA, + {"household": Weights(np.array([1.0, 2.0, 0.0]), WeightKind.DESIGN)}, + ) + with pytest.raises(ValueError, match=reason): + stage.FiscalMeasurementKernel().run(context(arguments, value)) + + +def test_artifact_digest_projection_and_order_tamper_refusals(): + c = context() + payload = stage.FiscalMeasurementKernel().run(c).artifacts["measurement"] + digest = hashlib.sha256(payload).hexdigest() + changed = json.loads(payload) + changed["data"]["hex"] = np.full(9, 99.0, dtype="