diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..324641f92 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Preserve exact unified-diff context markers in the archived measured experiment. +experiments/us-acs-code-memo-not-adopted-20260910.patch -whitespace diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f03b8573..71c21c387 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -168,6 +168,8 @@ jobs: # python3 (not bare `python`): guaranteed on the Ubuntu image, and this # step runs before any sync so it must hold with the stdlib alone. run: python3 tools/ci_test_groups.py --verify + - name: Check file-shard and workflow coverage without importing data tests + run: python3 -I -B -S packages/microcosm-build/tests/test_ci_test_groups.py - name: Verify the graph acceptance burndown is monotone # The charter's second process rule, executable: the number of red # properties in the microcosm-graph acceptance suite never rises @@ -186,7 +188,15 @@ jobs: fail-fast: false matrix: python-version: ["3.13", "3.14"] - group: [trade, spine-uk, rest] + lane: + - {"group": "trade", "shard": "1/1"} + - {"group": "spine-uk", "shard": "1/1"} + - {"group": "rest", "shard": "1/6"} + - {"group": "rest", "shard": "2/6"} + - {"group": "rest", "shard": "3/6"} + - {"group": "rest", "shard": "4/6"} + - {"group": "rest", "shard": "5/6"} + - {"group": "rest", "shard": "6/6"} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -194,9 +204,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Sync workspace without engines run: uv sync --all-packages --locked - - name: Engine-free tests (${{ matrix.group }}) + - name: Engine-free tests (${{ matrix.lane.group }} ${{ matrix.lane.shard }}) run: | - mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.group }}") + set -euo pipefail + selected="$(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.lane.group }}" --shard "${{ matrix.lane.shard }}")" + test -n "$selected" + mapfile -t files <<< "$selected" /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25 engine-shared: @@ -207,7 +220,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.13", "3.14"] + python-version: ["3.13", "3.14.4"] steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -217,9 +230,76 @@ jobs: run: uv sync --all-packages --locked --extra us --extra uk - name: Shared/spec engine tests run: | - mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list shared-spec) + set -euo pipefail + selected="$(uv run --no-sync python tools/ci_test_groups.py --list shared-spec)" + test -n "$selected" + mapfile -t files <<< "$selected" /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25 + spec-seed-diagnostics: + # Produce bounded diagnosis immediately, independently of the full suite. + # This remains required by ci-ok and never updates assertion goldens. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.14.4" + - name: Sync workspace with US and UK engines + run: uv sync --all-packages --locked --extra us --extra uk + - name: Check diagnostic bootstrap and failure reporting + timeout-minutes: 2 + env: + PYTHONDONTWRITEBYTECODE: "1" + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + run: >- + uv run --no-sync pytest --noconftest -p no:cacheprovider -q + packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py + packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py + packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py + packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py + packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py + - name: Derive candidate spec and seed identities (diagnostic only) + id: spec_seed_diagnostics + timeout-minutes: 15 + env: + DIAG_WORKFLOW_SHA: ${{ github.workflow_sha }} + DIAG_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + DIAG_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + DIAG_MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + PYTHONDONTWRITEBYTECODE: "1" + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + VECLIB_MAXIMUM_THREADS: "1" + NUMEXPR_NUM_THREADS: "1" + BLIS_NUM_THREADS: "1" + POPULACE_FIT_N_JOBS: "1" + POPULACE_FIT_PREDICT_WORKERS: "1" + run: | + DIAG_CHECKOUT_SHA="$(git rev-parse --verify HEAD)" + DIAG_UV_VERSION="$(uv --version)" + export DIAG_CHECKOUT_SHA DIAG_UV_VERSION + uv run --no-sync python tools/spec_seed_identity_diagnostics.py \ + --output-dir "$RUNNER_TEMP/spec-seed-diagnostics" + - name: Retain bounded candidate identity evidence + if: ${{ always() && steps.spec_seed_diagnostics.outcome != 'skipped' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: spec-seed-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}-py3.14.4 + path: | + ${{ runner.temp }}/spec-seed-diagnostics/candidate-digests.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-protocol.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-map.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-bindings.json + ${{ runner.temp }}/spec-seed-diagnostics/environment-and-source.json + ${{ runner.temp }}/spec-seed-diagnostics/diagnostic-status.json + if-no-files-found: error + retention-days: 3 + compression-level: 0 + include-hidden-files: false + engine-us: needs: changes if: github.event_name == 'push' || needs.changes.outputs.us == 'true' || needs.changes.outputs.shared == 'true' @@ -228,7 +308,14 @@ jobs: fail-fast: false matrix: python-version: ["3.13", "3.14"] - group: [us-p, us-qs, us-not, us-am] + lane: + - {"group": "us-p", "shard": "1/1"} + - {"group": "us-qs", "shard": "1/1"} + - {"group": "us-not", "shard": "1/1"} + - {"group": "us-am", "shard": "1/4"} + - {"group": "us-am", "shard": "2/4"} + - {"group": "us-am", "shard": "3/4"} + - {"group": "us-am", "shard": "4/4"} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -236,12 +323,18 @@ jobs: python-version: ${{ matrix.python-version }} - name: Sync workspace with US and UK engines run: uv sync --all-packages --locked --extra us --extra uk - - name: US engine tests (${{ matrix.group }}) + - name: US engine tests (${{ matrix.lane.group }} ${{ matrix.lane.shard }}) run: | - while IFS= read -r proc; do - mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.group }}:${proc}") + set -euo pipefail + selected_procs="$(uv run --no-sync python tools/ci_test_groups.py --procs "${{ matrix.lane.group }}")" + test -n "$selected_procs" + mapfile -t procs <<< "$selected_procs" + for proc in "${procs[@]}"; do + selected="$(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.lane.group }}:${proc}" --shard "${{ matrix.lane.shard }}")" + test -n "$selected" + mapfile -t files <<< "$selected" /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25 - done < <(uv run --no-sync python tools/ci_test_groups.py --procs "${{ matrix.group }}") + done engine-uk: needs: changes @@ -261,10 +354,16 @@ jobs: run: uv sync --all-packages --locked --extra us --extra uk - name: UK engine tests run: | - while IFS= read -r proc; do - mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list "uk:${proc}") + set -euo pipefail + selected_procs="$(uv run --no-sync python tools/ci_test_groups.py --procs uk)" + test -n "$selected_procs" + mapfile -t procs <<< "$selected_procs" + for proc in "${procs[@]}"; do + selected="$(uv run --no-sync python tools/ci_test_groups.py --list "uk:${proc}")" + test -n "$selected" + mapfile -t files <<< "$selected" /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25 - done < <(uv run --no-sync python tools/ci_test_groups.py --procs uk) + done wheels: # The charter-mandated packaging gate: editable installs hide packaging @@ -277,6 +376,15 @@ jobs: fail-fast: false matrix: python-version: ["3.13", "3.14"] + lane: + - {"group": "wheels", "shard": "1/8"} + - {"group": "wheels", "shard": "2/8"} + - {"group": "wheels", "shard": "3/8"} + - {"group": "wheels", "shard": "4/8"} + - {"group": "wheels", "shard": "5/8"} + - {"group": "wheels", "shard": "6/8"} + - {"group": "wheels", "shard": "7/8"} + - {"group": "wheels", "shard": "8/8"} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -336,10 +444,56 @@ jobs: PY - name: Spec identity section digests (cross-environment diffing aid) run: env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/spec_envelope_digests.py be uk - - name: Run the suite against the installed wheels + - name: Add engine-free source preparation extras + # Real source fixtures require microunit for tax-unit construction and + # PyTables for pandas HDF input, alongside h5py. Request the extras from + # built wheels so their metadata, not editable installs, supplies these + # dependencies. The preceding step still checks the base wheel boundary; + # neither source extra installs a country rules engine. + run: | + frame_wheel="$(ls dist/microcosm_frame-*.whl)" + build_wheel="$(ls dist/microcosm_build-*.whl)" + uv pip install --python /tmp/wheels-venv/bin/python \ + --constraint /tmp/wheel-constraints.txt \ + "microcosm-frame[us] @ file://$(realpath "$frame_wheel")" \ + "microcosm-build[source-io] @ file://$(realpath "$build_wheel")" + env -u PYTHONPATH /tmp/wheels-venv/bin/python -I - <<'PY' + from importlib import metadata, util + from pathlib import Path + from tempfile import TemporaryDirectory + + import h5py + import pandas as pd + import tables + + assert util.find_spec("microunit") is not None, ( + "microcosm-frame[us] must install microunit into the wheel venv" + ) + # Exercise the same HDF reader used by source preparation. Imports + # alone do not establish that the compiled PyTables backend works. + with TemporaryDirectory() as directory: + path = Path(directory) / "invented-source.h5" + expected = pd.DataFrame({"person_id": [1, 2], "amount": [0.0, 12.5]}) + expected.to_hdf(path, key="person") + pd.testing.assert_frame_equal(pd.read_hdf(path, key="person"), expected) + assert util.find_spec("policyengine_us") is None + try: + metadata.distribution("policyengine-us") + except metadata.PackageNotFoundError: + pass + else: + raise AssertionError("policyengine-us must be absent from the base wheel gate") + print("microunit", metadata.version("microunit")) + print("h5py", h5py.__version__, "tables", tables.__version__) + PY + - name: Run the suite against the installed wheels (${{ matrix.lane.shard }}) run: | + set -euo pipefail + selected="$(env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/ci_test_groups.py --list wheels --shard "${{ matrix.lane.shard }}")" + test -n "$selected" + mapfile -t files <<< "$selected" /usr/bin/time -v env -u PYTHONPATH /tmp/wheels-venv/bin/python -I -m pytest \ - packages/*/tests -p no:cacheprovider + "${files[@]}" -p no:cacheprovider --durations=25 ci-ok: # The single aggregate context for branch protection. Per-leg matrix names @@ -350,7 +504,7 @@ jobs: # dependency-skip lets a failed `changes` take the engine lanes down without # any required check going red: a lane skip is accepted only when # classification itself succeeded. - needs: [changes, lint, fast, engine-shared, engine-us, engine-uk, wheels] + needs: [changes, lint, fast, engine-shared, spec-seed-diagnostics, engine-us, engine-uk, wheels] if: always() runs-on: ubuntu-latest steps: @@ -365,6 +519,7 @@ jobs: require_success lint "${{ needs.lint.result }}" require_success fast "${{ needs.fast.result }}" require_success engine-shared "${{ needs['engine-shared'].result }}" + require_success spec-seed-diagnostics "${{ needs['spec-seed-diagnostics'].result }}" require_success wheels "${{ needs.wheels.result }}" for lane in "engine-us:${{ needs['engine-us'].result }}" "engine-uk:${{ needs['engine-uk'].result }}"; do name="${lane%%:*}"; result="${lane#*:}" diff --git a/.gitignore b/.gitignore index 1e77cefd2..a83de8598 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ tools/release.env # Finder metadata — never payload .DS_Store + +# Lane-local before/after measurement staging (see docs/us-native-verification-once.md) +.measure/ diff --git a/CLAUDE.md b/CLAUDE.md index fe23f470f..f21c8eb7a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,7 +26,8 @@ PR CI (`.github/workflows/test.yml`) has four lanes — `lint`, `fast`, the diff into `shared`/`us`/`uk`. `lint` verifies `tools/ci_test_groups.py --verify`, syncs with `--locked`, and runs ruff. `fast` runs the full tracked test-file inventory without engine extras in -three groups (`trade`, `spine-uk`, `rest`); engine-gated tests skip there +three groups (`trade`, `spine-uk`, `rest`), with `rest` split across six +parallel matrix jobs per Python version; engine-gated tests skip there through whichever guard they carry — the `requires_*` markers, or the `importorskip` calls that remain the norm on the US side. `engine-shared` always syncs `--extra us --extra uk` and runs the shared/spec group. `engine-us` and @@ -40,6 +41,31 @@ the wheel/import boundary and spec digests, and run the suite against installed wheels. The `fast` and engine lanes pass `--durations=25`, so each job log ends with its slowest tests; check there first when a lane's runtime jumps. +`us-am` likewise has four matrix subdivisions. Within each, build and other-shard +tests retain separate pytest processes to isolate their import state. Every wheel +job retains the complete build/install/smoke boundary and runs one of eight +disjoint file subdivisions, including engine-only files whose markers skip when +the engine is absent. Other groups and both Python versions are unchanged. +`tools/ci_test_groups.py --list GROUP[:PROCESS] --shard INDEX/COUNT` selects +sorted whole files round-robin; indices are 1-based and counts are capped at 64. +An invalid or empty selection fails before pytest. `--verify` proves the +configured shard unions and process partitions, while the stdlib-only +`python3 -I -B -S packages/microcosm-build/tests/test_ci_test_groups.py` checks +the actual matrix and selection contract. See [CI file sharding](docs/ci-file-sharding.md). +File counts are not measured runtime weights; a single expensive module can +still dominate a job, and module-scoped fixtures must remain intact. + +After checking the base wheel boundary, the wheel lane installs the built +`microcosm-frame[us]` and `microcosm-build[source-io]` extras for engine-free +source tests. An invented pandas HDF round trip checks the compiled reader; +the country rules engine must still be absent. + +The shared engine lane also runs a bounded source/seed identity diagnostic on +Python 3.14.4 after its normal tests, including when those tests fail. Its six +JSON artifacts retain candidate digests and their complete canonical seed +records for review. They never replace the test assertions or certify coverage; +review the captured source and dependency identities before updating any pins. + New commits to a PR cancel older unfinished CI runs for that same PR. Each main-push run has a unique concurrency group, so all main-push runs remain independent and can finish validating their merged changes. @@ -61,10 +87,9 @@ load, or it silently receives the spec an earlier test cached. **Adding a test file.** It must sit directly in `packages//tests/` — flat, no subdirectories; `fixtures/` and `golden/` hold data only — and be named -`test_*.py`. The lanes run explicit file lists built from a flat pathspec, while -local `uv run pytest` and the wheels lane discover recursively, so a test parked -next to its fixtures would run locally and stay green in CI without ever -executing against an engine. `--verify` fails on such a file rather than letting +`test_*.py`. All lanes, including wheels, run explicit flat file lists, while +local `uv run pytest` discovers recursively. A test parked next to its fixtures +could run locally without being selected in CI. `--verify` fails on such a file rather than letting it hide. Build tests that exercise a country engine must be named `test_us_*` or `test_uk_*` so they land in that country's lane; an engine-dependent file named anything else falls into the always-on `shared-spec` group and runs on every PR. @@ -171,6 +196,13 @@ A sealed deny-list in `microcosm.build.us_runtime.h5_io` overrides this opt-in for known-excluded publications while preserving their scoring-only diagnostic path. +The native survey development handoff is documented in +[the native handoff guide](docs/us-native-survey-development-handoff.md). +`prepare_native_survey_development_input` in the maintained US release tool +accepts a live issued enrichment owner and returns a verified Frame checkpoint +plus missing-input/gate inventory. This development checkpoint is not accepted +as native authority, a legacy pool manifest, or a certified dataset. + `tools/build_us_acs_donor_receipt_qualification.py` is a third local, non-publishing lane. It takes one of two exact pinned Build P lineage parents and appends the three reported-receipt inputs current main's ACS transfer diff --git a/DESIGN.md b/DESIGN.md index 27be01cf0..f9bb2e115 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -280,12 +280,10 @@ risks, not footnotes: - **Protected families are defined, not vibes.** The non-degradation clause names specific target families (income-tax-relevant: capital gains, dividends, interest, retirement income; and the benefit-program - families) with explicit tolerances. SPM poverty is protected through the - held-out regression gate (see the survey tax-benefit holdout bullet - below), never as a target family — and "SPM resource components" earn no - separate listing: their survey-measured versions are the prohibited - quadrant, and their administrative totals are already the - benefit-program families. A contribution may not + families) with explicit tolerances. Survey poverty comparisons are excluded + from these gates. SPM resource components earn no separate listing: their + survey-measured versions cannot be targets, and their administrative totals + are already covered by the benefit-program families. A contribution may not worsen any protected family beyond tolerance even if it improves aggregate loss. This list is versioned with the population and is the steward's call. - **Off-target validity.** Generate-big-then-prune *selects* records by the @@ -302,11 +300,18 @@ risks, not footnotes: all — may never be fitted. The rebuild replaces the survey's tax-benefit measurement with imputed, computed, and admin-calibrated values; fitting the survey-derived version launders its error back in and destroys the - held-out signal the evaluation depends on. Corollary: deviations from - official poverty metrics are expected by construction (corrected - underreporting should sit below survey-based rates, all else equal) and - are never inherently problematic — official numbers are comparators, not - truth. + held-out signal the evaluation depends on. +- **Survey poverty is comparison-only.** Report comparable US SPM/OPM, UK + HBAI and other survey poverty measures with their definitions and periods. + Do not use them as calibration targets, objective terms, informal tuning + goals, candidate-selection scores or release gates based on benchmark + proximity. This includes choosing weights, imputations, take-up assignments, + target deferrals or hyperparameters to bring poverty closer to a survey + estimate. Differences can prompt investigation; they do not establish a + defect or a preferred direction of change. Fix definition, measurement and + implementation errors on independent evidence, even when a correction moves + the comparison farther from the survey estimate. These correctness checks + remain release requirements; matching the poverty rate does not. - **Correlated evidence.** Target standard errors from one survey are design-correlated across its published cells; treating them as diagonal overweights cell-rich surveys (the standard GREG caveat). Evidence diff --git a/FINAL_REPORT.md b/FINAL_REPORT.md index 7648239b2..6f588fbcc 100644 --- a/FINAL_REPORT.md +++ b/FINAL_REPORT.md @@ -1,130 +1,269 @@ -# Final report: stacked-pool to release CD-vintage provenance - -## Outcome - -The producer/consumer defect is fixed. The production stacked route now -produces the same authenticated congressional-district contract that release -preflight requires: the canonical crosswalk SHA, target vintage -`119th_congress`, and positive household -`congressional_district_geoid` support. Geography remains outside source -assembly, and the release guard still fails closed on every mismatch -(`packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py:346-406`; -`tools/build_us_fiscal_refresh_release.py:2565-2613,8608-8612`). - -## Five-part repair - -1. The stacked CLI requires path/SHA pairs for the national PUMA ladder and - canonical 117th-to-119th CD crosswalk. Both declared hashes must equal the - repository pins before the source bytes are authenticated - (`tools/build_us_multispine_pool.py:563-594,897-1004`). The real geography - operator runs immediately after operator-free source assembly and before - gap fill or cloning - (`packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py:346-406`; - `tools/build_us_multispine_pool.py:5290-5334`). - -2. The configured namespace, checkpoint identity, persisted stage receipts, - and terminal manifest bind both byte authorities, source and target - vintages, assignment declaration, overlap algorithm, operator order, seed - site/stream/value, and ordered native-household output receipt - (`tools/build_us_multispine_pool.py:814-928,1300-1469,1711-1930,3530-3635,3989-4027,4133-4215`). - -3. Nullable H5 publication accepts validated root attributes, writes them in - the same temporary HDFStore as the fixed entity tables, verifies their - exact round trip, and atomically replaces the destination only after all - checks pass. The stacked publisher supplies the crosswalk-SHA and target- - vintage attrs - (`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:1463-1699`; - `tools/build_us_multispine_pool.py:4423-4519`). Schema-9 loading also binds - those physical attrs and the live household geography/clone lineage back to - the manifest - (`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:671-923,1032-1193,1321-1364`). - -4. Release preflight now reads root attrs and the household frame from one - HDFStore snapshot through the shared fixed/table-aware reader. Its existing - equality checks for crosswalk SHA, current target vintage, and positive - household district support are unchanged - (`tools/build_us_fiscal_refresh_release.py:2565-2677`). - -5. A tiny fixture invokes the real stacked entry point, post-assembly - geography operator, fixed-H5 publisher, and real release assertion. It - proves both root attrs and positive household district support; consumer - tests reject changed attrs, changed native geography, missing lineage, and - divergent clone geography - (`packages/microcosm-build/tests/test_us_multispine_pool_tool.py:2279-2335`; - `packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py:1357-1440`; - `packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py:1061-1184`). - -## Assignment design and anti-rot - -A one-value deterministic PUMA-to-CD lookup is not defensible because 2020 -PUMAs and congressional districts do not nest. The existing national ladder -preserves observed ACS PUMA, assigns missing ASEC PUMA proportional to 2020 -PUMA population, then draws CD and county within PUMA proportional to block- -population overlap. Stable row/state/PUMA order and the ledgered -`legacy_puma_ladder` / `geography_legacy` / seed `0` contract make the draw -reproducible for fixed inputs -(`packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py:1-20,293-383,638-698`; -`packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py:870-887`; -`packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml:421-432`). - -The spec and schemas now declare the crosswalk source/vintage authority and -geography assignment, while field-usage and inventory owners cover the new -surface exactly -(`packages/microcosm-build/src/microcosm/build/us/spec/geography.yaml:3-37`; -`packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml:69-86`; -`packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml:45-53`; -`packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py:387-393,671-692,803-822`; -`packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py:348-380,1649-1712`). - -The generated US spec SHA is -`5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e`. -Coverage is exact at 42,120/42,120 fields, 49 claims, and 41/41 inventory -checks. Principal legitimate movements are pointer inventory -`6d7353c6...` to `bc4a948a...`, full checkpoint `b6a47fac...` to -`a128a85f...`, and the new geography identity `f49425ca...`; executable -versions move checkpoint 11 to 12, terminal manifest 8 to 9, and nullable H5 -materializer 2 to 3 -(`tools/spec_engine_coverage.py:42-45`; -`packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py:348-380`; -`docs/evidence/spec-engine/us-f0-coverage.json:10,775-788,1890,2027-2057,2602`; -`tools/build_us_multispine_pool.py:332-360`; -`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:75-105`). - -## Verification - -- `microcosm-calibrate`: 203 passed. -- `microcosm-data`: 318 passed / 2 skipped. -- `microcosm-fit`: 93 passed. -- `microcosm-frame`: 295 passed / 36 skipped. -- `microcosm-build` partition A: 4,155 passed / 36 skipped. -- `microcosm-build` partition B: 2,177 passed / 3 skipped. -- Accepted workspace total: **7,241 passed / 77 skipped / 0 failed**. -- The high-risk real integration and H5 consumer cases are included in those - build partitions - (`packages/microcosm-build/tests/test_us_multispine_pool_tool.py:2279-2335`; - `packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py:1357-1440`). -- Bundle freshness reports the spec SHA above; coverage freshness reports - 42,120/42,120 fields and 41/41 inventory checks - (`tools/generate_us_bundle_from_constants.py:355-437`; - `tools/spec_engine_coverage.py:378-398`). -- Repository-wide Ruff, smoke-script shell syntax, and `git diff --check` pass. - The source-blind import-graph anti-rot pin now covers the exact 69-module - graph reached by the three new shared validators - (`packages/microcosm-build/tests/test_us_spine_blindness.py:3270-3315`). - -No production pool or release build ran, no push occurred, and -`logbook-pending-chain.txt` was not touched. - -## Candidate handoff - -Stage 1 must add the canonical PUMA-ladder and CD-crosswalk path/SHA pairs -listed exactly at the end of `_LANE-NOTES.md`, run off-chain, and rebuild under -the new checkpoint identity. The old smoke checkpoints and pool publication -are not reusable: configured identity now includes both new input pins and the -geography contract, checkpoint materializer is 12, terminal manifest schema is -9, and nullable H5 materializer is 3 -(`tools/build_us_multispine_pool.py:332-360,1300-1555`; -`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:75-105`). -The six pre-existing immutable source artifacts can be reused; the Stage-1 -checkpoint namespace and pool H5 must be regenerated with the two additional -authenticated authorities (`tools/build_us_multispine_pool.py:931-1004,1292-1319`). +# Bounded completion-host corrections — 2026-09-13 + +Source corrections are complete within the requested 20-minute window beginning +22:09 UTC. **All tests, collection, project imports and runtime acceptance remain +UNRUN.** This section supersedes the earlier implementation report preserved below. +No runtime, native-capacity, scientific, release, default-adoption or PR893 change +is claimed. No commit or staging occurred: the later explicit prohibition governs +the standing commit instruction. PROGRESS.md was maintained without committing. + +## Provenance and ownership + +Worktree: `/Users/maxghenis/PolicyEngine/_worktrees/microcosm-us-completion-host-integration-20260913`. +Unchanged HEAD: `6aa745c8762c3e4a140b8168470de5827e12c5cb`. +The dirty shared worktree was intentional. This pass edited exactly: + +- `packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_completion_host.py` +- `packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py` +- `docs/us-survey-completion-host.md` +- `PROGRESS.md` +- `FINAL_REPORT.md` + +Read CLAUDE.md and the full independent review and full approved R2 plan at the +exact supplied recovered paths under +`/Users/maxghenis/PolicyEngine/_recovered/scratch-backup/893/codex-us-followup-host-integration-20260913/`. +The review's plan-location objection resulted from its search scope; the supplied +absolute plan exists and its complete file hash matches the approved hash. + +| Input | SHA-256, unchanged at final source check | +| --- | --- | +| `HOST-INTEGRATION-PLAN-R2.md` | `b402d6be9d374774e27da7bf42fb83a216305d4cddd2ad8347c8149536269a29` | +| `HOST-SOURCE-INDEPENDENT-REVIEW.md` | `f98a9927e882b3f788eaebf55863d150023efd467a9bc2fc57770225ec5c5a4e` | +| Shared `graph_atomic_survey_financial.py` | `7545d1315b978735b8da4e02ad10ad78ce3d72854e2cc31a607bbd073210fb84` | +| Frozen `graph_survey_completion.py` | `858b4f42d9d61596901797976bfb2a9bf9bca9f9820b930fb3484e3f641ef0e1` | +| Frozen `graph_property_tax_leaves.py` | `883afb4a95652dd664d636b3a2eaf8481e18f4c43e3d7a24ae0cfb51b6b68c93` | + +Earlier review/failure packets were not changed. Shared source, other tests, +inventory, CI and changelog files were not edited. A new dirty entry for +`test_us_spine_blindness.py` appeared during the pass; it is another owner's work +and was neither inspected nor changed here. No reset, clean, rebase, commit, +staging, project imports, collection/tests, project scripts, dependency resolution, +network/browser/agents, native/private/installed payloads, or execution-evidence +containers were used. Source reads, metadata/hashes, standard-library AST parsing, +direct Ruff and whitespace checks were the inspection/validation operations. + +## Corrections and source evidence + +1. **Fragment identity/counts (S1):** exact ordered Node IDs now bind receiving1, + optional roles0/2, child6 and tax3 before compilation. Receiving and role + stages also check before their intermediate compilation. The child document + count derives from retained nodes. Direct boundary controls use real retained + declarations: moving an actual tax node into child7 while reducing tax to2 + preserves the total but refuses; wrong receiving identity and reversed roles + refuse too. No issuer, executor or compiler is replaced. +2. **Knownness provenance (S2):** the portable flag derives from the child + verification bytes produced by `_child_expected` reconstruction. The existing + materialized child verifier and exact payload comparison remain mandatory + before issuance. The document hashes those same bytes. The positive assertion + binds the flag and hash to the retained payload and issued artifact hashes. + Arbitrary JSON flags acquire no authority. +3. **Roles disabled (T1):** separate test selectors author actual cold/required + 49-node status-enabled and 45-node status-disabled host runs. Each explicitly + selects the partial ACS+ASEC fixture (2/3 selection, seed41) with the full-source + teenage donor outside the selected ASEC household. They assert actual checked + views, 39/35-node retained bases, no role fragment or writes, exact incumbent + role preservation/absence, final completeness and declaration/manifest/payload/ + artifact parity, staged base hits, extension misses on cold and all required + hits. The 51-node adverse suite is not multiplied. +4. **Child adversaries/copying (T2/T6):** exact full refusal strings are + `CHILD_PROPERTY_RETAINED_VALUES_CHANGED`, + `CHILD_PROPERTY_RECONSTRUCTED_ARTIFACT_CHANGED` and + `CHILD_PROPERTY_MATERIALIZED_VERIFICATION`. Source ordering is + `verify_materialized` evidence seal -> `_current`/`_pure` -> reconstruction -> + `_artifacts` -> `_verification_value`; the parent mutation fails in `_pure`, + artifact bytes at reconstruction comparison, and producer/key mismatch at + `_verification_value`. Revocation and original-base checks remain. The comment + now states verification can reconstruct/refit. Executor `_apply_result` FILTER + calls `Frame.select`; `Frame.__init__` copies every table. Role `_patch_columns` + calls `_copied_tables(deep=True)`. The test additionally asserts the actual + base stamp is unchanged immediately after mutating the reconstructed parent. + No counterfeit issuer or replacement parent is used to avoid this cost. +5. **Ordering and conservation (T3):** assertions cover all actual prefix9 nodes, + financial/property attachment and enabled status BIND before completion + receiving. Both actual observed FILTER transitions must preserve entity + dimensions, complete tables, schema, strata and weights and append precisely + one ledger record. Source uses separate `operation="filter"` and + `policy="conserve"` fields, not a `filter_conserve` operation token. Total, + stratum and partition mass must match. Actual owners/versions are checked from + property/status writers to completion FILTER, optional role BIND, child ATTACH, + tax FILTER and final tax-leaf writer. No pseudo-node or generic API change. +6. **Authentic identity controls (T4/T5):** the existing issued tax-rebased51 run + refuses as a second completion parent. The 49-node warm handle undergoes a + same-values/different-Population-identity substitution; restoration cannot undo + revocation and its base/cold peer remain checked. Extracted private union + observation and final-I/O checks remain in the production `_extend` path. + Tests reuse actual observed51 evidence to refuse a base miss (even though the + manifest content key stays equal), a missing observation and changed retained + source/node/implementation/artifact identities. The final-I/O checker re-reads + the real source/store and implementations, without executing kernels or adding + a fit. No fake runner/owner is used. Other original final-I/O population-mutation + controls remain distinct from these exact final-identity refusals. + +The fixture's source-construction wrappers now restore before issuance so later +fixture composition cannot recursively wrap an earlier fixture's closure, replace +its new literals, or reuse the earlier restoration output directory. Source pins +remain under the existing real fixture's monkeypatch lifetime. This is authored +source logic, not demonstrated fixture execution. + +S3's empty-ordering generic suggestion is not a live host defect: the host supplies +two typed property edges and the completion FILTER follows the full cloned spine. +Frozen receiving/tax APIs and compiler/source-blindness guards remain unchanged. +The independently confirmed old status+tax42 complement-owner defect is unrelated; +its isolated successor remains root-owned. This pass does not change shared +`graph_atomic_survey_financial.py`, broaden accepted owners, or certify that path. + +## Authored selectors and finite scope — all UNRUN + +AST only: **16 test functions / 32 source-expanded cases**, up from10/20. Counts +come from literal parameter-list lengths; no pytest collection was performed. +Prefix every selector below with +`packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py::`. + +| Selector | Cases | +| --- | ---: | +| `test_refused_options_precede_source_io` | 6 | +| `test_roles_disabled_status_enabled_49_cold_required_and_output_identity` | 1 | +| `test_roles_disabled_status_disabled_45_cold_required` | 1 | +| `test_disabled_signature_and_serialized_option_parity` | 1 | +| `test_exact_51_roster_order_and_actual_required_hits` | 1 | +| `test_actual_donor_outside_selection_and_paired_child_draws` | 1 | +| `test_full_weights_geography_status_roles_and_complete_gate` | 1 | +| `test_detached_parent_refuses_without_owner_substitution` | 1 | +| `test_actual_tax_rebased_parent_refuses` | 1 | +| `test_wrong_fragment_roster_refuses_before_compile` | 3 | +| `test_actual_union_observation_and_base_cache_refusals` | 2 | +| `test_actual_union_final_io_identity_refusals_without_execution` | 4 | +| `test_explicit_scientific_identity_changes_declarations` | 3 | +| `test_incomplete_adult_refuses_actual_puf_qualification_before_fitting` | 1 | +| `test_real_child_verifier_refuses_changed_custody` | 3 | +| `test_final_source_io_mutation_permanently_revokes_host` | 2 | + +The 49 and45 selectors each construct their own pair and are selectable without +the shared51 fixture. The complete file authors seven top-level host constructions: +three cold/required pairs plus one incomplete-adult run. Each host stages a base +and extension; seven is not a graph-execution count or a fit budget. Repeated +checked views and child materialized verification can requalify/reconstruct/refit. +The four final-identity parameters each perform one positive and one adverse real +I/O check. Root must budget these reads and reconstructions explicitly. Final +population-poisoning controls remain last on their shared handles. + +## Source checks and unresolved scope + +- Standard-library `ast.parse` passed for both owned Python files; selector counts + were obtained from their AST without importing project code. +- Direct `/opt/homebrew/bin/ruff check` passed for both owned Python files. +- Direct `/opt/homebrew/bin/ruff format --check` passed: both files formatted. +- `git diff --check` returned0 without findings. Individual untracked source/test/ + doc checks used `git diff --no-index --check /dev/null PATH`; each reported no + whitespace diagnostics and returned the no-index difference status1. +- No runtime failure/success evidence was created. Some source-location searches + missed guessed filenames and were resolved with `rg`; no execution containers + were opened. Direct Ruff found no lint failures in this correction pass. + +| Remaining scope | Concrete reason and next action | +| --- | --- | +| T4 financial-only parent (`property_income=None`) | No authentic19/23 financial-only run is retained in this fixture. Author/budget a separate real issuance; do not substitute a forged state. The tax-rebased-parent branch is now covered by source-authored test. | +| T5 initial base-key/artifact and final manifest/declaration adversaries | Final source/key/implementation/artifact identity and observed/base-hit controls are authored, but not every distinct earlier/later refusal is provoked. Exact additional boundaries need authentic retained evidence and separate finite scope; do not fake executor/compiler output. | +| T5 role reconciliation and union compiler roster | A conflicting original-source role needs coherent pre-issuance source ownership, not post-issuance erasure. Exact fragments now prevent compensating counts; a compiler-corruption control would require prohibited substitution or a separately reviewed authentic construction. | +| T6 fixture/checked-view behavior | All new and existing composition assumptions remain UNRUN. Guarded execution must validate source attachment restoration, member pins, selected-child/adult completeness, and actual role/status custody. Source confirms `puf55_survey_recipients.qualify_puf55_survey_recipients` calls `require_complete_property_taxes` before recipient/source admission. | +| T7 unsupported GQ / zero weights / modeled-zero science | Positive sources remove GQ and the upstream helper deliberately requests positive weights. A valid GQ/zero-weight source fixture needs separate source-owner review/changes. The selected positive child draw does not establish modeled-zero scientific coverage. Keep unknowns unknown; no zero-fill or scope waiver. | +| Root integration and resources | Independent combined-source review; source/import-use inventories, test-spine/CI classification, identity/golden reconciliation, status42 repair, generic/country/status gates, fresh guarded stage/resource budget and all runtime acceptance remain outstanding. | + +## Exact authored file identities + +| File | Preimage SHA-256 | Final SHA-256 | +| --- | --- | --- | +| `graph_survey_completion_host.py` | `6ce066b4e1b33ed1572a5e82a3414070a5f6b7360ee451028634d0862109beac` | `25852246f2690ec754dbc62ae88173f0ba1fb8115d9aa1ea86375f81daf3df40` | +| `test_us_graph_atomic_completion_host.py` | `54678f4758f9b8a96d9dc885e2f22a5fa997b1458033dcafc754075b6f8c9857` | `14735a01e119f853935d04ea574f38f452b230d16717a0ef3ea898548753abae` | +| `docs/us-survey-completion-host.md` | `f59702f85d69298bddd8ccfe2c3e3399267c66daf70b28e348c9312fb187553f` | `d654164704eb728b75c01fa8e02f54fcae6a235164f540626827a266905fc372` | +| `PROGRESS.md` | `8680141c6857cf7a8fde48c625dc47e1fba5a1f868f07f1be09bf7bf96aeeb2f` | `b42bfb5e4de0e13a1edf45229ae857e75cc4032756d21744ecad356873274071` | +| `FINAL_REPORT.md` | `95a74f7d0a49c896e1e4934e998eb7d63166a1103978d3a6fff192491a34100c` | Reported externally in the final response to avoid a self-referential hash. | + +## Historical implementation report (preserved) + +The following report predates the independent correction pass. Its10/20 counts, +45/49 coverage gaps and earlier verification descriptions are historical; the +correction section above is the current source-only handoff. + +# US completion host source report — 2026-09-13 + +Source implementation is authored for root inspection. **All tests and runtime +checks remain UNRUN. This is not combined-host acceptance.** + +Branch: `us-completion-host-integration-20260913`. +Resolved base / unchanged HEAD: `6aa745c8762c3e4a140b8168470de5827e12c5cb`. +Frozen R2 plan verified SHA-256: +`b402d6be9d374774e27da7bf42fb83a216305d4cddd2ad8347c8149536269a29`. +Read the original independent APPROVE_SOURCE_PLAN review and repository instructions. + +Implemented explicit `household_roles=False` and `child_property=None`, strict +unsupported-mode refusals and exact explicit ChildPropertyOptions. Disabled +serialization remains conditional. The actual issued 35/39 property parent is +retained with its original preparation, store, keys, registry and legacy/property +outputs. Registries are copied before extension. A privately reconstructed full-row +receiving version, optional reconciled roles, unchanged child6 and separate tax3 +compose the union; all issued prefix nodes must hit cache. Tax declarations use +the independently completed Frame and bind the actual child verification edge. + +The existing issuer/state/checker retain the full union, all node populations and +states, registry identity and completion custody. Actual completed/support +Populations and typed store evidence reach the real child materialized verifier, +including required replay. Source/store work precedes final pure seals. Failed +completion checks revoke permanently. Old property verification receives its +retained original output. Documentation labels old routing as pre-child source +gaps, preserves modeled-zero semantics and the all-person tax/PUF gate. + +| Changed path | SHA-256 | +| --- | --- | +| `packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py` | `7545d1315b978735b8da4e02ad10ad78ce3d72854e2cc31a607bbd073210fb84` | +| `packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_completion_host.py` | `6ce066b4e1b33ed1572a5e82a3414070a5f6b7360ee451028634d0862109beac` | +| `packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py` | `54678f4758f9b8a96d9dc885e2f22a5fa997b1458033dcafc754075b6f8c9857` | +| `docs/us-survey-completion-host.md` | `f59702f85d69298bddd8ccfe2c3e3399267c66daf70b28e348c9312fb187553f` | +| `changelog.d/us-survey-completion-host.added.md` | `0f16800fb200eca15adcce43d8b44f8a73e6d89c1af79b8c85bfdee00006eef6` | +| `PROGRESS.md` | `8680141c6857cf7a8fde48c625dc47e1fba5a1f868f07f1be09bf7bf96aeeb2f` | + +Output file: `FINAL_REPORT.md` (this report). PROGRESS.md was maintained in place. +No commit or push was made: the task-specific prohibition overrides the standing +commit order. No reset, rebase, clean, staging or other-owner edit occurred. + +Static verification: +- AST parsing passed for all three owned Python files. +- `_FinancialRunState` has 41 annotated fields and its constructor has 41 + positional arguments; the new state-tail alignment was inspected statically. +- Direct `/opt/homebrew/bin/ruff check` passed for the three owned Python files. +- Direct Ruff format checks passed; only owned files were formatted. +- `git diff --check` passed for the edited tracked host/journal files. +- Exactly **10 test functions / 20 source-expanded cases** authored. No collection + or tests ran. Positive fixture: one shared cold51/required51 pair. Separate + incomplete-adult case: one host run and actual PUF qualifier refusal. Three + adverse child-custody cases reuse actual run evidence with separately constructed + private boundaries; they do not add graph fits. Final-I/O controls poison the + two positive handles last. No mocked owner or executor substitutes were added. + +Remaining work / missing acceptance seams: +- Independently review the full combined diff and the other agent's receiving/tax + seam APIs. Their files were read but never edited by this task. +- Execute the new fixture only under root's fresh bounded resource approval. + Its pre-issuance ACS housing restriction, ASEC donor sampling, coherent role and + status literals, final member pins and monetary attachment composition are + authored assumptions, not demonstrated runtime results. No outside-owned + fixture edit has been established as necessary by source inspection. +- 45/49 counts are source-derived; this test file's full positive execution target + is 51. Dedicated 45/49 execution controls, unsupported-GQ and zero-weight host + fixture coverage, and broader identity/adversarial cases remain to be added or + reconciled with existing component controls by root. +- Root owns graph/source inventories, AST import-use records, test-spine/CI + classification, identity/golden reconciliation and the exact final closure and + resource-call budget. None were updated or run here. Existing upstream component + passes do not certify this composition; generic/country/status/runtime gates + remain independently required. +- No national-scale/native/PUF-fit, scientific age-transfer, fiscal, release, + publication or default-adoption claim is made. + +Failure evidence: initial direct Ruff runs found only import-order findings in +the new helper and new test file; these were corrected and subsequent checks +passed. Several source-search commands returned missing-path/no-match statuses +while locating maintained modules; no runtime failure evidence was produced or +removed. No project imports, collection, dependency resolution, network/browser/agents, +source staging, preflight, guard, native/private payload access or installed-resource +inspection was performed. Work stayed inside the requested 20-minute source window. diff --git a/PROGRESS-908-target-snapshot-fixes.md b/PROGRESS-908-target-snapshot-fixes.md new file mode 100644 index 000000000..708880f6d --- /dev/null +++ b/PROGRESS-908-target-snapshot-fixes.md @@ -0,0 +1,145 @@ +# microcosm#908 — closing the four reviewed findings on the snapshot slice + +Lane journal for branch `calibration-target-snapshots-908-fixes-20260912`. +Journals are history, not state (see CLAUDE.md): check git/GitHub for current truth. + +Historical note, 12 September 2026: this journal records the earlier repair +lane. The lane closed at `9a325c8b0`; root independently repaired the remaining +strict-null codec case in `b8380c475`, with 52 tests passing. Current scope and +remaining integration work are in [the maintained guide](docs/calibration-target-snapshots.md). +The earlier work-in-progress and not-pushed statements below are historical. + +## State + +Merged the completed slice (`bae1887ff`) onto `origin/main` `116d46ee9`, resolved +the four overlapping spec-engine identity pins by recomputing them on the merge +ref, and am now implementing the four reviewed findings. Not pushed, no PR. + +## Review basis + +`908-final-review-review-full-report.md` (head `bae1887ff`, merge base +`35fc76dd1`) with `probe.py` / `reproductions.json`. Four open findings: + +1. **C1** — the aggregate-only claim is enforced by a denylist over arbitrary + nested JSON, so record-level vectors ride through `context` and a + mapping-valued `candidate_id`. +2. **A1** — nested metadata is copied only by `dict(...)`, so a sink mutating a + delivered payload changes the caller's mapping and the next snapshot. +3. **A2** — `_write_chunk` creates the final `history/.json` name before + writing it, so a concurrent reader sees a partial chunk, and a failed write + leaves an invalid immutable chunk behind. +4. **A3** — the public codec admits impossible metadata: `epoch > epochs`, a + non-timestamp `created_at`, a non-string `candidate_id`, and a + `best_retained` whose availability, epoch and loss contradict each other. + +## Done + +- Merged `bae1887ff` into this branch and recomputed every conflicting + identity pin on the merge ref (table below). +- 14 red regressions in `packages/microcosm-calibrate/tests/test_target_snapshots.py` + (9 of them failing on the reviewed head), then the implementation: + - **C1** — a closed, typed, bounded aggregate metadata contract + (`normalize_metadata`, `normalize_best_retained`, `METADATA_LOCATIONS`, + `MAX_METADATA_*`) applied uniformly to `context`, `search`, `selection` + and `best_retained`, with string-only identifiers checked rather than + coerced. A list is not a scalar, so a record vector has no shape to ride + in at any depth. The record-level key-name rule is kept on top, because a + *scalar* `household_id` is still record-level identity. + - **A1** — every metadata container in a delivered payload is freshly built + from immutable scalars, pinned by a structural test asserting the payload + shares no mutable object with the caller, the observer, the bound view or + the next snapshot. + - **A2** — history chunks are written and `fsync`-ed to a hidden temporary + and then published atomically with `os.link`, which refuses rather than + overwrites an existing immutable chunk. Failed writes clean up their + temporary. Atomic `latest.json`, run ownership, duplicate refusal and + bounded retention are unchanged. + - **A3** — strict public-codec validation of `created_at` (timezone-aware + ISO-8601), identifiers, `epoch <= epochs`, `sequence >= 1`, + `non_finite_rows <= n_targets`, the closed `best_retained` triple's + availability/epoch/loss consistency, and a `selected` snapshot whose epoch + contradicts its own selection receipt — paired with normalization at the + emitting edge so honest nonfinite values still serialize as explicit nulls + and counts and the observer still cannot abort a run. +- `experiments/908_review_findings_recheck.py` replays the review's own four + counterexamples against this branch; receipt in + `experiments/908-review-findings-recheck.json`. All four report closed. + +## Deliberately narrow choices + +- `context` has **no in-tree producer**. Rather than invent a wider shape for a + caller that does not exist, it takes the same flat scalar contract as the + seams `solve.py` really emits. A caller that needs structure should add a + typed seam and extend the contract, in the PR that adds the caller. +- The metadata bounds (32 entries, 64-character keys, 256-character strings) + are the narrowest values that comfortably hold everything `solve.py` emits. + +## Verification (isolated interpreter, no installs, threads=1) + +`/Users/maxghenis/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910/.venv/bin/python -I -B -S` +with this worktree's shard `src` roots ahead of that venv's site-packages and +an import-path assertion, `OMP/MKL/OPENBLAS/NUMEXPR/VECLIB_NUM_THREADS=1`. + +| battery | result | +| --- | --- | +| `packages/microcosm-calibrate/tests` | 307 passed (12 s) | +| `packages/microcosm-fit/tests` | 124 passed (13 s) — main's QRF pins intact | +| `test_spec_engine_loader.py` + `test_us_multispine_pool_tool.py` | 197 passed (309 s) | +| `packages/microcosm-build/tests -k "inventory or coverage or seed"` | 448 passed, 2 skipped (343 s) | +| `packages/microcosm-graph/tests -k parity` | 25 passed (88 s) | +| `tools/spec_engine_coverage.py --check` | 42156/42156 fields, 41/41 inventory | +| `tools/ci_test_groups.py --verify` | `verification=ok`; the snapshot test file lands in fast `rest` / engine `us-am`, never `[defaulted]` | +| `ruff check .` | clean; `ruff format --check` clean on both touched files | + +`test_target_snapshots.py` holds 52 tests, 16 of them new here and 2 existing +ones updated to the closed contract's refusal points. Replaying the whole file +against the reviewed head's `target_snapshots.py` (the only source that +differs) gives **12 failed, 40 passed** — 10 new red regressions plus the 2 +updated tests. Every finding has at least one red regression: + +- **C1** — `test_context_refuses_the_record_vectors_the_review_smuggled_through`, + `test_metadata_refuses_arbitrary_nested_payloads`, + `test_identifier_fields_must_be_strings`, + `test_supported_scalar_metadata_survives_and_is_bounded`. +- **A1** — `test_metadata_refuses_arbitrary_nested_payloads` is the review's + own aliasing counterexample; the detachment invariant itself is pinned by + `test_a_delivered_snapshot_shares_no_mutable_object_with_its_caller` and + `test_sink_mutation_cannot_reach_caller_metadata_or_the_next_snapshot`, + which pass on both heads because the flat case was never the defect. +- **A2** — `test_a_history_chunk_is_published_only_after_its_bytes_are_complete`, + `test_a_failed_chunk_write_leaves_no_partial_or_leftover_file`. +- **A3** — `test_codec_refuses_the_impossible_payloads_the_review_reproduced`, + `test_non_finite_diagnostics_stay_null_statuses_rather_than_aborting`, + `test_the_codec_is_stricter_than_the_emitting_edge_and_says_so`, + `test_created_at_is_a_timezone_aware_timestamp`. + +The remaining 6 new tests are preservation tests (the structured metadata +`solve.py` really emits, the epochs the solver really selects, duplicate chunk +refusal, and observer-off/observer-on weight parity on the L0, budget-search, +refit and proximal paths); they pass on both heads by design. + +## Next + +- Independent re-review by main before integration/PR updates. + +## Identity pins recomputed on the merge ref + +`microcosm.calibrate.solve` is attested (`_DIRECT_KERNEL_MODULES`); main's #912 +edited the attested `microcosm.fit.qrf`, so both sides' pins were stale and the +merge conflicted on all four. Each was recomputed on the merged tree, after +first proving with the same interpreter that restoring **main's** `solve.py` +into the merged tree reproduces **main's** committed pins exactly — so the +drift is attributable to the slice's `solve.py` edit, not to an environment +leak. + +| pin | main `116d46ee9` | slice `bae1887ff` | merged | +| --- | --- | --- | --- | +| `EXPECTED_HASHES["seed_protocol"]` | `553d5e0b…` | `f4dc507c…` | `d052fd87…` | +| `EXPECTED_HASHES["seed_map"]` | `20058e54…` | `32dea304…` | `d5a9694a…` | +| US resolved-spec `spec_sha256` | `1eeca53a…` | `35a3623b…` | `ff2c9703…` | +| minimal-spec loader golden | `b4659890…` | `a67bb78c…` | `8a240898…` | + +`docs/evidence/spec-engine/us-f0-coverage.json` was regenerated with +`tools/spec_engine_coverage.py` (42156/42156 fields, 41/41 inventory checks). +The graph `calibrate.adam@1` parity pin merged cleanly and re-verified green +(25 parity tests). `fit.qrf` and `simulate` pins are untouched. diff --git a/PROGRESS-908-target-snapshots.md b/PROGRESS-908-target-snapshots.md new file mode 100644 index 000000000..7a7f9dedc --- /dev/null +++ b/PROGRESS-908-target-snapshots.md @@ -0,0 +1,171 @@ +# microcosm#908 — per-target calibration estimate snapshots (bounded slice 1) + +Lane journal for branch `calibration-target-snapshots-908-20260912`. +Journals are history, not state (see CLAUDE.md): check git/GitHub for current truth. + +Historical note, 12 September 2026: the initial implementation below was +subsequently merged onto main `116d46ee9` and repaired through `b8380c475`. +Its original review and source-pin claims describe earlier revisions. +[The maintained guide](docs/calibration-target-snapshots.md) records current +component scope; country integration and issue 908 remain incomplete. + +## State + +Bounded first slice implemented, tested and committed on this branch. Not +pushed, no PR opened. #908 is NOT complete — see "Remaining #908 acceptance +scope" below. + +## Scope of this slice + +1. A shared, leaf-level typed snapshot codec in `microcosm-calibrate`: + schema name + version, aggregate-only per-target rows, ordered target + identity digest, finite-value and ordering validation, signed relative + error with the repo's existing zero-target convention. +2. Real emission from the actual Adam solver loop in + `microcosm.calibrate.solve`, at a configurable bounded cadence with an + explicit every-epoch option, opt-in (off by default). +3. Honest iterate labelling: current / best_retained / selected, with the + selected snapshot taken from the weights the solver actually returns. +4. Atomic local `latest.json` plus immutable, bounded history chunks. +5. A synthetic-only benchmark at US/UK-sized dimensions (invented matrices). + +## Out of scope for this slice (remaining #908 acceptance scope) + +- Version-2 staging upload / remote publication (PR896's country host). +- Calibration Diagnostics UI (different repository, by the issue's own text). +- Real UK/US native calibration measurements — this host is forbidden native + microdata, engine runs and installs, so no real-run cadence default is + announced here. + +## Done + +- `packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py`: + shared leaf codec. Schema name + version, aggregate-only validation + (unknown top-level keys and record-level key names refused at any depth), + compiled-order row check, finite-value check, signed relative error with + the solver's existing zero-target convention, ordered `(name, value)` + sha256 identity digest, `TargetSnapshotCadence` (bounded + `EVERY_EPOCH`), + and `TargetSnapshotWriter` (atomic `latest.json`, write-once history + chunks, bounded retention with recorded drops). +- Solver integration in `solve.py`: emission from the Adam loop and the + proximal loop off the estimate tensor the epoch's loss was already + computed from (no second forward pass, so the hard-concrete gates' RNG + stream is untouched); budget-search probes carry their own search + identity; `l0_selection` / `post_l0_refit` carry their phase; one monotone + sequence counter spans every phase; the closing `selected` snapshot comes + off the float64 estimates the final diagnostics use. +- Public opt-in parameter `target_snapshots=` on `calibrate`, + `refit_l0_selection` and `calibrate_l0_refit`; exports in the shard + `__init__`. Off by default. +- 24 new tests in `packages/microcosm-calibrate/tests/test_target_snapshots.py` + (flat path; `tools/ci_test_groups.py --verify` = ok, lands in fast `rest` + and engine `us-am`, never `[defaulted]`). +- Synthetic benchmark + receipts under `experiments/`, explicitly labelled + synthetic. +- changelog.d fragment `908-calibration-target-snapshots.added.md`. + +## Remaining #908 acceptance scope (NOT done here) + +- Version-2 staging persistence/upload of the snapshot and its history. + PR #896 is open, draft, CONFLICTING and absent from `main`; its + `calibration_progress` filters on `kind == "calibration_epoch"` and its + per-event schema is `additionalProperties: false`, so it needs a new + schema name, not a widened one. Nothing here claims remote publication. +- US host wiring (`tools/build_us_fiscal_refresh_release.py` still builds v1 + `StagingTelemetry`) and UK host cadence flags. +- Exact-k ladder phases in `microcosm.build.us_runtime.exact_k_ladder` and + the UK size-search phases in `uk_runtime.dataset_size` are not wired. +- A test asserting intermediate snapshots never substitute for the canonical + `calibration_diagnostics.json`. +- Real UK/US target/epoch counts, wall-clock, and upload time, and therefore + the production default cadence. Forbidden on this host; the default stays + off. +- Calibration Diagnostics UI (different repository, by the issue's own text). + +## Adversarial review round (same lane, before hand-off) + +Three independent review lenses (determinism, label honesty, codec/store) +raised 21 findings; each was handed to an adversarial verifier told to refute +it, and 10 survived. All 10 are fixed on this branch, with a regression test +each: + +- Enabling the observer aborted runs that succeed without it, in two ways — + duplicate compiled row labels (`row_name` is the lossy `f"{name}@{period}"`, + so `("income", 2024)` and `("income", "2024")` collide) and a non-finite + float32 in-loop estimate (the capped loss absorbs it and the run returns + weights). The digest now carries the row index instead of refusing + duplicates, and a non-finite estimate or target serializes as null with a + `non_finite_rows` count, following `diagnostics._finite`. +- The `selected` snapshot stamped the closing epoch on a retained-best + iterate, and claimed `best_retained.available: false` on a run that + retained and returned a best. Both now read the selection receipt. +- The store silently collided across runs sharing a directory (sequences + restart at 1 per observer) and could prune another run's chunks, or the + chunk it had just written. Construction now refuses a populated directory + unless explicitly adopted, a store refuses a second run's snapshots, and + pruning is scoped to the chunks that writer wrote. +- The aggregate-only scan covered a hand-listed subset of keys; it now walks + the whole payload. Numeric fields are type-checked, so a stringified number + no longer validates. + +## Source identity re-pins (disclosed) + +`microcosm.calibrate.solve` is an attested module: it is in +`_DIRECT_KERNEL_MODULES` (`spec_engine/seeds.py`) and it defines the graph +shard's `calibrate.adam@1` kernel. Editing it legitimately moves several +pinned identities. Each was re-pinned to a value computed from this branch, +after first confirming with the *same* interpreter that pristine `origin/main` +reproduces the committed pins exactly — so the drift is attributable to the +source edit, not to an environment leak: + +- `packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json` + (`implementation_hash`, `node_key`). `fit.qrf` and `simulate` untouched; the + regeneration asserted `graph.json`, `inputs.csv` and `direct.csv` come back + byte-identical, so the kernel's numbers did not change. +- `EXPECTED_HASHES["seed_protocol"]` and `["seed_map"]` in + `spec_engine/inventory_coverage.py`, plus the regenerated + `docs/evidence/spec-engine/us-f0-coverage.json`. +- The resolved-spec golden vectors in + `test_spec_engine_loader.py` and `test_us_multispine_pool_tool.py`. + +No other pin was touched. + +**These pins are correct for this branch's base (0c3f4f651) only.** +`origin/main` has since advanced to 116d46ee9 (#912, +`amend-keyed-seed-and-uniform-draws`), which itself edited the attested +`microcosm.fit.qrf` and re-pinned the same loader golden vector and pool-tool +spec digest. Every identity pin above must therefore be recomputed on the +merge ref before this branch merges, per CLAUDE.md's "CI tests the merge ref, +so merge main and re-pin". Do not treat the values here as final. + +## How this lane ran the tests (no installs, no uv sync) + +An isolated interpreter with this worktree's shard sources ahead of a shared +venv's site-packages, plus an import-path assertion so `microcosm.*` can never +resolve outside this worktree: + +```python +# run.py — invoke as: /bin/python -I -B -S run.py +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VENV_SITE = Path("/site-packages") +sys.path[:0] = [ + *sorted(str(p) for p in ROOT.glob("packages/*/src")), + str(VENV_SITE), + str(ROOT), +] +import microcosm.calibrate as _cal +assert str(ROOT) in str(Path(_cal.__file__).resolve()), _cal.__file__ +import pytest +raise SystemExit(pytest.main(sys.argv[1:])) +``` + +Note pytest `addopts` already carries `-q`, so gate on the exit code rather +than adding another `-q` (which hides the summary line). + +## Next + +- Independent review of this slice, then the staging/host wiring as a + separate slice once #896 lands. diff --git a/PROGRESS-current-asec-income-routing.md b/PROGRESS-current-asec-income-routing.md new file mode 100644 index 000000000..9f76d23d9 --- /dev/null +++ b/PROGRESS-current-asec-income-routing.md @@ -0,0 +1,175 @@ +# Current ASEC income reporting/routing source qualifier — lane progress + +Branch `current-asec-income-routing-20260912`, worktree +`_worktrees/microcosm-current-asec-income-routing-20260912`, base +`d5cbe60b2c6402648565f5139dbf7d94be209def` (reviewed US integration). + +Journal, not state. Check git/GitHub for current truth. + +## Scope + +Deliver a thin **source qualifier** plus a **pure reporting/routing projection** +for the five original-channel ASEC income families the current graph does not +yet supply: pension/annuity, IRA distributions, net property income, farm, and +other income. Consumed later by the already-owned `graph_us_survey_enrichment` +host, which root's active amount owner owns. This lane creates **no** host, no +issuer, no engine execution, no native cell writes, no release claim. + +## State + +- [x] Read the actual-current-graph trace + (`_recovered/scratch-backup/893/codex-takeover-20260912/original-tax-graph-gap-source-trace.md`) + and confirmed the current host does not call `derive_cps_carried_current_leaves` + or the preclone gap-fill. +- [x] Verified the official 2025 dictionary bytes locally: + SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +- [x] Read the accepted UC pattern + (`current_asec_unemployment_source.py`, reviewed hash + `624c76ae058e7b2a466a017eba348327c94751088d97011d3474cd8314187519`) read-only + in the successor worktree; not mutated. +- [x] Read the in-base siblings `current_social_security_source.py`, + `current_asec_demographics.py`, `current_survey_predictors.py`. +- [x] Extracted every target field's printed literal (length/position/range, + verbatim Universe and Values) from the verified dictionary, and + cross-checked each against `asec_current_money_domains_v1.json`: exact match + on position, length, universe and values for all nine money fields. +- [x] Module + `packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py` + (commit 12ac3a75b, corrected in aa7bde7d4 and 7e2c7e77b). +- [x] Tests `packages/microcosm-build/tests/test_us_current_asec_income_routing.py` + — 64 passing, no engine, no PUF fixture. +- [x] Source contract note `docs/us-current-asec-income-routing-source.md` + plus `changelog.d/us-current-asec-income-routing-source.added.md`. +- [x] `ruff check .` and `ruff format --check` clean; + `tools/ci_test_groups.py --verify` reports `verification=ok` and the new + file lands in `[fast] rest` and `[engine] us-am`, not `[defaulted]`. +- [x] Mutation-checked the regressions: default-zero completion of an ambiguous + recipient zero, reading NIU from the raw number instead of the parent + status axis, collapsing receipt-with-net-zero into known nonreceipt, and + resolving an unknown slot account into a known non-IRA zero each turn the + suite red; reverting each returns it green. + +## Second review round: corrections after adversarial review + +An independent five-lens adversarial review of the finished lane (source +fidelity, projection semantics, owner pattern, tests, repository fit) found a +further set of defects, every one of which is fixed on this branch: + +1. **Fail-closed roster (CI-breaking).** `test_us_spine_blindness.py` globs + every `us_runtime` module and fails closed on an unregistered one. The + qualifier is now classified there and listed among the reviewed source-spine + provenance owners. Three of that file's tests remain red for four modules + already unclassified at the reviewed base `d5cbe60b2` — + `graph_fiscal_dense_calibration.py`, `graph_fiscal_measurement.py`, + `graph_puf55_route_attachment.py`, `graph_survey_puf55.py` — which this lane + neither owns nor touched. **Root should route those to their owners.** +2. **Recipient zeros were completed on an unprinted rationale.** `RNT_VAL` and + `FRSE_VAL` print the same `0 = none or niu` label the gross entries print, so + a signed net measure's recipient zero is no longer a known amount. Only + `ANN_VAL`, whose printed zero is `valid_zero_dollars`, resolves; that test is + read from the pinned domains artifact rather than decided in code. +3. **Off-route distribution evidence yielded an affirmative zero.** Off-route + dollars or an answered off-route recipiency now contradict the row, and an + applicable slot declaring an account whose amount is a "none or niu" zero + leaves both the total and the regular-IRA share unresolved. +4. **Other-income routing ignored the printed receipt universe**, so an + out-of-universe row read as reported alimony. +5. **Allocation origins asserted publisher-confirmed non-allocation** from flags + whose printed universes are conditional and unevaluated. +6. **The printed zero receipt label was collapsed** to one shared `niu_or_none` + across nine entries that print three different labels. +7. **`asec_literals` escaped with no digest**, outside the final identity check. +8. **`I_FRMYN` prints an empty `Values:` block**; its codes were described as + following `I_ANNVAL`. **`FARM_AMOUNT_SCOPE` dropped its composite clause.** + The claim that every other family prints a 15+ floor was false — the farm + family prints none. +9. **The doc called `TAXABLE_PENSION_FRACTION` archived**; it is live in + `cps_carried_current.py` under the prepared-ASEC stage, so a second split + attached here would double-count. +10. Printed entry tables are `NamedTuple`s now, the account code domain is + compared against `retirement_distributions._VALID_ACCOUNT_CODES` rather than + a re-typed literal, and `project_income_routing` validates its routing token + arrays. + +## Corrections made after independent review of the first draft + +1. The money owner normalizes `ANN_VAL`'s printed `-1` to a stored zero and + records `DECLARED_NIU` (`asec_current_money.py:974-977`). The first draft + re-derived NIU from the stored number, which would have read that cell as a + zero dollar annuity and also failed the literal-identity join. The amount + reading now comes from the parent's status axis. +2. The nine printed money entries were retyped by hand; they are already + attested per vintage in `asec_current_money_domains_v1.json`. They are now + read from that packaged artifact under `money.RESOURCE_PINS[0]`. +3. The published-allocation roster wrongly listed `DST_VAL1`, `DST_VAL2`, + `DST_YN`, `DST_SC1`, `DST_SC2` and `FRMOTR` as unflagged; all six are + flagged. `OI_YN` is unflagged and was missing from the list. +4. Three printed universes were transcribed with ASCII `>=` where the + dictionary prints U+2265; they now carry the printed character. + +## Next + +Root review and integration into `graph_us_survey_enrichment`. The open +decisions are listed under "Remaining work" in the contract note: canonical +attachment and clone policy for these PUF-overlapping leaves, the ACS clone0 +conditional model and its reconciliation against the ACS aggregate anchors, the +unobserved pension and distribution tax composition, the net property +decomposition, and any other-income residual rule. + +## Verified source literals (2025 dictionary, pages 43-49 + allocation pages) + +All quoted verbatim from the pinned PDF. + +| Field | Len | Pos | Range | Universe as printed | +| --- | --- | --- | --- | --- | +| PNSN_VAL | 7 | 571 | (0:9999999) | `PEN_YN = 1` | +| PEN_YN | 1 | 570 | (0:2) | `All Persons aged 15+` | +| ANN_VAL | 6 | 438 | (-1:999999) | `ANN_YN = 1` | +| ANN_YN | 1 | 444 | (0:2) | `All Persons aged 15+` | +| DST_VAL1 | 6 | 495 | (000000:999999) | `DST_SC1 = 1` | +| DST_VAL1_YNG | 6 | 501 | (000000:999999) | `DST_SC1_YNG = 1` | +| DST_VAL2 | 6 | 507 | (000000:999999) | `DST_SC2 = 1` | +| DST_VAL2_YNG | 6 | 513 | (000000:999999) | `DST_SC2_YNG = 1` | +| DST_SC1 | 1 | 491 | (0:7) | `DST_VAL1 > 0 and a_age >= 58` | +| DST_SC1_YNG | 1 | 492 | (0:7) | `DST_YN_YNG = 1 and a_age < 58` | +| DST_SC2 | 1 | 493 | (0:7) | `DST_VAL2 > 0 and a_age >= 58` | +| DST_SC2_YNG | 1 | 494 | (0:7) | `DST_VAL_YNG > 0 and a_age < 58` | +| DST_YN | 1 | 519 | (0:2) | `Persons aged 58 and over (a_age >= 58)` | +| DST_YN_YNG | 1 | 520 | (0:2) | `Persons under age 58 (a_age < 58)` | +| RNT_VAL | 6 | 621 | (-9999:999999) | `RNT_YN = 1` | +| RNT_YN | 1 | 627 | (0:2) | `All Persons aged 15+` | +| FRSE_VAL | 7 | 390 | (-9999999:9999999) | `ERN_YN=1 or FRMOTR=1` | +| FRSE_YN | 1 | 397 | (0:2) | `ERN_YN=1 or FRMOTR=1` | +| ERN_YN | 1 | 381 | (0:2) | `WORKYN=1 OR WTEMP=1` | +| FRMOTR | 1 | 389 | (0:2) | `ERN_OTR = 1` | +| OI_VAL | 6 | 549 | (0:999999) | `OI_YN = 1` | +| OI_OFF | 2 | 547 | (0:20) | `OI_YN = 1` | +| OI_YN | 1 | 555 | (0:2) | `All Persons aged 15+` | + +Source-level questions that stay open (evidence, not modeling judgments): + +- `PNSN_VAL` is printed as "total combined amount of pension income received + from **all** pension sources". It is not an observed private/taxable amount. + The legacy 0.590 split is a modeled assumption and is not applied here. +- `RNT_YN`'s printed question covers rent, royalties, roomers/boarders **and** + estates or trusts; `RNT_VAL`'s printed question asks only about "income from + rent after expenses". The receipt and amount questions do not have the same + printed coverage, so the total is not independently labelled rental. +- `DST_SC*` code 4 is `Regular IRA`. Account identity does not observe a taxable + fraction, so no taxable component is derived. +- `DST_VAL1`'s printed universe is `DST_SC1 = 1` (401k account) although its + label is the source-1 distribution amount; preserved verbatim as an + unresolved printed-universe ambiguity rather than silently corrected. +- `DST_SC2_YNG`'s printed universe names `DST_VAL_YNG`, a field with no + dictionary entry; preserved verbatim. +- `OI_YN`'s printed `0` label is `none or niu`, unlike `PEN_YN`/`ANN_YN`/ + `RNT_YN`/`DST_YN` whose `0` is `niu`. Zero receipt literals are therefore not + interchangeable across the five families. +- Published allocation flags exist for `ANN_VAL` (`I_ANNVAL`), `ANN_YN` + (`I_ANNYN`), `PEN_YN` (`I_PENYN`), `RNT_VAL` (`I_RNTVAL`), `RNT_YN` + (`I_RNTYN`), `OI_VAL` (`I_OIVAL`), `ERN_YN` (`I_ERNYN`), and the DST + composites (`I_DSTSC`, `I_DSTSCCOMP`, `I_DSTVAL1COMP`, `I_DSTVAL2COMP`, + `I_DSTYNCOMP`). The dictionary publishes **no** direct flag for `PNSN_VAL`, + `FRSE_VAL`, `FRSE_YN`, `OI_OFF`, `DST_VAL*`, `DST_YN` or `FRMOTR`; their + allocation provenance is therefore recorded as unresolved, not as "raw". +- `I_DSTVAL1COMP`'s printed `Universe:` line is empty in the dictionary. diff --git a/PROGRESS-native-byte-transports.md b/PROGRESS-native-byte-transports.md new file mode 100644 index 000000000..076195c5a --- /dev/null +++ b/PROGRESS-native-byte-transports.md @@ -0,0 +1,111 @@ +# Lane journal: the US native build's byte-transport ceilings + +Branch `native-byte-transports`, from `native-row-ceilings` (PR #949's head, +`c5ac78d2d`). Worktree `~/PolicyEngine/_worktrees/microcosm-native-byte-transports`. +Opened 2026-09-18. Session-handoff journal: accurate when written, historical +afterward (see `CLAUDE.md`, "Root journals are history, not state"). + +Design authority once written: `docs/us-native-byte-transports.md`. Lane report: +`experiments/native-byte-transports/out.md` (root `out.md` is another lane's +tracked report and is not touched). + +## State (fifth session, 2026-09-18, from 17:20) + +The fourth session (16:20-16:17+, ended on its limit) committed the remaining +whole-roster transports, the budget view's streamed comparison, the agreement +sweep and the 1/15 harness copy; its salvage ref held only the untracked raw +census and pytest logs, both now in the tree. This session verified the two +test files that failed before the fix commit (73 and 54 passed at +`475c9ba9d`), found no pin moving, and then closed what the raw census and a +coverage grep showed open: the child lane's `graph_joint_empirical.MAX_RECIPIENTS` +(1,048,576, a row count over every stacked person, the first refusal on that +lane above about 1/3 of source) takes the row-count rule's 15,000,000; the +three `raise ... from None` sites still in `survey_population_preparation` +(`REQUEST_REFUSED`, two `PREPARATION_VERIFICATION_REFUSED`) chain their cause +like the seven already converted; and every moved bound now has a test that +drives its refusal at a patched-down value and its acceptance at a +full-source-sized count without allocating one (numeric bounds, budget +consumer, age artifact, predictor projection, `MAX_ORIGINALS`, +`MAX_RECIPIENT_ROWS`, `MAX_RECIPIENTS`), plus the byte family's rule table in +`test_us_native_row_ceilings.py`. The 1/15 harness copy keeps the retention +lane's file with three named edits: the venv is an argument (this tree's lock +moved), and the two receipt fields that lane found stale (`sample.fraction`, +`scope`) say 1/15. A 59-agent verification workflow (43 refuters, 13 batch +checkers, 3 completeness critics) is running read-only over the raw census. + +Still open at this entry: the verified census table, the design note, the +1/15 run (detached worktree `microcosm-byte-transports-fifteenth` and its +venv are built), the dependent-file battery at the finished head, the report, +the draft PR. + +## State (third session, 2026-09-18) + +The moved base (`origin/native-row-ceilings` at `0319b1c73`, merged level with +main) is merged in at `77bafbb3a`; the venv is resynced to the merged lock +(policyengine-us 2.2.1). The second session's uncommitted edits were committed +unchanged (`2ed7b0715`) and then tested: two of its new tests failed, both +fixed in the commit after this journal entry -- the digest and segmented +encoders no longer require the segment to be at most the maximum (a digest has +no accumulation), and the budget's `checked_view` no longer re-encodes the +whole document through the 64 MiB encoder (`graph._json_matches` streams the +comparison). Pins re-derived after the merge: 124 inventory contracts and all +four `_ACCEPTED` entries unchanged, ten stage manifests build. A census +workflow (nine module families, adversarial verification, completeness critic) +is running read-only over this head; no source file moves until it returns. + +Earlier state, first session, at `c5ac78d2d`: both prior reports read (#949 row ceilings, +#950 retention seal §8/§11), the transport design note, the prior census +(146 bounds, 91 byte-shaped rows), and every module that holds a named byte +bound on the 19-node and 45-node paths. `uv sync --all-packages --locked +--extra us` built `.venv` (Python 3.14.4; `uv.lock` identical to the +`microcosm-native-v5` venv the prior harnesses ran on). Baseline gates at the +untouched head: `tools/ci_test_groups.py --verify` ok; spec-engine coverage +42156/42156 and 41/41. + +## Done + +- Read at this head: `acs_person_coverage_authentication.py` (whole), + `acs_native_coverage_binding.py` (whole), the roster transport in + `survey_population_preparation.py` (`_roster_segments`, `_spill_roster`, + `_roster_payload`, the three `raise ... from None` sites), `_bounded_json` + and `_allocation_payload` in `graph_survey_population.py`, + `survey_origin_budget._document`, `graph_survey_budget`, + `graph_survey_calibration`, `graph_survey_age_artifact`, + `puf_diagnostic_consumer.current_survey_recipient_matrix`, the housing + prepared receipt, the child-property projections, and the fit adapter's + draw document. +- Established reachability: the 45-node pilot ran with household roles and + person status disabled (`_roles_disabled_pair`; pilot report §"The 45-node + graph, derived from the code"), so the roles projection and status artifact + are reachable only from the 49/51-node variants. The origin budget, numeric + bounds and age artifact run in `survey_age_calibration`, not in either + graph, but the origin budget is one of the three the brief names, and the + numeric bounds consume its payload. + +## Done (third session) + +- Merged the moved base; resynced; committed and tested the second session's + edits (ACS 116 passed with the row ceilings; child property 61; graph survey + population 70 after the fix; origin budget 54 after the fix; spine blindness + 503; preparation 75; calibration and age 67; fit adapter, child source and + predictors 95). +- Found and closed a consumer gap the second session missed: the budget's + `checked_view` re-encoded the whole document through `_bounded_json` + (`survey_origin_budget.py`, `FINAL_BUDGET_VIEW_DOCUMENT`), which would refuse + at the same 5.7% of source the producer used to. +- Wrote the census brief (`experiments/native-byte-transports/census-brief.md`) + and the seeded agreement sweep (`agreement_sweep.py`). + +## Next + +0. Act on the census: any bound it finds binding that this branch has not + lifted; then pins, the design note, the report, the 1/15 run from a + detached worktree at the finished head, the draft PR. +1. (first session's plan) Byte census: measurement script over the recovered 1/1000 artifact and the + captured public ACS archive; agent fan-out per module family with an + adversarial pass over every binding verdict; `census.json`. +2. Commit 1: the ACS serialno and body budgets. +3. The preparation consumer, the origin budget, the numeric bounds, the age + artifact, the diagnostic matrix; catch-alls carry their cause; pins. +4. Tests as CI runs them; the 1/15 run; the design note; the report; draft PR + against `native-row-ceilings`. diff --git a/PROGRESS-native-row-ceilings.md b/PROGRESS-native-row-ceilings.md new file mode 100644 index 000000000..fe0985017 --- /dev/null +++ b/PROGRESS-native-row-ceilings.md @@ -0,0 +1,95 @@ +# Lane journal: lift the native build's row-count ceilings (`native-row-ceilings`) + +Branch `native-row-ceilings`, from `origin/native-scale-transport` at `a64f7b733`. +PR base is `native-scale-transport`, draft, and it stays draft. + +This file is a session-handoff journal. Per the repo's CLAUDE.md, its +"State"/"Next" sections are accurate when written and historical afterward — +check git and GitHub for current truth. + +## The rule (the one argument) + +**A bound moves only if a full-source native build meets it, and a bound that +moves becomes four times the measured full-source count of exactly what it +counts, rounded up to the next whole million.** + +Byte transports are *not* this rule's. The transport lane's answer to a byte +ceiling is a segmented stream under `MAX_ROSTER_BYTES`, not a larger single cap, +and applying this rule to one would be the wrong argument. + +## The counts are measured, not extrapolated + +The recovered 1/1000 pilot artifact +(`_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json`, +sha256 `34b362d8…`) carries the whole ACS and ASEC **catalogues**, which do not +scale with the selection fraction. Its ACS occupied + institutional-GQ + +noninstitutional-GQ households plus its ASEC households equal its own +`selection.supplied_households` exactly (1,348,408 + 84,422 + 98,784 + 55,762 = +1,587,376), so "the whole catalogue" and "what a full-source selection supplies" +are the same set: + +| | households | persons | +|---|---|---| +| ACS | 1,531,614 | 3,422,888 | +| ASEC | 55,762 | 142,125 | +| stacked | 1,587,376 | 3,565,013 | +| combined clone | 3,174,752 | 7,130,026 | + +The transport lane's ~3,471,000 stacked / ~6,943,000 cloned were the 1/1000 +per-household ratio extrapolated; these supersede them. +`experiments/native-row-ceilings/roster_census.py` re-derives them and asserts +the reconciliation. + +## Done + +- **Census**: 146 bounds across five module families, reachable from the 19-node + financial graph and the 45-node pilot graph, each with its enforcement site, + refusal code and exception type, what it protects, its counts at 1/10 and at + full source, and whether it binds. 41 of those verdicts then went through an + adversarial pass; no headline binding call was overturned. + **14 bind at full source, 6 of them at 1/10.** +- `survey_origin_budget.MAX_GROUPS` **established**: `allocation_instructions` + requires one instruction per selected household, so a full-source budget has + 1,587,376 groups against a 1,000,000 bound. +- **Seven ceilings lifted** under the rule, each with a boundary test. +- Pins re-derived through their generators: one moved + (`acs_native_coverage_binding._ACCEPTED["acs_pums.py"]`). 124 inventory + contracts checked, 10 stage manifests built. +- **An inherited break re-pinned**: base-branch `b6081efcb` added + `path.read_bytes()` to `_spill_roster` without regenerating + `survey_population_preparation.py`'s `resource_accesses_sha256`, so + `implementation_manifest()` raised for every stage containing it. Proven + pre-existing by recomputing the contract from `origin/native-scale-transport`, + `a64f7b733` and `b6081efcb`'s own blobs, and by running the affected tests + against a base worktree: 7 fail there with that exact error and all pass here. +- `docs/us-native-row-ceilings.md`, and the rule as an executable table. + +## The three findings the report leads with + +1. **`acs_person_coverage_authentication.MAX_BODY_BYTES` admits 12,911 selected + ACS persons — 0.38% of source.** Measured over 200,000 real records of the + pilot's captured public archive. The tightest ceiling on the path, 265× under + at full source, and below 1/100. +2. **The preparation-receipt ceiling the transport lane lifted is still enforced + one module downstream** at `PREPARATION_MAX_BYTES` = 64 MiB, which admits + 96,839 households — to within rounding the exact 96,860 that lane reported as + lifted. +3. **`survey_origin_budget.MAX_PAYLOAD_BYTES` admits 87,838 households, 5.53%**, + and cannot be raised at all in that module: `graph._bounded_json` refuses any + limit above 64 MiB before encoding a byte. + +All three are byte transports and take the transport lane's argument, not this +one. All three are pinned in tests. + +## State + +Complete. **[PR #949](https://github.com/PolicyEngine/microcosm/pull/949)**, +draft against `native-scale-transport`, MERGEABLE. CI does not run on it by +design (`test.yml` triggers on `pull_request: branches: [main]`); the local +gates in the report's section 8 are the only ones it has, and all are green: +399 passed over the seven touched test files, `verification=ok`, spec-engine +42156/42156 and 41/41, ruff clean, all three pin generators idempotent. + +## Next +- Open for Max: whether the inherited re-pin stays here or moves to #945; and + whether the byte transports above are one follow-up lane or several. diff --git a/PROGRESS-native-scale-transport.md b/PROGRESS-native-scale-transport.md new file mode 100644 index 000000000..fba72b90d --- /dev/null +++ b/PROGRESS-native-scale-transport.md @@ -0,0 +1,134 @@ +# Lane journal — native scale: transport ceiling and per-node retention + +Branch `native-scale-transport`, worktree +`~/PolicyEngine/_worktrees/microcosm-native-scale`, stacked on +`native-verify-once` (PR #935). Draft PR only; never ready, never merged. + +This file is a session-handoff journal in the sense CLAUDE.md means: accurate +when written, history afterwards. Check git/GitHub for current truth. + +## State + +**Started 2026-09-17.** Orientation complete; baseline run not yet launched. + +## Charter, in one line + +Lift the two hard limits the 2026-09-16 profiler found — the 64 MiB +whole-roster receipt transport (refuses above 6.1% of the US source) and +per-node full-population retention (~300 GiB at full source) — then prove a +build above the old ceiling at 1/10. + +## Sites, line-cited in this tree (not the profiler's tree) + +| what | where, at `5ff889814` | +|---|---| +| preparation payload limit | `survey_population_preparation.py:50` `MAX_PAYLOAD_BYTES`, `:104` `_encode`, `:108` `PAYLOAD_LIMIT` | +| the whole-roster document | `survey_population_preparation.py:1698-1699` (`"selection": _plan_document(plan)`, `"origins": origins`) | +| second origin budget | `survey_population_preparation.py:816-818` `_bounded_append` → `ORIGIN_LIMIT` | +| allocation payload limit | `graph_survey_population.py:62` `ALLOCATION_MAX_BYTES`, `:344-369` `_allocation_payload` | +| per-cell frame identity | `survey_population_preparation.py:601` `_frame_cell_encode`, the `_frame_identity` loop | +| the observer that retains 19 populations | `graph_atomic_survey_financial.py` `observe(...)`, called from `executor.py` `_observer_snapshot` | + +## Done + +- Read the profiler's §5, §9, §10, §11 and the verify-once report's measurement + section. Baseline to beat: **2,010.07 CPU-s / 2,016.08 s wall / 13.31 GB peak + RSS** for the 19-node financial graph at 1/1000 (verify-once file C). +- Confirmed no live microcosm build on the machine; 128 GiB / 18 cores. +- Located the 19-node harness: + `~/PolicyEngine/_worktrees/microcosm-verify-once/.measure/harness19_verify_once.py`. + +## Next + +1. Baseline run of the 19-node harness at 1/1000 in a throwaway worktree at the + base commit, with an RSS-over-time sampler and runner-vs-node-loop split. +2. Design note `docs/us-native-scale-transport.md`. +3. Implement, with refusal tests, identity-equality proofs, replay proof. +4. 1/10 run above the old ceiling. +5. CI-shaped battery, draft PR. + +## Session log, 2026-09-17 + +### Baseline (requirement 1) — done, and it settles the profiler's open question + +One uncapped-by-node run of the 19-node financial graph at 1/1000, on the +branch point `5ff889814`, in the throwaway worktree +`~/PolicyEngine/_worktrees/microcosm-native-scale-baseline`, pid 70342 / pgid +70342 (session leader), RLIMIT_CPU hard 9,060 s, harness soft 9,000 CPU-s / +12,000 wall-s / 48 GiB, launched only after `vm_stat` showed 69.9 GB available. +Status `COMPLETED_NINETEEN_NODE`. + +| | | +|---|---| +| wall | 2,048.93 s | +| CPU | **2,028.69 s** | +| peak RSS | 13,157,728,256 B = 13.16 GB / 12.25 GiB | +| loadavg | start [6.68, 6.97, 6.40], end [10.75, 9.68, 8.66] | +| runner call | 2,045.91 s wall / 2,026.78 s CPU; 3.02 s of process wall sits outside it | +| financial node loop, 19 nodes | 277.36 s | +| prefix node loop, 9 nodes | 55.93 s | +| **runner outside both node loops** | **1,712.62 s = 83.7% of the runner call** | + +The four ASEC predictor nodes are 267.99 s of the financial loop's 277.36 s; +the other fifteen are 9.37 s. The retention slope is **not visible** in the RSS +trace at this fraction: 19 snapshots at the measured 9.6–9.8 B/cell over +1.70e6 cells is 0.31 GiB, inside the trace's own 1–2 GB sawtooth (peak 13.16 GB +at t=1303 s, 5.92 GB at exit). That corroborates the report's §9 rather than +adding to it. + +### Done since + +- Vectorised per-column frame seal in `survey_population_preparation`, proved + byte-for-byte against the predecessor traversal oracle that + `test_us_survey_frame_identity_encoding.py` already carried (160 tests; the + file's own runtime fell 7.16 s → 0.67 s). +- Segmented, content-addressed transport for all four whole-roster receipts. + `docs/us-native-scale-transport.md` has the design, the five measured ceilings + in binding order, and the two refusals that narrow. +- 1,013 line citations across seven mechanisms read and adversarially verified. + +### Next + +1. After-run at 1/1000 on this head — **queued behind a memory gate** (another + lane's `smoke.py`, pid 68076, holds 52.7 GB; the waiter polls every 60 s). +2. 1/10 run above the old ceiling. +3. Replay proof, pins, CI-shaped battery, draft PR. + +### 2026-09-17, later + +**Landed:** the segmented transport for all four whole-roster receipts, the +vectorised per-column frame seal, the hardening of the spill path, the ceiling +receipt, the seal rate receipt, the row-count ceiling census, and draft PR +[#945](https://github.com/PolicyEngine/microcosm/pull/945) against +`native-verify-once`. + +**Measured:** + +| | | +|---|---| +| ceiling, full source | single `_encode` **REFUSES** `PAYLOAD_LIMIT`; the segmented transport carries 1,099,892,722 B in 17 segments, digest equal to the stream digest | +| ceiling, 1/10 | single `_encode` **REFUSES**; transport carries 109,804,304 B in 2 segments | +| seal, blended on the frame's own dtype census | 0.5195 → 0.0677 µs/cell, **7.68×**, bytes asserted equal per column first | +| pins | **none move**: every inventory contract matches the tree, all ten stage manifests build | +| graph suite | 778 passed, 1 skipped | +| touched `us_runtime` tests | 359 passed | +| frame identity file | 187 passed (was 160), 15.62 s | + +**Zero `packages/microcosm-graph` hunks**, so nothing in this PR is main-only. + +**Not implemented, by decision and with the mechanism recorded:** the memory half +of the retention. `store.py` refuses a store-backed population snapshot three +ways (masked-storage zeroing at `:456`, `MultiIndex` at `:619`, `CategoricalDtype` +at `:486-487`), object identity is pinned at +`graph_atomic_survey_financial.py:1897`, and `same_replayed_population` needs the +whole object while the replay cannot exist until `run_graph` has returned. So +nineteen detached populations must live in RAM or on disk between observation and +comparison unless the object comparison becomes a content seal — a +verification-contract decision, put to Max as a question. +`docs/us-native-scale-transport.md` §4 specifies the executor mode item by item. + +### Next + +1. The 1/10 graph run (worktree `microcosm-native-scale-tenth`, launcher waits + for 70 GB), after the 1/1000 after-run's required replay finishes. +2. Fold the after-run and 1/10 numbers into the report and the PR body. diff --git a/PROGRESS-spm-composition.md b/PROGRESS-spm-composition.md index 15fe34fb2..e0037c7b4 100644 --- a/PROGRESS-spm-composition.md +++ b/PROGRESS-spm-composition.md @@ -1,5 +1,13 @@ # SPM composition preflight + role for a fresh base (#893 lane) +> **Historical integration note (2026-09-23).** This journal arrived through +> main's merge into `native-integration-20260923`. Every branch, PR/CI, "Next", +> "Remaining", and "Verified at this head" statement below describes its +> original lane snapshot, including later continuations. Recorded measurements +> are not validation of the merged integration tree. See +> [the integration record](docs/native-integration-20260923.md) and git/GitHub +> for current integration status. + Branch `spm-composition-preflight`, cut from `origin/main` at `d1196af10`. ## State diff --git a/PROGRESS-us-canonical-household-roles.md b/PROGRESS-us-canonical-household-roles.md new file mode 100644 index 000000000..4491117af --- /dev/null +++ b/PROGRESS-us-canonical-household-roles.md @@ -0,0 +1,57 @@ +# US canonical household-role fragment — lane journal + +Lane: `us-canonical-household-roles-20260913`, based on integration +`a96fc850942d3fd96d24251c07d9ebc38b79f8de`. Independent parallel US +source-only build. No push, no merge, no runtime staging, no test execution. + +## Scope + +Bounded canonical household-role fragment for the current Microcosm US graph: + +- A thin **qualifier** over the genuine retained ACS/ASEC preparation/issuers. +- A **common graph fragment** mapping exact original ACS/ASEC source + identities onto both initial clone families, preserving int64 ids, + household relationships, assigned geography, weights, provenance and every + other Frame cell. +- `is_household_head` bound **only** where the actual documented source + definition matches. Ambiguous source roles stay unbound and explicitly + unknown. +- Independent-minor roles: **source/consumer scope audit only** unless an + existing explicit canonical contract plus genuine source authority + establish the mapping. + +Out of scope (owned elsewhere or deferred): canonical sex, raw +disability/student observations, housing participation, native restoration, +parent-link preservation, wiring the shared country host. + +## Original lane checklist (historical) + +- [x] Worktree/branch/base verified; AGENTS.md + CLAUDE.md read. +- [ ] Trace maintained source owners + canonical `is_household_head`. +- [ ] Mapping/unknownness specification. +- [ ] Qualifier module. +- [ ] Graph fragment module. +- [ ] Pure/graph/fixture tests (written, not run). +- [ ] CI source classification. +- [ ] Docs + changelog fragment. +- [ ] BUILD-RESULT.md. + +## Original next step (historical) + +Trace owners with a parallel read sweep over the ACS/ASEC source runtime. + +## Source-only continuation, 2026-09-13 + +The external builder stopped after commit `593aa157a46d4736238b4535ed3f9d9eb6dd7a52`, containing a qualifier module without graph/tests/docs. Its commits are preserved. The continuation rechecked the clean assigned branch and fetched main: 239 commits ahead, zero behind before further edits. + +Review found the declared live fence was never called by the qualifier. The continuation checks it around every owner callback, binds individual column aliases, and independently reconstructs the final projection after owner I/O. Root also explicitly confirmed that unsupported known incumbents must refuse at canonical binding, including artificial group-quarters heads; reconciliation retains the original evidence for adjudication. + +The source continuation adds a common two-node graph fragment, pure/graph/actual-owner tests, the mapping and independent-minor audit in `docs/us-current-survey-household-roles.md`, and a changelog fragment. No country-host wiring, housing routing, independent-minor calculation, source staging, project import, pytest, native or installed-model read, push or release is performed. The packet's source pins and bounded runtime proposal record the final scope; written tests remain unrun until separately authorized. + +Source AST parsing/compilation, Ruff formatting/lint, diff whitespace and CI inventory verification pass. The inventory contains 122 planned cases (58 pure, 50 graph, 14 actual owners), not executed results. All three new files receive the existing fast/rest and engine/us-am/build classification; no CI classifier change is required. + +## Independent-review corrections, 2026-09-13 + +Fable reviewed exact `8a3226c37a2b0a3e9aee0090f901bfdca4efd549`. The original commit, review and source inventory remain preserved. The concrete graph-test receipt key is corrected from `acs_allocation_provenance` to the actual `relationship_allocation_provenance`. Five additional pure cases cover non-nullable Boolean preservation/refusal, invalid incumbent dtypes and the published current ASEC maximum roster line. Five additional real-owner cases exercise detached ASEC native/line/household mismatches and ACS household/retained-line coverage failures without modifying the issuer, original source or owner buffer. + +The source review's proposed legitimate ASEC line above 20 is not supported by the exact 2025 dictionary: A_LINENO is printed 01–16. No production bound is widened. The docs explicitly distinguish the operative 64 MiB serialized-artifact cap, the qualifier row ceiling and the ASEC owner's 600,000-row cap; no national capacity is asserted. The revised planned roster is 132 cases: 63 pure, 50 graph and 19 actual-owner cases (A113/B19). All remain unrun. The two production modules are unchanged. diff --git a/PROGRESS.md b/PROGRESS.md index abec1b7be..bcfe6d5eb 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,3 +1,232 @@ +# Retention: replay by content seal, not by retained population + +> **Historical (2026-09-23).** This journal was written on +> `native-retention-seal` (#950). That branch is now merged into +> `native-integration-20260923`, which also carries current main, so its +> "State", "Pull request" and "Next" sections describe #950 as it was, +> not the current tree. See docs/us-native-retention-seal.md, "Integration +> note", for how the seal and compact retention were combined. + +Lane: `native-retention-seal`, off `origin/native-scale-transport` at +`a64f7b733`, worktree `~/PolicyEngine/_worktrees/microcosm-native-retention`. +Started 2026-09-17. Everything below the `---` rule at the end of this section +is prior-lane history; see "Root journals are history, not state" in +`CLAUDE.md`. + +## State + +Implemented; both required runs have run. The 1/1000 cold run **and its +required replay completed** and export byte-identical bytes (identical manifest +key, 0 store paths added or changed), at 1,714.92 CPU-s against the transport +after-run's 1,907.58. The **1/15 run refused** 605.65 CPU-s in, at 8.28 GB, +having run no node: `PREPARATION_ISSUANCE_REFUSED`, the catch-all that discards +its cause. The predicted cause is a 1 MiB canonical-JSON cap on the +selected-ACS-`SERIALNO` list — **below** the row-count ceiling the base branch +lifted, and in no census — and a non-invasive 1/15 diagnostic is running to +name it rather than infer it. The discrimination battery is 116 comparisons, +116 agreements, 0 disagreements. Draft PR #950 is open and stays draft. + +## Done + +- Verified worktree/branch/head; synced `uv sync --all-packages --locked --extra us`. +- Read both authorities in full and the code at this head. +- `docs/us-native-retention-seal.md` — the four required answers. +- `survey_population_replay.py` — the purpose-built seal, beside the + comparison it replaces. +- `test_us_survey_population_replay.py` — every mutation now runs through BOTH + the object comparison and the seal, and must reach the same verdict with the + same code; plus the discriminations the first battery missed. +- `executor.py` — `run_graph(_population_observer_detach=False)`, its own + commit, four tests, no US import. +- `graph_atomic_survey_financial.py` — declared-consumer retention, both + comparisons sealed, `_node_population_stamp` on both arms. +- Repaired a **stale implementation contract inherited from the transport + branch** and added the guard that would have caught it. +- Armed both runs with their gates and their source trees, and ran both. +- The 1/1000 cold run plus required replay, under a byte-identical copy of the + transport lane's committed harness: `COMPLETED_NINETEEN_NODE_AND_REQUIRED_REPLAY`, + every node a store hit on the replay, manifest key identical, store bytes + identical. Recorded with the three causes that move the key off the base + branch's `bd511d92…`, one of which is this lane's own executor commit. +- The 1/10 refusal diagnostic, three rounds, the third non-invasive: the cause + the three nested catch-alls discard is + `ACSCoverageAuthenticationError: CANONICAL_SIZE`, and the ceiling it belongs + to was computed from measured inputs and committed **before** the 1/15 run. +- The 1/15 run: refused as predicted, recorded with its CPU, wall, peak RSS and + the fact that no node ran. +- Found and recorded a defect in the 1/15 harness's own receipt: `sample.fraction` + and the `scope` sentence were left as the tenth harness's while the call site + ran `Fraction(1, 15)`. The harness stays as it ran; the 1/15 diagnostic + harness corrects both fields and says so. + +## Pull request + +**[PolicyEngine/microcosm#950](https://github.com/PolicyEngine/microcosm/pull/950)** +— draft, base `native-scale-transport`, `MERGEABLE`. It stays draft. + +## Done (2026-09-17, second session) + +- **The 1/15 refusal is named from the run's own raises.** The measurement + refused with the catch-all; `harness19_diag_fifteenth.py` (the 1/15 harness + plus the committed `sys.monitoring` RAISE block, diffed both ways) re-ran it + and the trace ends in `ACSCoverageAuthenticationError: CANONICAL_SIZE` from + `issue_acs_native_coverage` with exactly two `visit` frames — the same guard + and signature as 1/10. The prediction committed before the run is confirmed. +- Recorded a defect in the 1/15 harness's own receipt: `sample.fraction` and + the `scope` sentence were left as the tenth's while the call site ran + `Fraction(1, 15)`. The harness stays as it ran; the diagnostic corrects both. +- Corrected the guard's citation: `acs_native_coverage_binding.py:563-565`, + not `:562-564`, in the report and the ceiling receipt's generator. +- **Pinned five of the six refusal codes this change adds**, which had none, + and verified each of the three seal tests goes red against its own reverted + guard. `ATOMIC_OBSERVER_RETENTION` stays unpinned and both documents now say + why, including that its second conjunct is a self-comparison that cannot + fire. That is report question 7. +- **`battery_receipt.py`** rebuilds the battery's receipt from its own rows and + reproduces the committed one exactly; the design note's stale 109 comparisons + and eleven accepted pairs are corrected to 116 and fourteen. +- **The identity receipt now emits every ordered pair.** Section 7c had cited + `transport_after → us_only` for "the US half moves no key", and that row says + the opposite; `base → us_only` is the row, and it did not exist. Regenerating + at this head also re-ran the contract arm: ten of ten accepted. +- Verified, rather than assumed: both measured trees carry `packages/*/src` + byte-identical to this head; §7a's table regenerates cell for cell; §7b's + replay proof and §7d's RSS phase split match the artifacts exactly; the §2 + battery table matches the receipt row for row; and the base branch's + `bd511d92…` key is the one the brief quoted. +- Report restructured to the brief's order — tests (9), PR (10), questions (11), + caveats (12) — and five wrong cross-references repointed, including §12's + claim that the committed 1/10 trace came from a round that patches the + catch-all. It came from the round that patches nothing. + +## Done (2026-09-17, second session) — the adversarial pass + +**68 agents over six review dimensions, every finding put to two independent +verifiers whose default answer is refuted: 31 findings, 25 survived.** Five +answered in code, four in tests, sixteen in the documents, none left open. +Receipt: `experiments/native-retention-seal/adversarial-verification-receipt.json`. + +The three that mattered, each reproduced before being believed and again after +being fixed: + +- **The seal ACCEPTED a pair the comparison refuses.** An object-axis value + carrying its own `__eq__` has equal codec bytes to the plain int of the same + value and is not `==` to it. That is the dangerous direction — a run + accepting a replay the comparison refuses. +- **The seal REFUSED a pair the comparison accepts.** pandas holds `None` and + every NaN interchangeable on an object axis; the codec spelled them apart, so + a green run would have turned red. +- **A masked-integer axis folded through float64**, because `np.asarray` on a + masked integer array returns float64 with NA as NaN — lossy above `2**53`, + and the refusal code moved. + +The fold now reproduces the equivalence classes `Index.equals` actually has, +**measured** rather than read off pandas' source, and refuses `AXIS` at seal +construction for the one case a digest cannot represent. 124 new battery cases +pin all three; each half of the fix was reverted to confirm its own cases go +red. + +**And the change broke a gate that passes at the base.** +`test_us_spine_blindness.py::test_runtime_population_operators_are_source_spine_blind` +refuses a statically unresolvable subscript on a name it infers to be a column +container; every seal record is a positional tuple, so it reported 48 sites. +Verified both ways: `1 passed` at `a64f7b733`, red here. The records are now +unpacked into named fields, the one dynamic `getattr` reads through a literal +reader per comparable, and the gate passes (`1 passed in 217.42s`). Nothing +moved: no record changed shape, order or `repr`. + +**Two tests were vacuous.** An executor test's default-mode arm compared one +run's objects against another run's; the dtype census asserted the same +predicate twice and called no seal function. Both now fail against the defect +they pin. + +**Four documents described mechanisms that are not in the code** — a dtype +token, a `_comparables` digest, a `repr()` fallback and the residual risk built +on it — and the design note's own §4(1) had argued the token would be wrong. + +## Next + +1. The **quiet-machine 1/1000 re-run at the finished head** (pid 7656, launched + 2026-09-18T01:58:01Z at 79.8 GB available, head `6e3b3091c`). It re-proves + the replay at the head that ships and answers question 5's request for a + repeat without contention. `RUN-RECORD-after2.txt` states what it must + reproduce before the outcome. +2. The 21-file dependent battery, **re-run clean**. The first run had three red + files: the spine-blindness failure was real and is fixed; the completion-host + and person-status results were invalidated by my own concurrent edits to + `survey_population_replay.py` during the run, which an implementation hash + is over — the report says so rather than quoting them. +3. The final CI-shaped verification at the finished head, the PR body, and the + report's remaining open questions (now eight). + +## Inherited defect, repaired here (for the report and for Max) + +`survey_population_preparation._spill_roster` gained a `Path.read_bytes` on +`native-scale-transport` at **`b6081efcb`** ("Hash a spill segment that was +already there"). That added a `resource_accesses` entry, leaving +`graph_implementation_inventory.json`'s declared `resource_accesses_sha256` +stale, so `implementation_manifest("authenticated_survey_population_v1")` +**refused** — and `SurveyPopulationCreateKernel.implementation_hash` calls it. +**No 19-node graph run was possible at `a64f7b733`.** Bisected across +`5ff889814` → `5307249b3` (both clean) → `baaf4270c` onward (all refusing). +The transport report's §4 "No committed pin moves" was recomputed before +`b6081efcb` and is stale at its own tip. Repaired in `43fb39270`, guarded by +`test_us_implementation_inventory_contracts.py` (135 assertions; reverting the +re-pin turns both arms red, verified). + +## What moves, measured exactly + +`experiments/native-retention-seal/implementation-identity-receipt.json`: + +| comparison | roster module digests that move | +|---|---| +| `a64f7b733` → US-only commits | **none** | +| `a64f7b733` → head with the executor commit | `microcosm.graph/executor.py`, in **all ten** stages | + +`survey_population_replay.py`, `graph_atomic_survey_financial.py` and +`survey_atomic_geography.py` are in no stage roster; `executor.py` is in every +one. The re-pin above moves `inventory_sha256`, which is also in every stage +manifest. + +## Findings, proved by running code at this head + +**Neither existing seal is the seal.** The transport report's §10 question 2(b) +says "`_population_stamp` already folds everything `same_replayed_population` +compares except the *type* assertions". That sentence is wrong in **both** +directions, and both halves are proved by scripts committed under +`experiments/native-retention-seal/`: + +1. **`_population_stamp` is too strict.** + `probe_stamp_vs_comparison.py` builds a US_SCHEMA population, round-trips its + frame through `ContentStore.put_frame`/`load_frame`, and gets: + `same_replayed_population` **ACCEPTS** (the store zeroed `_data` under the + null mask, which `NONCANONICAL_NULL_BACKING`, + `survey_population_replay.py:79-82`, deliberately permits) while + `_population_stamp(expected) != _population_stamp(actual)`. So a + stamp-equality seal would **refuse a required replay**. Its own docstring + says so (`survey_atomic_geography.py:231-237`). +2. **`_frame_identity` is too weak.** `probe_frame_identity_gaps.py` finds three + discriminations `same_replayed_frame` makes that `_frame_identity` does not: + float64 NaN **payload bits** (two quiet NaNs; `NATIVE_BITS`), quiet versus + signalling NaN (`NATIVE_BITS`), and + `DataFrame.flags.allows_duplicate_labels` (`TABLE_TYPE_OR_FLAGS`). `_cell` + maps every NaN to `None`, so `_frame_identity` spells all of them `null`. + It also cannot be applied to a non-US_SCHEMA frame at all (`FRAME_TYPE`). + +**Therefore the seal is purpose-built and lives in `survey_population_replay.py`, +beside the comparison it replaces**, folding exactly the bytes each comparison +compares — no more, no less. + +**A fourth gap, from pandas rather than from this repo.** +`pd.DatetimeIndex._comparables == ['name', 'freq']`, and two DatetimeIndexes with +equal values, dtype and name but different `freq` are **not** `identical()` while +their bytes are equal. Any seal that folds only (class, dtype, name, bytes) +misses it. The seal folds `type(index)._comparables` generically. + +**Admitted dtypes are narrow**, which bounds the problem: `_series` refuses +`CategoricalDtype`, `Float64Dtype` and `DatetimeTZDtype` with +`UNSUPPORTED_EXTENSION_DTYPE`; only masked integer/boolean, `StringDtype`, +`object` and plain numpy dtypes get through. # US Chronicle feed re-pin (dimension labels) Lane: `us-chronicle-feed-repin`, off `origin/main` at @@ -420,6 +649,148 @@ one tolerance so none can report a bundle differently from the others. - Whole-workspace run and PR CI to finish; hand to human review. Do not merge; do not publish. +# Grouped solver x calibration target snapshots integration - 2026-09-12 + +Historical note, 12 September 2026: this grouped-lane journal was subsequently +integrated with PR #914's corrected iteration identities and the actual fiscal +host observer. Its 443-test evidence and statements that host wiring is absent +describe that earlier lane. Current source and acceptance are recorded in +[the fiscal-host evidence](experiments/fiscal-target-snapshot-host-20260912.md). + +Lane: `microcosm-grouped-target-snapshot-integration-20260912`, branch +`grouped-target-snapshot-integration-20260912`. Base: fresh `origin/main` +`116d46ee9dc2aafdc68259b7c06e4c3462522e8b` (unmoved; it is exactly the shared +base both reviewed heads were cut from). Integrates the exact reviewed heads +`536f1ceefcdafda3cc619c14b4da18e012a7be57` (G, US grouped/fixed-zero Adam) and +`b43369dc49e175803f62020cc1e72fa53926aed8` (S, calibration target snapshots, +PR #914) against the read-only checklist +`grouped-snapshot-integration-review.md`. + +Everything below this section is prior-lane history and was accurate when +written; see "Root journals are history, not state" in `CLAUDE.md`. + +## State + +Complete and local. Four commits: the merge, the grouped instrumentation, the +identity recomputation, and the cross-product controls. Nothing pushed, no PR, +no release action. Root reviews and integrates. + +Scope is the shared solver seam only. `graph_fiscal_dense_calibration.py` still +calls `calibrate` without an observer, so this branch emits no snapshot for the +real US fiscal path; that host wiring is a separate step, and no observer +registry, `Node.params` callback or replay-reruns-the-optimizer claim was +invented here. See +[the lane experiment](experiments/grouped-target-snapshot-integration-20260912.md) +for scope, evidence and residual risks. + +## Done + +- Fetched `origin/main`; confirmed it is still `116d46ee9`, so no main drift + had to be preserved. Recorded as the integration base. +- Created this worktree on a new branch from that base; fast-forwarded to G, + then merged S. Verified all four pinned source hashes + (`solve.py` and `calibrate/__init__.py` on both heads) match + `grouped-snapshot-integration-review-pins.json` byte for byte before merging. +- `calibrate/__init__.py` auto-merged as a true union: `GroupedUpperBounds` + export retained alongside every snapshot export. +- `solve.py`: the single textual conflict was the `calibrate()` signature; + resolved as a union of S's `target_snapshots` and G's + `grouped_upper_bounds` / `grouped_preserve_zeros` / + `_post_projection_observer`. +- `test_us_multispine_pool_tool.py`: resolved semantically. G's unrelated US + integration delta (schema_version 2 PUMA-ladder fixture with joint + PUMA/tract/CD overlap arrays and per-layer `source` labels) merged cleanly + and is retained; the only textual conflict was the `spec_sha256` pin. +- The five source-derived identity conflicts + (`inventory_coverage.py` EXPECTED_HASHES, `us-f0-coverage.json`, + `test_spec_engine_loader.py` golden, the multispine `spec_sha256`, and the + calibrate parity `pins.json`) carry a placeholder in this merge commit. None + of them is an ours/theirs decision: both sides' values describe their own + tree, and neither describes the merged one. They are recomputed against the + final merged sources in a later commit on this branch. + +## Done (continued) + +- Instrumented `_optimize_grouped`: the grouped early return in `_optimize` + bypassed every hook S added, so grouped runs emitted only a closing snapshot. + The in-loop emission sits after the progress callback and before `backward()` + and reads the exact float32 estimate tensor the epoch's loss was computed + from. Returned weights, trajectory and RNG state are bit-identical with the + observer on and off, and no evaluation is added. +- Separated retain-best detection from the receipt for grouped runs, so a later + change that populated a grouped receipt cannot make the reused final emitter + claim a retained best. The receipt stays empty. +- Recomputed six source-derived identity artifacts against this checkout (the + five conflicted ones plus the country-bundle digests, which were not + conflicted because only G had touched them but which move for the same + reason). No value equals either branch's. The simulate and fit.qrf pins were + deliberately left alone. +- Added the checklist's cross product to the three existing grouped test files + rather than a new one, so every case reuses fixtures already there. + +## Next + +Root's independent review and integration. Open follow-ups, none owned here: +the host observer seam for the US fiscal dense calibration path, a country-scale +cadence choice backed by a real measurement, and the dashboard consumer. + +--- + +# #893 reconciliation to main's amended graph interface (amendments 19 and 20) + +Lane: `microcosm-us-launch-verified-lanes-20260910` (the live integration +worktree for PR #893, branch `microcosm-us-launch-integration-20260909`). +Started 2026-09-12 at `069d5ed9a`. Everything below the `---` rule at the end +of this section is prior-lane history; see "Root journals are history, not +state" in `CLAUDE.md`. + +## State + +Merges done, four graph pieces re-applied, graph package green (518 passed), +fit green (191), spec check / groups verify / ruff clean. Build-side consumer +suites running at the time of this entry; final results in `out.md` §5 and +`experiments/893-reconciliation-amendments-19-20-20260912.md`. Nothing +pushed, no PR, no new branch, `uv.lock` untouched. + +## Done + +- Read `CLAUDE.md`, the charter's "Interface freeze" at `23ba24770`, PR + #893's body, the Amendment 19 lane's "Scope" list; diffed the graph package + against `23ba24770` per file; AST-scanned every non-graph consumer. +- `uv sync --all-packages --locked --extra us --extra uk` exit 0. Baseline + graph suite: 5 failed / 362 passed (the five the brief names). +- `3010b7788` merge `origin/main`: graph package, fit sources, lock and + charter resolved to main's bytes; branch-only `attachments.py`, + `availability.py`, `schema.py` and the two branch-only graph tests removed + in the merge, to return only where consumed. +- `051357909` merge `23ba24770` (amendment 20, merged from the branch head + because #912 was still on CI; the dispatcher re-runs `git merge origin/main` + after it lands — expected no-op for graph/fit/lock/charter). +- `dc621c14c` seed digests re-pinned (the branch's `acs_transfer` and + `housing_inputs` plus amendment 20's `fit.qrf` move them); coverage report + regenerated; `--check` exit 0. +- `cff8fbf32` calibrate/simulate H1 pins re-recorded (the branch's solver and + `Frame.__reduce__` changes move them); parity files 27 passed. +- Pieces re-applied on main's files, each with graph-level tests the branch + never had: `752ab840f` raw-byte codec (13 US consumers; 8 codec tests), + `db1b7821a` Frame-metadata store (3 named consumers; 274 passed across the + store/population/executor/manifest files), `d1019762b` execution states + (the post-clone geography gate's typed artifact; replaces the amendment-19 + gate refusal and its test; whole graph package 518 passed), `a16eeacf8` + population observer (4 consumers; 94 passed on the executor + B/F files). +- Dropped for lack of a consumer: lazy retention (`attachments.py`, layer + 08), `schema.py`, `keys._stream_file`, the `_write_node` refactor. +- Layer map computed from `git log --name-only` attribution plus an AST + import scan with hard / name-hard / soft link classes; written to `out.md`. + +## Next + +- Record the build-side consumer results in `out.md` §5, commit the + `experiments/` copy of the report, leave `out.md` uncommitted (it is + another lane's tracked report; see the memory note). +- For Max: piece C supersedes amendment 19's gate refusal (`out.md` §8.1); + the dropped lazy retention / `_stream_file` / `_write_node` pieces (§8.2–3). + --- # Amendment 19 — typed opaque artifacts on the graph interface @@ -1384,3 +1755,354 @@ The still-earlier PolicyEngine-US 1.819.0 lock-bump lane merged into `origin/main` at `7b90bb18` on 2026-08-24; its final state remains at commit `05d254aa` and its detailed receipts remain in the historical section of `_LANE-NOTES.md`. + +## US launch integration staging — 2026-09-09 + +### State +Source-only staging in progress. Execution and source/data admission remain root-owned. + +### Done +Verified requested clean branch, base HEAD, main ancestry and preservation pins. + +### Next +Apply thirteen explicit source layers, commit each, then separately review isolated ordinary execution. Existing journal history above is retained. + +Layer 1: SAFE-ADDITIVE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 2: GRAPH-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 3: ACCEPTED-SHARED-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 4: PUF-SUPPORT-MERGE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 5: SOLVE-MERGE-PROPOSAL.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 6: J-GRAPH-COMPATIBILITY.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 7: F-CATALOGUE-OPTIMIZATION.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 8: GRAPH-ATTACHMENT-METADATA.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 9: F-JOINT-GEOGRAPHY-GATE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 10: SOURCE-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 11: PLACEMENT-ADDITIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 12: ORDINARY-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 13: INTEGRATION-REGRESSIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +### State +All thirteen source layers staged and committed; behavioral qualification pending. + +### Done +Per-layer pins and actual commit messages checked; store/current-main preservation retained. + +### Next +Root reviews exact ordinary guard/source/resource admissions before execution. Full65 findings and survey/SCF lanes remain separately owned. No remote action is authorized. + +### Integration source-resource closure — 2026-09-09 + +Root preflight found seven JSON source definitions declared by the ordinary guard but omitted by the staged patch delivery. Added the exact previously reviewed resource bytes from the accepted full65 source projection; no new resource admission or genuine payload. Preserved source staging and earlier evidence. + +## Layer14 full65 replay correction — 2026-09-09 + +State: exact accepted two-file correction staged; successor57 integration execution pending. +Done: verified clean84243 preimages, exact r2 postimages, fixed store/current-main sources and frozen36 evidence. +Next: root admits the separate exact-allowlist57-case guard and final source identities before execution; no new resources. + +## Source review publication — 2026-09-09 + +The user explicitly authorized pushing the current source work and creating PRs for Anthony to review. This supersedes earlier source-only local restrictions for source publication; it does not authorize a data release, merge or deployment. + +Integration controls passed 36 cases; the exact replay correction subsequently passed all 57 cases at dfa7f872cd3eba3c42adf5758cde8b3ca38f3d17. Source/control, model-declaration and resource hashes matched externally after both runs. A final formatting/import cleanup and explicit test-observer loop binding are included for CI; no release result is claimed. See docs/us-launch-review.md for current scope, evidence and related PRs. + + +# US completion host integration — 2026-09-13 + +State: source implementation and bounded tests authored for root review; all runtime checks and tests UNRUN. +The task-specific no-commit instruction overrides the standing commit order. +Base: 6aa745c8762c3e4a140b8168470de5827e12c5cb. + +Done: inspected status, branch, HEAD, AGENTS.md, CLAUDE.md, frozen R2 plan and +independent source-plan review. Concurrent completion/tax seam files are excluded +from this work's ownership. + +Done: added private completion custody, retained issued property parent, registry +copies, actual union artifact/state checks, role/child reconstruction and separate +tax gate wiring. Authored 10 test functions / 20 source-expanded cases and docs. +AST, direct Ruff lint/format, and tracked whitespace checks pass. + +Next: root review and reconciliation, inventories/test-spine updates, bounded +resource/guard proposal, and actual fixture/cold/required/adversarial validation. +45/49 counts remain source-derived; combined 51 and fixture assumptions are UNRUN. +Final report: FINAL_REPORT.md. No acceptance or publication claim. + +## Bounded completion-host source corrections — 2026-09-13 + +State: bounded source corrections complete; all runtime checks remain UNRUN. +This correction pass used the 20-minute source-only window beginning 22:09 UTC. +The later explicit no-commit/no-staging instruction governs this correction pass. +Ownership is limited to graph_survey_completion_host.py, +test_us_graph_atomic_completion_host.py, docs/us-survey-completion-host.md, +and this exact PROGRESS.md / FINAL_REPORT.md update. + +Done: read the full independent source review and full approved R2 plan from +the supplied recovered absolute paths. The R2 SHA-256 matches +b402d6be9d374774e27da7bf42fb83a216305d4cddd2ad8347c8149536269a29. +Confirmed intentional dirty worktree and HEAD +6aa745c8762c3e4a140b8168470de5827e12c5cb. Earlier journal content is preserved. + +Done: exact receiving1/roles0-or-2/child6/tax3 ordered rosters are bound before +compilation; child document count and knownness derive from retained nodes and +reconstructed/materialized-verified evidence. Authored separate 49/45 cold+required +acceptance, equal-values/replaced-Population refusal, real observed base-hit/union +and final-I/O identity controls, actual tax-rebased parent refusal, exact child +adversary reasons, full-spine writer ordering, both FILTER conservation ledgers, +dimensions and owner/version transitions. Source proves table copying; immediate +base preservation assertions remain alongside permanent revocation. Fixture-only +composition wrappers now restore before issuance to avoid cross-fixture chaining. + +Done: AST parsing, direct Ruff lint/format and git diff --check pass. AST reports +16 test functions / 32 source-expanded cases. All tests, collection and project +imports remain UNRUN. Only the five explicitly owned files were edited; no +commits, staging, agents, dependency/network/browser access or payload execution. + +Next: root reviews exact source/hashes and budgets separately selected execution. +T4 financial-only-parent coverage, remaining T5 initial/final/reconciliation +adversaries, T6 actual fixture validation and T7 GQ/zero-weight scientific fixtures +remain open with concrete reasons in FINAL_REPORT.md. The independently confirmed +old status/tax42 owner repair, inventories, CI/source closure, scientific/native +capacity and release/default/PR893 decisions remain separately owned. + +--- + +# Lane: verify native sources once per run (`native-verify-once`) + +Branch `native-verify-once`, worktree `~/PolicyEngine/_worktrees/microcosm-verify-once`, +base `f7bb88525a78786f91bc3ebe2083ef4b1c85de18` (PR #893 head). Started +2026-09-15. + +**State (2026-09-16, third session):** all five mechanisms landed with proofs +and the epoch wired into both atomic capsules; draft PR #935 open; the +CI-shaped test battery is green (no test group failed; `ruff format --check .` +is red on 77 pre-existing files that this branch does not touch, and CI does +not run it); the before/after probe and the nineteen-node harness have run; the +lane report `experiments/native-verify-once/out.md` is complete, with the +measurement, the pins, the verbatim summaries and the open questions. The open +work is the main-only split (its commits sit on +`graph-verify-once-main-stale-20260916-0057`, unpushed) and Max's answers to the +report's open questions. +*(Historical as of 2026-09-17: the split is now draft PR #938, and an +adversarial verification of this branch has since been resolved by ten further +commits on a rebased branch — see "Done (2026-09-17, fourth session)" below.)* + +**Goal.** The 9/15 pilot v5 measurement +(`~/PolicyEngine/_recovered/pilot-runs/native45-v5/out.md` §2) attributes ~79 % +of a native 1/1000 run to repeated admission and verification of 3.48 GiB of +staged source: node execution is 1,143 s of a 5,362 s run. Make source +authentication and identity verification happen once per run instead of once +per accessor use and once per executed node, without weakening any refusal and +without moving any digest value. + +**Done:** read the evidence base; located all five mechanisms at base HEAD. + +**Done (2026-09-16):** + +- `docs/us-native-verification-once.md` — the design note, written before any + capsule changed. +- **Mechanism 5**: `_object_stream` builds each plain float/int/bool column's + context-digest bytes with numpy. Byte-identical against a verbatim copy of the + pre-change body over every column kind, the float specials, non-canonical NaN + payloads, both int64 endpoints and a 200-frame random sweep. 0.47 s -> 0.09 s + on a 6,928 x 240 frame. +- **Mechanism 4**: the ACS record fence now uses `bytes.find`/`bytes.count` with + cached terminator cursors. Proven against the byte loop on 46,655 exhaustive + short strings across five ceiling settings, 4,500 random strings, and the real + staged `csv_pus.zip`: 3,422,890 records, identical digests, 226.9 s -> 4.6 s + (49x). Re-pins `_ACCEPTED["acs_person_coverage_authentication.py"]`. +- **Mechanism 3**: `run_graph` carries a `_SourceIdentities` cache keyed on stat + signatures, and re-derives every source in full before building the manifest. + `RunManifest.source_identities` records it without moving any existing value. + The refusal it protects had no test at all; `test_graph_executor_source_identity` + now pins it. Graph suite 108 s -> 72 s. +- **Mechanisms 1 and 2**: `survey_population_preparation.verification_epoch()`, + an opt-in scoped memo. Cheap tier every borrow (live authority, attached + payloads, producer encoding); expensive tier skipped only while a signature + over every path it reads — the roster stat identities among them — and every + live buffer it digests is unchanged; unconditional full re-validation on + leaving the epoch. The two moved inventory contracts are re-derived (below). + (The roster stat identities were compared in the cheap tier when this line + was first written; the fourth session moved them into the signature so a + memo miss raises the code an unmemoised borrow raises. See below.) + +**Done (2026-09-16, second session):** + +- Draft PR [#935](https://github.com/PolicyEngine/microcosm/pull/935), base + `microcosm-us-launch-integration-20260909`. +- Main-only split branch `graph-verify-once-main`, worktree + `~/PolicyEngine/_worktrees/microcosm-graph-verify-once-main`, branched from + `origin/main` (51c314382). It carries the graph-shard change only: the three + commits re-applied, plus a changelog fragment. Its one deliberate difference + from this branch is that `_update_scalar` has no exact-float `struct.pack` + shortcut on main -- that belongs to PR #893 -- so the parity test's reference + copy was aligned to main's actual pre-change body, which makes it a literal + verbatim copy there too. +- Before/after measurement staging in `.measure/` (gitignored): a parameterised + copy of the v5 pilot probe that takes the source tree, the staged run inputs + and the output directory from the environment, asserts every imported + `microcosm` module resolves inside the measured tree, and enforces a 16 GiB + resident ceiling alongside the CPU one. Baseline worktree + `~/PolicyEngine/_worktrees/microcosm-verify-once-baseline` is detached at + `f7bb88525`. + +**Done (2026-09-17, fourth session) — the verification findings:** an +adversarial static verification of the branch returned four medium and seven low +findings and no high one +(`~/PolicyEngine/_recovered/scratch-backup/893/lanes/verify-once-findings-20260916.json`). +Ten commits answer them, on a branch rebased onto the base tip `363a9033b`: + +- rebased onto `363a9033b` and force-pushed, so two-dot and three-dot diffs + agree again (26 files, +5,422 / −819); +- a mutation test per memoised capsule that changes file **content** while the + cheap tier's view of it stays byte-identical, so the memo-miss branch itself + refuses rather than a stat comparison in front of it; +- the roster stat identities moved out of the refusing cheap tier into the + signature, so an in-epoch refusal carries the code an unmemoised borrow + carries (behaviour, not documentation: the design note promised it and the + fix is smaller than the divergence, and it deletes a redundant `_file_stats` + walk per borrow); +- `verification_epoch()`'s record is bound by both runners and attached to the + manifest as `RunManifest.verification_epoch` — outside the key, the JSON, + every node receipt and every cache record. A real nine-node run over invented + sources reports `{"capsules": 1, "hits": 20, "misses": 3, + "final_validations": 1}`; +- `_source_stat_signature` follows a member symlink, as `_directory_identity` + does, so a target rewritten mid-run refuses at the next node rather than at + run end; `_object_stream` packs floats in native order, as the loop it + replaces does; +- the dead `_stat_or_absent` is deleted; the four `_path_stat` copies and the + two `_stat_identity` definitions stay, with the pin cost of consolidating + them re-derived through the modules' own generator and stated in the report; +- the design note's five inaccuracies are corrected, the nested finalizer + chains its refusal instead of dropping it, and the report's stale items are + historicized. + +Re-run afterwards, all rc 0: `769 passed, 1 skipped in 44.56s` +(`packages/microcosm-graph/tests`), `128 passed in 188.39s` (the four new test +files), `1811 passed, 1 skipped, 38 warnings in 11200.54s (3:06:40)` (the 46 +build test files that reach a changed module), plus `ruff check` clean, +`17 files already formatted`, `uv lock --check`, both `ci_test_groups` checks. + +**Done (2026-09-17, second fix pass) — the two residuals:** a re-verification of +the ten commits above left two low findings, and two commits answer them. (1) +Only the nine-node runner had a test that read `RunManifest.verification_epoch` +back off a real run, so +`test_nineteen_node_financial_cold_and_required_replay` now reads it off both +manifests the financial runner returns — its own epoch's and the nested +population epoch's — for the cold run and the required replay, asserting the +protocol label, `final_validations == capsules >= 1`, and that `to_json` still +cannot see it; and the committed probe +(`experiments/native-verify-once/probe_verify_once.py`) now copies the record +into its output as `verification_epoch`, so the next measured run is the first +that will carry counts. Nothing was re-measured, and the three measurement files +the report quotes predate that line, which the report, the README and the design +note all state. (2) This journal's "Next" and the report's open question 4 +restated #938 at its current head and size, with the mirror recorded as landed. +No `packages/*/src` file changed in either commit. + +Re-run, both rc 0: `7 passed in 119.98s (0:01:59)` +(`test_us_graph_atomic_survey_financial.py`) and +`24 passed in 157.61s (0:02:37)` (`test_us_native_verify_once_epoch.py`), plus +`ruff check` and `ruff format --check` clean on the two changed Python files. + +**Done (2026-09-17, third fix pass) — one medium and two low:** a second +re-verification found that the 2026-09-16 refusal-code fix had weakened one +at-borrow refusal, and three commits answer it and the two low findings. +(1) `_finalize_epoch` recorded its memo signature *after* validating, so at an +inner nested close — whose memo survives into the outer epoch — a roster file +whose stat moved during the trailing part of `_validate` was absorbed as the +new normal and the outer borrows were hits until the outermost close. Each +close now takes the signature before validating and again after and records +none when they differ, so the next borrow is a miss that refuses; +`asec_2024_native_population._epoch_exit` had the same ordering and is fixed +the same way. Two new tests move a source from a profile hook as that close's +own validation returns and fail with `DID NOT RAISE` against the previous +ordering. `_memoized_validate`'s signature handler also narrowed from +`BaseException` to `Exception`, which is what its own comment describes. +(2) `RunManifest.verification_epoch` and `run_graph(_verification_epoch=)` had +no graph-shard test on either branch: `test_graph_verification_epoch.py` adds +nine, and two build-shard tests cover the signature-cannot-be-taken handler and +the `raise own from nested` branch. (3) The stale cheap-tier descriptions the +2026-09-16 fix left behind — test-file docstrings, the module block comment, the +design note and this report — now say what the code does, and `executor.py:2100` +is corrected to `:2105`. Two `packages/*/src` files changed and no pin moved: +`graph_implementation._dependency_contract` was re-run over both edited modules +and `imports`, `resource_accesses_sha256` and `unbound_uses_sha256` are all +unchanged. + +Re-run, all rc 0: `778 passed, 1 skipped in 37.72s` +(`packages/microcosm-graph/tests`, up from 769 — the new +`test_graph_verification_epoch.py` adds nine), `28 passed in 190.65s (0:03:10)` +(`test_us_native_verify_once_epoch.py`, up from 24), +`1607 passed, 38 warnings in 8311.33s (2:18:31)` (the 34 build test files naming +either changed module), `8 passed in 0.68s` +(`test_us_asec_prepared_resources.py`, which builds the stage implementation +manifests), and `597 passed, 2 skipped in 17.75s` in the +`graph-verify-once-main` worktree. Plus `ruff check .`, `ruff format --check` on +all five changed Python files, `uv lock --check`, both `ci_test_groups` checks. + +**Next:** Max's rulings on the report's remaining open questions. Open question +4 is answered and needs nothing further: the split is draft PR +[#938](https://github.com/PolicyEngine/microcosm/pull/938), opened 2026-09-16 +and now at head `33f3150bb` (base `main`, 7 files +1,430/−8, MERGEABLE, still +draft; `gh pr view 938`, 2026-09-17 07:43 UTC). The graph-shard commits made +after the verification findings were mirrored into it on 2026-09-17 at +06:45–06:46 UTC — the verification epoch on the manifest, the member-symlink +follow, the native-order float cast and the docstring fix — and the third fix +pass's new `test_graph_verification_epoch.py` followed as `33f3150bb`. Diffed +the same morning, its five earlier graph files differ from this branch's only by +the documented `import struct` and exact-`float` hunks that belong to #893's +base. +#935's body already carries the measurement table. Measured: nine-node prefix +before 1,803.87 CPU s without completing (ceiling) against after 1,444.78 CPU s +completing; nineteen-node 5,278.61 -> 2,010.07 CPU s (2.63x) against the v4 cold +receipt, peak RSS 9.31 -> 13.31 GB. + +**Pins re-derived so far:** + +| pin | old | new | +|---|---|---| +| `acs_native_coverage_binding._ACCEPTED["acs_person_coverage_authentication.py"]` | `475aa795…fe85bcff` | `9ec68721…d88e8e49f` | +| `graph_implementation_inventory.json` `survey_population_preparation.py` `unbound_uses_sha256` | `29c09f6f…d296ef91` | `d114117c…dd4005910` | +| `graph_implementation_inventory.json` `asec_2024_native_population.py` `unbound_uses_sha256` | `71463df4…d7f287608` | `7bb20439…40d58cdf4` | + +**Lane notes.** Root `out.md` is a tracked file holding the Amendment 19 lane's +committed report; this lane's report goes to +`experiments/native-verify-once/out.md`. The v5 cold run (`run_v5.py`, pid +81194) is live on this machine, so probe runs wait for the window the lane +brief allows (`pgrep -f run_v5.py` empty, or > 40 GB free). +## Stack reconcile after #938 — 2026-09-18 + +Lane: bring `microcosm-us-launch-integration-20260909` (#893) level with +`origin/main` 8c44daa52 (#938 verify-once merged 2026-09-17), then carry the +merge down the stacked branches `native-verify-once` (#935), +`native-scale-transport` (#945), `native-row-ceilings` (#949) and +`native-retention-seal` (#950). Merges only, never rebases; pins regenerate +through their generators. Report: `experiments/stack-reconcile-20260918/out.md`. + +State: worktree `_worktrees/microcosm-stack-reconcile`, local branches +`reconcile/`, pushed with `git push origin HEAD:`. +Starting heads: main 8c44daa52; integration 8a7e1b12d; native-verify-once +0d04abfcf; native-scale-transport 6fca96f31 (one commit past the brief's +886f777eb); native-row-ceilings c5ac78d2d; native-retention-seal db7c93871. + +Done: survey. Main's delta since the d1196af10 merge base is #938 alone: +nine files, all under `packages/microcosm-graph/` plus one changelog fragment; +uv.lock and every pyproject are unchanged, so the approved-lock digest does +not move. `git merge-tree` predicts one textual conflict, an import-line clash +in `graph/executor.py` (`import struct` on the branch, `import stat` on main). +The four downstream merges preview conflict-free at today's heads. + +Next: step 1 merge, checks, push; then steps 2–4 in order; then the report. diff --git a/changelog.d/20260906-frame-metadata-storage.fixed.md b/changelog.d/20260906-frame-metadata-storage.fixed.md new file mode 100644 index 000000000..dfa6dd022 --- /dev/null +++ b/changelog.d/20260906-frame-metadata-storage.fixed.md @@ -0,0 +1 @@ +Preserve complete Frame metadata in graph content-store format v2, including nested source evidence, and refuse older frame caches that cannot restore it or reuse of one frame key with different metadata. diff --git a/changelog.d/413-snap-final-export-checks.added.md b/changelog.d/413-snap-final-export-checks.added.md new file mode 100644 index 000000000..7b582048c --- /dev/null +++ b/changelog.d/413-snap-final-export-checks.added.md @@ -0,0 +1 @@ +Add scoped SNAP checks that reopen an identified final export, recompute 51-state caseload and benefit estimates, verify county/state consistency, and preserve pending status when runtime evidence or reviewed tolerances are missing. diff --git a/changelog.d/893-age-count-fixture.internal.md b/changelog.d/893-age-count-fixture.internal.md new file mode 100644 index 000000000..1e691423f --- /dev/null +++ b/changelog.d/893-age-count-fixture.internal.md @@ -0,0 +1 @@ +Use the maintained Frame store API in the national age-count cold and fresh-process replay tests after removal of the transfer example module. diff --git a/changelog.d/893-asec-prepared-resources.fixed.md b/changelog.d/893-asec-prepared-resources.fixed.md new file mode 100644 index 000000000..95dc74d99 --- /dev/null +++ b/changelog.d/893-asec-prepared-resources.fixed.md @@ -0,0 +1 @@ +Restore three pinned ASEC resource files omitted during integration so prepared and composed source stages can compute their implementation manifests. Preserve the existing empty engine-default allowlist, source declarations, and reported-income contract. diff --git a/changelog.d/893-ci-reshard-durations.fixed.md b/changelog.d/893-ci-reshard-durations.fixed.md new file mode 100644 index 000000000..5f5581ea7 --- /dev/null +++ b/changelog.d/893-ci-reshard-durations.fixed.md @@ -0,0 +1 @@ +CI: rest lane split six ways and wheels eight ways, each sharded pytest run reporting its 25 slowest tests. diff --git a/changelog.d/893-diagnostic-current-lock.fixed.md b/changelog.d/893-diagnostic-current-lock.fixed.md new file mode 100644 index 000000000..cff257d2e --- /dev/null +++ b/changelog.d/893-diagnostic-current-lock.fixed.md @@ -0,0 +1 @@ +Align the source/seed diagnostic with the lockfile already approved for the primary QRF worker after the UK openpyxl dependency declaration. diff --git a/changelog.d/893-diagnostic-refusal-context.internal.md b/changelog.d/893-diagnostic-refusal-context.internal.md new file mode 100644 index 000000000..c6e330a2f --- /dev/null +++ b/changelog.d/893-diagnostic-refusal-context.internal.md @@ -0,0 +1 @@ +Record bounded code-path context for refused CI spec diagnostics while preserving the existing read restrictions and incomplete result status. diff --git a/changelog.d/893-diagnostic-source-io-lock.fixed.md b/changelog.d/893-diagnostic-source-io-lock.fixed.md new file mode 100644 index 000000000..f192b5b45 --- /dev/null +++ b/changelog.d/893-diagnostic-source-io-lock.fixed.md @@ -0,0 +1 @@ +Update the source identity diagnostic's exact lockfile pin after adding the engine-free source reader extra, and verify that the checked-in lock passes while an incorrect pin still refuses. diff --git a/changelog.d/893-fiscal-measurement-spm-selection.fixed.md b/changelog.d/893-fiscal-measurement-spm-selection.fixed.md new file mode 100644 index 000000000..9e05e6336 --- /dev/null +++ b/changelog.d/893-fiscal-measurement-spm-selection.fixed.md @@ -0,0 +1 @@ +The declared fiscal measurement kernel now binds the release SPM selection into its model contract and passes it to the engine, and a model output that reaches an SPM measurement variable must declare the household county, the unit's tenure and each member's age as leaves, which the static dependency closure does not see. diff --git a/changelog.d/893-historical-zero-provenance.internal.md b/changelog.d/893-historical-zero-provenance.internal.md new file mode 100644 index 000000000..5c7d83751 --- /dev/null +++ b/changelog.d/893-historical-zero-provenance.internal.md @@ -0,0 +1 @@ +Keep the exact historical ASEC zero-origin citations in the dependency scan while continuing to reject other incumbent data-package references in that module. diff --git a/changelog.d/893-merged-uk-identity.fixed.md b/changelog.d/893-merged-uk-identity.fixed.md new file mode 100644 index 000000000..b6a1ef320 --- /dev/null +++ b/changelog.d/893-merged-uk-identity.fixed.md @@ -0,0 +1 @@ +Refresh the UK country-spec identity from the merged source and resource set after incorporating the UK monthly-target correction. diff --git a/changelog.d/893-native-shared-fiscal-solve.changed.md b/changelog.d/893-native-shared-fiscal-solve.changed.md new file mode 100644 index 000000000..8ae6af2a5 --- /dev/null +++ b/changelog.d/893-native-shared-fiscal-solve.changed.md @@ -0,0 +1 @@ +Extract the US release builder's existing dense and L0 calibration calls into a shared helper for prepared support. The legacy entry uses the same helper; exact-k selection, solve settings, loss reporting, and release gates retain their existing behavior. diff --git a/changelog.d/893-native-shared-fiscal-targets.changed.md b/changelog.d/893-native-shared-fiscal-targets.changed.md new file mode 100644 index 000000000..70e074528 --- /dev/null +++ b/changelog.d/893-native-shared-fiscal-targets.changed.md @@ -0,0 +1 @@ +Reuse the maintained US ledger target compilation, reviewed Medicaid substitutions and target-parity checks through a shared helper. Source-specific SSI and target-profile checks retain their legacy order. diff --git a/changelog.d/893-population-observer-isolation.fixed.md b/changelog.d/893-population-observer-isolation.fixed.md new file mode 100644 index 000000000..710af6485 --- /dev/null +++ b/changelog.d/893-population-observer-isolation.fixed.md @@ -0,0 +1 @@ +Give graph population observers detached snapshots so callback mutations and retained references cannot change subsequent calculations or cached populations. Preserve the complete frame, provenance, weights and mass accounting on both cold runs and cache hits. diff --git a/changelog.d/893-population-only-block-source.added.md b/changelog.d/893-population-only-block-source.added.md new file mode 100644 index 000000000..09da0ddba --- /dev/null +++ b/changelog.d/893-population-only-block-source.added.md @@ -0,0 +1 @@ +Normalize pinned population-only Census API responses into atomic block support. Reconcile every positive block against independent state totals and CD/PUMA mappings, retain exact source bytes and request descriptions, and keep publisher qualification separate from byte integrity. diff --git a/changelog.d/893-puf-original-application.added.md b/changelog.d/893-puf-original-application.added.md new file mode 100644 index 000000000..1b7f769c6 --- /dev/null +++ b/changelog.d/893-puf-original-application.added.md @@ -0,0 +1 @@ +Add explicit original-arm PUF application declarations and a strict detached conditioning merger that retain raw draws, qualified fixed values, and separate fit/application seed provenance. diff --git a/changelog.d/893-refusal-expectations.fixed.md b/changelog.d/893-refusal-expectations.fixed.md new file mode 100644 index 000000000..fb7fe18b8 --- /dev/null +++ b/changelog.d/893-refusal-expectations.fixed.md @@ -0,0 +1 @@ +Recognize the earlier graph-node population check in mutation regression tests while retaining the required refusal and mutation-restoration assertions. diff --git a/changelog.d/893-source-demographic-conditioning.added.md b/changelog.d/893-source-demographic-conditioning.added.md new file mode 100644 index 000000000..88bc7fb66 --- /dev/null +++ b/changelog.d/893-source-demographic-conditioning.added.md @@ -0,0 +1 @@ +Add optional source-qualified sex and state predictors to survey financial imputation. Preserve observed-year distinctions and default behavior, refuse unresolved source demographics, and verify complete population preservation through fitting, attachment and required replay. diff --git a/changelog.d/893-spec-identity-diagnostics.internal.md b/changelog.d/893-spec-identity-diagnostics.internal.md new file mode 100644 index 000000000..3eb49f5a0 --- /dev/null +++ b/changelog.d/893-spec-identity-diagnostics.internal.md @@ -0,0 +1 @@ +Capture bounded spec and seed identity evidence after shared CI tests so changed source fingerprints can be reviewed without weakening coverage assertions. diff --git a/changelog.d/893-survey-atomic-prefix.added.md b/changelog.d/893-survey-atomic-prefix.added.md new file mode 100644 index 000000000..30658e355 --- /dev/null +++ b/changelog.d/893-survey-atomic-prefix.added.md @@ -0,0 +1 @@ +Connect qualified survey geography to shared Census-block assignment before cloning and age calibration. Preserve raw sampling allocation separately, reconstruct geography in calibration budgets, and verify complete geography inheritance through cold execution and required replay. Add bounded source-byte normalization controls and record the remaining native-source and release checks. diff --git a/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md b/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md new file mode 100644 index 000000000..9614ce4df --- /dev/null +++ b/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md @@ -0,0 +1 @@ +Add supplied-source UK atomic-area assignment keyed to stable post-clone household identity, and load UK runtime exports only when requested. diff --git a/changelog.d/893-worker-resource-trace.fixed.md b/changelog.d/893-worker-resource-trace.fixed.md new file mode 100644 index 000000000..a48093a90 --- /dev/null +++ b/changelog.d/893-worker-resource-trace.fixed.md @@ -0,0 +1 @@ +Update the worker resource identity regression to follow the actual lazy import closure, and verify that opened JSON resources affect identity while unlisted resources do not. diff --git a/changelog.d/908-calibration-target-snapshots.added.md b/changelog.d/908-calibration-target-snapshots.added.md new file mode 100644 index 000000000..127a2b385 --- /dev/null +++ b/changelog.d/908-calibration-target-snapshots.added.md @@ -0,0 +1 @@ +Shared microcosm-calibrate can emit aggregate-only per-target estimate snapshots from real solver iterations at a configurable bounded or every-epoch cadence, with a versioned schema, stable ordered target identity digest, honest current/best-retained/selected iterate labelling, and an atomic latest-snapshot plus immutable bounded history store (opt-in; off by default). diff --git a/changelog.d/908-target-snapshot-review-fixes.fixed.md b/changelog.d/908-target-snapshot-review-fixes.fixed.md new file mode 100644 index 000000000..373ab23d0 --- /dev/null +++ b/changelog.d/908-target-snapshot-review-fixes.fixed.md @@ -0,0 +1 @@ +Close the four #908 target-snapshot review findings: a closed typed aggregate metadata contract, detached snapshot delivery, atomic history publication, and strict public codec validation. diff --git a/changelog.d/914-snapshot-iterate-identity.fixed.md b/changelog.d/914-snapshot-iterate-identity.fixed.md new file mode 100644 index 000000000..2563244d6 --- /dev/null +++ b/changelog.d/914-snapshot-iterate-identity.fixed.md @@ -0,0 +1 @@ +Use completed optimizer updates consistently in target snapshots, retain the selected budget probe, and encode overflowing relative errors as counted nulls. diff --git a/changelog.d/acs-native-declaration-cache.changed.md b/changelog.d/acs-native-declaration-cache.changed.md new file mode 100644 index 000000000..fa9e57827 --- /dev/null +++ b/changelog.d/acs-native-declaration-cache.changed.md @@ -0,0 +1 @@ +Cache immutable ACS source declaration summaries by exact source bytes and parser context while preserving fresh source reads and live implementation checks. diff --git a/changelog.d/acs-serial-lookup.fixed.md b/changelog.d/acs-serial-lookup.fixed.md new file mode 100644 index 000000000..0f7e90355 --- /dev/null +++ b/changelog.d/acs-serial-lookup.fixed.md @@ -0,0 +1 @@ +Reuse ACS household-key lookups across person archive chunks while preserving source validation and selection. diff --git a/changelog.d/acs-spm-source-assembly.added.md b/changelog.d/acs-spm-source-assembly.added.md new file mode 100644 index 000000000..c3bde0eff --- /dev/null +++ b/changelog.d/acs-spm-source-assembly.added.md @@ -0,0 +1 @@ +Add opt-in ACS SPM source construction with a selected-roster identity registry, preserving tax units and household design weights before source amounts are mapped. Bind explicit native construction requests to captured-source rowwise evidence and reviewed assembler identity without enabling a default or supplying engine roles. diff --git a/changelog.d/asec-farm-allocation-meaning.fixed.md b/changelog.d/asec-farm-allocation-meaning.fixed.md new file mode 100644 index 000000000..ab570a7c3 --- /dev/null +++ b/changelog.d/asec-farm-allocation-meaning.fixed.md @@ -0,0 +1 @@ +Keep farm allocation provenance unresolved when only I_FRMYN has a nonzero code: the published dictionary supplies its range but no code meanings. Preserve allocation established by documented flags. diff --git a/changelog.d/asec-source-domain-contract.fixed.md b/changelog.d/asec-source-domain-contract.fixed.md new file mode 100644 index 000000000..e6b385f9c --- /dev/null +++ b/changelog.d/asec-source-domain-contract.fixed.md @@ -0,0 +1,3 @@ +Prevent returned ASEC amount metadata from mutating cached definitions. Require +unique current-vintage entries, supported interest and child-support domains, +and agreement with the retained current-money owner before source capture. diff --git a/changelog.d/asec-usual-hours-observations.added.md b/changelog.d/asec-usual-hours-observations.added.md new file mode 100644 index 000000000..0cd4bb50f --- /dev/null +++ b/changelog.d/asec-usual-hours-observations.added.md @@ -0,0 +1 @@ +Add an all-age ASEC usual-hours observation recoder that separates source nonwork from survey-universe absence, preserves allocation flags, and leaves unresolved histories unfilled. diff --git a/changelog.d/atomic-property-financial-host.added.md b/changelog.d/atomic-property-financial-host.added.md new file mode 100644 index 000000000..460a98daf --- /dev/null +++ b/changelog.d/atomic-property-financial-host.added.md @@ -0,0 +1 @@ +Add an explicit property-income extension to the checked US financial host, retaining complete legacy populations and validating model receipts, reconstructed outputs and required cache replay. diff --git a/changelog.d/atomic-property-tax-host.added.md b/changelog.d/atomic-property-tax-host.added.md new file mode 100644 index 000000000..f3ef5b12a --- /dev/null +++ b/changelog.d/atomic-property-tax-host.added.md @@ -0,0 +1,5 @@ +Add an explicit optional 38-node survey financial host that retains its complete +19- and 35-node populations, rebases property tax leaves with strict unknowns, +and verifies the new full population and numerical artifacts. Add a conditional +typed PUF gate dependency and refuse PUF qualification when required rebased +inputs remain unknown. The existing 19- and 35-node options remain unchanged. diff --git a/changelog.d/calibrate-parity-repin.fixed.md b/changelog.d/calibrate-parity-repin.fixed.md new file mode 100644 index 000000000..c84ba9eec --- /dev/null +++ b/changelog.d/calibrate-parity-repin.fixed.md @@ -0,0 +1 @@ +Refresh calibration parity implementation and node-key pins after the validated Weights copy-protocol change, preserving all recorded direct outputs and platform entries. diff --git a/changelog.d/child-completion-structural-reads.fixed.md b/changelog.d/child-completion-structural-reads.fixed.md new file mode 100644 index 000000000..e26ec99e7 --- /dev/null +++ b/changelog.d/child-completion-structural-reads.fixed.md @@ -0,0 +1 @@ +Fix child property-income graph compilation by taking entity IDs and membership IDs from the executor's structural view while retaining all source-identity column dependencies. diff --git a/changelog.d/child-property-empirical-candidate.added.md b/changelog.d/child-property-empirical-candidate.added.md new file mode 100644 index 000000000..b5a9be753 --- /dev/null +++ b/changelog.d/child-property-empirical-candidate.added.md @@ -0,0 +1,3 @@ +Add an explicit weighted paired empirical model and a full-source ASEC teenager donor qualifier for an opt-in child property-income candidate, preserving observed values, source-knownness and original household design support. The source-only candidate supplies no released default or native-data acceptance. + +Preserve small supported zero-pattern mass during reference transport, require support for every present pattern, refuse wider floating-point inputs before narrowing, and bind consumed allocation interpretation constants into the live source seal. diff --git a/changelog.d/child-property-income-graph.added.md b/changelog.d/child-property-income-graph.added.md new file mode 100644 index 000000000..829fe308c --- /dev/null +++ b/changelog.d/child-property-income-graph.added.md @@ -0,0 +1 @@ +Add an opt-in source-owned child property-income graph fragment with explicit private model-support schemas, paired empirical draws and independent full-population verification; integration and runtime acceptance remain separate. diff --git a/changelog.d/child-property-support-frame-stamp.fixed.md b/changelog.d/child-property-support-frame-stamp.fixed.md new file mode 100644 index 000000000..5b7b99b94 --- /dev/null +++ b/changelog.d/child-property-support-frame-stamp.fixed.md @@ -0,0 +1 @@ +Seal child-property donor and recipient Frames using their private support schema and complete physical contents across the final owner callback. Keep the receiving US population's strict source identity checks unchanged. diff --git a/changelog.d/child-receiving-fixture-strings.fixed.md b/changelog.d/child-receiving-fixture-strings.fixed.md new file mode 100644 index 000000000..6af7ff29f --- /dev/null +++ b/changelog.d/child-receiving-fixture-strings.fixed.md @@ -0,0 +1 @@ +Declare canonical nullable string dtypes in the invented child-property receiving fixture, preserving source-channel values, census-block leading zeros, exact large IDs, and income knownness. diff --git a/changelog.d/child-verification-operation.fixed.md b/changelog.d/child-verification-operation.fixed.md new file mode 100644 index 000000000..1a2eedd84 --- /dev/null +++ b/changelog.d/child-verification-operation.fixed.md @@ -0,0 +1 @@ +Reuse independently verified child completion reconstruction within bounded owner operations, retaining source checks, actual evidence comparisons and uncached closing verification. diff --git a/changelog.d/ci-file-shards.fixed.md b/changelog.d/ci-file-shards.fixed.md new file mode 100644 index 000000000..3825691c4 --- /dev/null +++ b/changelog.d/ci-file-shards.fixed.md @@ -0,0 +1 @@ +Split concentrated CI test groups and the full installed-wheel inventory into four parallel file shards per Python version, preserving module fixtures, environment coverage and process isolation, and fail before pytest when file selection fails or is empty. Enforce the flat inventory depth so nested tests are rejected by verification rather than absorbed by Git's ordinary wildcard matching. diff --git a/changelog.d/common-frame-export-contract.added.md b/changelog.d/common-frame-export-contract.added.md new file mode 100644 index 000000000..ea5fadddb --- /dev/null +++ b/changelog.d/common-frame-export-contract.added.md @@ -0,0 +1 @@ +Add supplied-parent comparison for US full, pruned and local frame exports, preserving retained inputs and missingness through checkpoint readback. diff --git a/changelog.d/country-bundle-shared-source-pins.fixed.md b/changelog.d/country-bundle-shared-source-pins.fixed.md new file mode 100644 index 000000000..089ac9751 --- /dev/null +++ b/changelog.d/country-bundle-shared-source-pins.fixed.md @@ -0,0 +1 @@ +Update the AM, BE and UK country-bundle identity checks after the reviewed calibration snapshot metadata changes. Their authored resources and draw-site bindings remain unchanged; their resolved spec digests include the shared solver source attestation. diff --git a/changelog.d/current-acs-immigration-source-projection.added.md b/changelog.d/current-acs-immigration-source-projection.added.md new file mode 100644 index 000000000..bb2cf7309 --- /dev/null +++ b/changelog.d/current-acs-immigration-source-projection.added.md @@ -0,0 +1 @@ +Add an opt-in selected ACS immigration source projection that binds original age, sex, household state and literal entry/birthplace evidence through existing owners, preserves NIU and grouped-code limits, and grants no status or stock admission. diff --git a/changelog.d/current-asec-child-support-source.added.md b/changelog.d/current-asec-child-support-source.added.md new file mode 100644 index 000000000..4b951e9a3 --- /dev/null +++ b/changelog.d/current-asec-child-support-source.added.md @@ -0,0 +1 @@ +Qualify retained ASEC child-support amounts, receipt and obligation answers, preserving paid NIU and unknown voluntary payments alongside original source and allocation metadata. diff --git a/changelog.d/current-asec-interest-source.added.md b/changelog.d/current-asec-interest-source.added.md new file mode 100644 index 000000000..73de9e322 --- /dev/null +++ b/changelog.d/current-asec-interest-source.added.md @@ -0,0 +1 @@ +Qualify ordinary and retirement-account ASEC interest from retained source owners, preserving published components, combined totals, knownness and allocation metadata without assigning tax treatment or balancing observations. diff --git a/changelog.d/current-survey-property-graph.added.md b/changelog.d/current-survey-property-graph.added.md new file mode 100644 index 000000000..45b5f9bd0 --- /dev/null +++ b/changelog.d/current-survey-property-graph.added.md @@ -0,0 +1 @@ +Add an opt-in 16-node survey property-income fragment with original DESIGN donor selection, joint ACS draws, signed reconciliation, explicit unknownness, and exact clone attachment while preserving legacy tax leaves and their stated capital-gains limitation. diff --git a/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md new file mode 100644 index 000000000..8f4a31c03 --- /dev/null +++ b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md @@ -0,0 +1 @@ +Bind fiscal measurement caches to target arithmetic and engine table materialization, refuse missing consumed predicate inputs, and count boolean terms arithmetically in shared sum expressions. diff --git a/changelog.d/full-asec-immigration-donor.added.md b/changelog.d/full-asec-immigration-donor.added.md new file mode 100644 index 000000000..334f0ad00 --- /dev/null +++ b/changelog.d/full-asec-immigration-donor.added.md @@ -0,0 +1 @@ +Add a checked full-original-ASEC immigration donor projection with exact household design-weight anchors, original demographic predictors and strict literal-token qualification, without status assignment or graph attachment. diff --git a/changelog.d/graph-scalar-float-packing.changed.md b/changelog.d/graph-scalar-float-packing.changed.md new file mode 100644 index 000000000..0cb0b67fe --- /dev/null +++ b/changelog.d/graph-scalar-float-packing.changed.md @@ -0,0 +1 @@ +Pack an exact Python float straight into the graph executor's scalar digest framing with `struct.pack("=d", ...)` instead of a one-element float64 array, preserving the payload bytes, the length prefix and every other scalar's behaviour. diff --git a/changelog.d/graph-signed-reconciliation.added.md b/changelog.d/graph-signed-reconciliation.added.md new file mode 100644 index 000000000..96be9e554 --- /dev/null +++ b/changelog.d/graph-signed-reconciliation.added.md @@ -0,0 +1 @@ +Expose signed-total component reconciliation as a graph node with explicit scales, bounds, and row-level adjustment diagnostics. diff --git a/changelog.d/graph-storage-parts-whole-series.changed.md b/changelog.d/graph-storage-parts-whole-series.changed.md new file mode 100644 index 000000000..afe3d3e1c --- /dev/null +++ b/changelog.d/graph-storage-parts-whole-series.changed.md @@ -0,0 +1 @@ +Select whole series with `slice(None)` instead of an all-True mask when folding physical storage parts, and skip the null recomputation that masked storage never uses, preserving the emitted value bytes, null bytes and subset-selection behaviour exactly. diff --git a/changelog.d/grouped-target-snapshot-integration.added.md b/changelog.d/grouped-target-snapshot-integration.added.md new file mode 100644 index 000000000..c5df41593 --- /dev/null +++ b/changelog.d/grouped-target-snapshot-integration.added.md @@ -0,0 +1 @@ +Instrument the grouped/fixed-zero Adam solver with the opt-in calibration target snapshot observer, reusing the estimate tensor each epoch's loss was computed from and the closing float64 diagnostics estimate. diff --git a/changelog.d/invented-enrichment-ss-universe.fixed.md b/changelog.d/invented-enrichment-ss-universe.fixed.md new file mode 100644 index 000000000..f0fa89fb8 --- /dev/null +++ b/changelog.d/invented-enrichment-ss-universe.fixed.md @@ -0,0 +1 @@ +Keep the invented enrichment fixture's Social Security nonreceipt answer consistent when its age-14 donor is deliberately changed to age 15, and check the genuine source qualifier preserves other unknown reporting statuses. diff --git a/changelog.d/joint-empirical-graph.added.md b/changelog.d/joint-empirical-graph.added.md new file mode 100644 index 000000000..668f16fc3 --- /dev/null +++ b/changelog.d/joint-empirical-graph.added.md @@ -0,0 +1 @@ +Add typed private joint empirical fit and keyed draw graph operations with exact source-coordinate lineage and explicit support and transport contracts. diff --git a/changelog.d/medicaid-substitution-hierarchy.fixed.md b/changelog.d/medicaid-substitution-hierarchy.fixed.md new file mode 100644 index 000000000..20e3e17fd --- /dev/null +++ b/changelog.d/medicaid-substitution-hierarchy.fixed.md @@ -0,0 +1,5 @@ +Preserve a complete, truthful calibration hierarchy when substituting Rhode +Island's reviewed November 2024 Medicaid enrollment count for its missing +December snapshot. Rebind the target identity and state labels/dimension while +retaining provider/category and source-substitution provenance; keep legacy +targets without hierarchies supported. diff --git a/changelog.d/native-aca-frame-context.fixed.md b/changelog.d/native-aca-frame-context.fixed.md new file mode 100644 index 000000000..d37658b04 --- /dev/null +++ b/changelog.d/native-aca-frame-context.fixed.md @@ -0,0 +1 @@ +Preserve Frame metadata and mass-change history when refreshing ACA marketplace inputs. diff --git a/changelog.d/native-acs-anchor-iteration.changed.md b/changelog.d/native-acs-anchor-iteration.changed.md new file mode 100644 index 000000000..a6e51289e --- /dev/null +++ b/changelog.d/native-acs-anchor-iteration.changed.md @@ -0,0 +1 @@ +Iterate retained ACS income-anchor columns without allocating full record-dictionary lists, preserving literal checks, exact projections, nullable amount bits, and final source-owner checks. diff --git a/changelog.d/native-acs-health-completion.added.md b/changelog.d/native-acs-health-completion.added.md new file mode 100644 index 000000000..c92ec3eb1 --- /dev/null +++ b/changelog.d/native-acs-health-completion.added.md @@ -0,0 +1 @@ +Add optional graph-based completion of seven ACS health coverage categories from authenticated original ASEC donors, preserving source knownness and drawing once per original person before clone transport. Scientific and release qualification remain separate. diff --git a/changelog.d/native-acs-spm-scope.added.md b/changelog.d/native-acs-spm-scope.added.md new file mode 100644 index 000000000..a43343a34 --- /dev/null +++ b/changelog.d/native-acs-spm-scope.added.md @@ -0,0 +1 @@ +Add a pure ACS 2024 household-only SPM scope classifier that preserves nullable roles, source-evidence labels and household-wide uncertainty, and validates native membership mappings without issuing source authority or running a country model. diff --git a/changelog.d/native-agi-tail-fixture-contract.added.md b/changelog.d/native-agi-tail-fixture-contract.added.md new file mode 100644 index 000000000..363e862a0 --- /dev/null +++ b/changelog.d/native-agi-tail-fixture-contract.added.md @@ -0,0 +1 @@ +Add a bounded, invented-fixture-only native PUF tail role-projection contract and deterministic selection algebra: five shared PUF filing statuses, int64/float64-only money with an exact-integer AGI floor decision, a descriptive nullable spouse-payload flag, per-donor weight treatment, full capital-gains and both-arm count reconciliation, and residual correction on a largest cell weight. Actual person allocation, source admission, population expansion and release qualification remain unimplemented. diff --git a/changelog.d/native-all-person-hours.changed.md b/changelog.d/native-all-person-hours.changed.md new file mode 100644 index 000000000..7270c7a5e --- /dev/null +++ b/changelog.d/native-all-person-hours.changed.md @@ -0,0 +1 @@ +Qualify usual-hours proposals for both original survey arms, retaining ASEC earnings evidence and explicitly labeling child zero completion; transport proposals through checked two-clone ancestry without redrawing or overwriting owned fields. diff --git a/changelog.d/native-asec-immigration-assignment.added.md b/changelog.d/native-asec-immigration-assignment.added.md new file mode 100644 index 000000000..a17e0f33b --- /dev/null +++ b/changelog.d/native-asec-immigration-assignment.added.md @@ -0,0 +1 @@ +Add a retained development immigration assignment over the complete original ASEC donor, using qualified current source evidence, 2025 observation years and original household design weights. Preserve original-key status pairs before selection, disclose unchanged mixed-date control assumptions, and keep national stock alignment unqualified. diff --git a/changelog.d/native-calibration-attachment.added.md b/changelog.d/native-calibration-attachment.added.md new file mode 100644 index 000000000..fd5e1c485 --- /dev/null +++ b/changelog.d/native-calibration-attachment.added.md @@ -0,0 +1 @@ +Add a private dense/L0 calibration bridge that attaches weights to unchanged projected inputs and verifies whole-household retention against the supplied parent without claiming native ancestry or release readiness. diff --git a/changelog.d/native-canonical-state-binding.changed.md b/changelog.d/native-canonical-state-binding.changed.md new file mode 100644 index 000000000..a808c2eed --- /dev/null +++ b/changelog.d/native-canonical-state-binding.changed.md @@ -0,0 +1 @@ +Add an independent graph fragment that binds canonical household state FIPS from existing block-derived geography and checks observed survey state equality. Explicit rewrites replace carried canonical values while preserving other inputs, rows, memberships, and weights. Enrichment-host integration and tenure-source completion remain separate. diff --git a/changelog.d/native-canonical-state-host.changed.md b/changelog.d/native-canonical-state-host.changed.md new file mode 100644 index 000000000..49a777ffe --- /dev/null +++ b/changelog.d/native-canonical-state-host.changed.md @@ -0,0 +1 @@ +Add an explicit default-off canonical state binding to native US enrichment, with a keep-all population boundary, typed geography/terminal dependencies, and complete reconstruction on cold and required replay. diff --git a/changelog.d/native-categorical-gate.added.md b/changelog.d/native-categorical-gate.added.md new file mode 100644 index 000000000..dabac088c --- /dev/null +++ b/changelog.d/native-categorical-gate.added.md @@ -0,0 +1 @@ +Add a separate DESIGN-weighted categorical classifier artifact and typed probability graph adapter with explicit class support, source/model/matrix bindings and no changes to QRF or canonical survey outputs. diff --git a/changelog.d/native-comparison-geography-id-binding.fixed.md b/changelog.d/native-comparison-geography-id-binding.fixed.md new file mode 100644 index 000000000..2aac9d5de --- /dev/null +++ b/changelog.d/native-comparison-geography-id-binding.fixed.md @@ -0,0 +1 @@ +Bind a US replacement-scorecard row's area identifier to the level and declaration that resolved it. The national/state/congressional-district view chose a level from one piece of evidence and then searched an ordered metadata key list that consulted neither, so a row without a view-level hierarchy id would have reported the parent `state_fips` the target compiler derives from a district geoid, or a sub-state hierarchy id as a state's. That search was unreachable for every spec the US compiler can currently emit -- each carries a hierarchy whose geography id is non-empty -- so no scorecard's counts change; this closes the hole before a producer reaches it. An identifier is now read only from a declaration at the row's own level, a row with none keeps an empty identifier, and the artifact reports which declaration named each area. Distinct-area counts are taken over census GEOIDs alone, with bare-form and unidentified rows counted beside them, so one area cannot be counted twice under two spellings. Scorecard schema 4 -> 5. diff --git a/changelog.d/native-comparison-geography-validation.fixed.md b/changelog.d/native-comparison-geography-validation.fixed.md new file mode 100644 index 000000000..39eeb8243 --- /dev/null +++ b/changelog.d/native-comparison-geography-validation.fixed.md @@ -0,0 +1 @@ +Validate geographic identifier encodings before counting census areas, preserve unsupported explicit scopes, and exclude unresolved scopes from the known-level missing-area count in schema-6 US comparison scorecards. diff --git a/changelog.d/native-comparison-national-cd-view.added.md b/changelog.d/native-comparison-national-cd-view.added.md new file mode 100644 index 000000000..b86f39d73 --- /dev/null +++ b/changelog.d/native-comparison-national-cd-view.added.md @@ -0,0 +1 @@ +Decompose the US head-to-head replacement scorecard on a national/state/congressional-district view. Every fiscal row now carries the comparison view its declared target evidence names, each artifact payload and the head-to-head comparison roll up on that axis, and a row no declared evidence places is reported `unresolved` rather than counted as national. This is the previously unproduced `national_and_cd_target_fit` item of the native survey release-evidence list; it adds no threshold, tolerance or verdict. diff --git a/changelog.d/native-completion-retention.changed.md b/changelog.d/native-completion-retention.changed.md new file mode 100644 index 000000000..61c8cb04b --- /dev/null +++ b/changelog.d/native-completion-retention.changed.md @@ -0,0 +1 @@ +Add private opt-in compact observation retention for the native financial and full completion graph paths, preserving the default retained snapshots and replay checks. diff --git a/changelog.d/native-consumer-metadata.added.md b/changelog.d/native-consumer-metadata.added.md new file mode 100644 index 000000000..b9219b33e --- /dev/null +++ b/changelog.d/native-consumer-metadata.added.md @@ -0,0 +1 @@ +Add an optional live-consumer representation check for exact native survey projections, using actual variable dtypes, enum domains, periods and source ownership without filling inputs or claiming release qualification. diff --git a/changelog.d/native-csv-record-scan.changed.md b/changelog.d/native-csv-record-scan.changed.md new file mode 100644 index 000000000..276b3c91f --- /dev/null +++ b/changelog.d/native-csv-record-scan.changed.md @@ -0,0 +1 @@ +Speed up bounded, complete unquoted ASEC CSV record checks while preserving byte-level limits, budget accounting, source authentication and refusal behavior. diff --git a/changelog.d/native-engine-projection-20260919.added.md b/changelog.d/native-engine-projection-20260919.added.md new file mode 100644 index 000000000..a241253fa --- /dev/null +++ b/changelog.d/native-engine-projection-20260919.added.md @@ -0,0 +1 @@ +Add an in-process native development projection that preserves selected source cells and context exactly, while keeping consumer qualification, simulation readiness, and release eligibility false. diff --git a/changelog.d/native-financial-check-epoch.changed.md b/changelog.d/native-financial-check-epoch.changed.md new file mode 100644 index 000000000..3ad59aa87 --- /dev/null +++ b/changelog.d/native-financial-check-epoch.changed.md @@ -0,0 +1 @@ +Reuse authenticated preparation validation within one native financial run check. Every independent check still fully validates its sources at entry and close; checks inside an existing verification epoch join that scope, whose close must succeed before the caller returns. Direct Frame seals and source-change refusals remain enabled. Changes invisible to the memo signature may be refused at the enclosing close instead of an intervening borrow. diff --git a/changelog.d/native-full-original-amount-donors.added.md b/changelog.d/native-full-original-amount-donors.added.md new file mode 100644 index 000000000..cefb18f57 --- /dev/null +++ b/changelog.d/native-full-original-amount-donors.added.md @@ -0,0 +1 @@ +Add optional full-original ASEC DESIGN donor support for native current survey amount fitting, independent of selected receiving support and preserving observed amounts, recipient applicability and existing clones. diff --git a/changelog.d/native-handoff-engine-free.fixed.md b/changelog.d/native-handoff-engine-free.fixed.md new file mode 100644 index 000000000..299bf7af0 --- /dev/null +++ b/changelog.d/native-handoff-engine-free.fixed.md @@ -0,0 +1 @@ +Keep native development checkpoint inspection free of implicit country-engine loading. Its report now identifies the historical and release-source rosters checked and explicitly leaves the legacy pool surface and live take-up ABI inventory unqualified; explicit export-contract diagnostics and release gates remain unchanged. diff --git a/changelog.d/native-hours-graph.changed.md b/changelog.d/native-hours-graph.changed.md new file mode 100644 index 000000000..e70abd979 --- /dev/null +++ b/changelog.d/native-hours-graph.changed.md @@ -0,0 +1 @@ +Execute authenticated ACS and ASEC usual-hours recoding as native graph operations and attach original-person hours and provenance to exact clones after survey enrichment, with independent replay checks. diff --git a/changelog.d/native-hours-handoff-controls.fixed.md b/changelog.d/native-hours-handoff-controls.fixed.md new file mode 100644 index 000000000..67b0d0272 --- /dev/null +++ b/changelog.d/native-hours-handoff-controls.fixed.md @@ -0,0 +1 @@ +Classify native development handoff and retained population witnesses in the runtime inventory, check hours/provenance checkpoint preservation, and recognize only the exact reviewed SPM receipt transfer from Frame metadata to its source owner. diff --git a/changelog.d/native-hours-response-status.fixed.md b/changelog.d/native-hours-response-status.fixed.md new file mode 100644 index 000000000..a6e4f6638 --- /dev/null +++ b/changelog.d/native-hours-response-status.fixed.md @@ -0,0 +1 @@ +Accept and preserve the Census ASEC complete supplement nonresponse code in usual-hours proposals without treating it as source or donor qualification. diff --git a/changelog.d/native-hours-source-owner.added.md b/changelog.d/native-hours-source-owner.added.md new file mode 100644 index 000000000..6ca1145ff --- /dev/null +++ b/changelog.d/native-hours-source-owner.added.md @@ -0,0 +1 @@ +Add a retained source owner for ACS usual-hours proposals, binding original survey records and the complete current ASEC age-15 donor cohort independently of selected household support. ASEC all-age hours and release qualification remain separate. diff --git a/changelog.d/native-household-role-namespace.fixed.md b/changelog.d/native-household-role-namespace.fixed.md new file mode 100644 index 000000000..868a31435 --- /dev/null +++ b/changelog.d/native-household-role-namespace.fixed.md @@ -0,0 +1 @@ +Preserve legacy prepared headship separately from source-qualified nullable household roles, retaining disagreement diagnostics and strict canonical-role checks. diff --git a/changelog.d/native-immigration-clone-attachment.added.md b/changelog.d/native-immigration-clone-attachment.added.md new file mode 100644 index 000000000..55df31bd7 --- /dev/null +++ b/changelog.d/native-immigration-clone-attachment.added.md @@ -0,0 +1 @@ +Optionally attach a genuine original-survey immigration transfer's realized pair to both existing survey clones through the retained enrichment host, preserving source ancestry, geography, weights and replay checks without another fit or draw. diff --git a/changelog.d/native-immigration-consumed-controls.added.md b/changelog.d/native-immigration-consumed-controls.added.md new file mode 100644 index 000000000..7b89170ba --- /dev/null +++ b/changelog.d/native-immigration-consumed-controls.added.md @@ -0,0 +1 @@ +Allow ACS immigration reconciliation to consume explicitly captured controls, including after target-bank replay, and parse those controls from a captured source stage while preserving packaged defaults. diff --git a/changelog.d/native-immigration-observation-years.added.md b/changelog.d/native-immigration-observation-years.added.md new file mode 100644 index 000000000..9b5859331 --- /dev/null +++ b/changelog.d/native-immigration-observation-years.added.md @@ -0,0 +1 @@ +Adopt the reviewed immigration rule and paired ACS reconciliation surface, with explicit per-person observation years, current labor-force inputs, year-independent absent-arrival features, and feature-bound checkpoint replay. Preserve existing control values and the default scalar observation year; native source assignment remains separate. diff --git a/changelog.d/native-immigration-source-bundle.fixed.md b/changelog.d/native-immigration-source-bundle.fixed.md new file mode 100644 index 000000000..27c0cd932 --- /dev/null +++ b/changelog.d/native-immigration-source-bundle.fixed.md @@ -0,0 +1 @@ +Regenerate the typed immigration source bundle from the reviewed source manifest, declare its closed humanitarian-stock schema, and check source projection and renderer equality without requiring the optional country engine. diff --git a/changelog.d/native-immigration-source.added.md b/changelog.d/native-immigration-source.added.md new file mode 100644 index 000000000..e3de3341f --- /dev/null +++ b/changelog.d/native-immigration-source.added.md @@ -0,0 +1 @@ +Add a native immigration source qualifier that retains the complete current ASEC readset and selected ACS literals, with exact source custody and clone evidence transport. Status assignment, stock alignment and release qualification remain separate reviewed steps. diff --git a/changelog.d/native-immigration-transfer-spec.fixed.md b/changelog.d/native-immigration-transfer-spec.fixed.md new file mode 100644 index 000000000..1e63e7b3c --- /dev/null +++ b/changelog.d/native-immigration-transfer-spec.fixed.md @@ -0,0 +1 @@ +Regenerate the typed ACS transfer declaration from its live v4 contract without country-engine metadata, retain immigration evidence and humanitarian reconciliation semantics, and match existing adult-care and Schedule D activation rules. diff --git a/changelog.d/native-input-name-profile.added.md b/changelog.d/native-input-name-profile.added.md new file mode 100644 index 000000000..34c433089 --- /dev/null +++ b/changelog.d/native-input-name-profile.added.md @@ -0,0 +1 @@ +Add an explicit 159-name native US national/CD diagnostic profile that excludes the canonical prior-year income family and reports those scope exclusions. Preserve the existing 163-name historical and 161-name national/CD profiles and their defaults; native source and release qualification remain separate. diff --git a/changelog.d/native-original-immigration-transfer.added.md b/changelog.d/native-original-immigration-transfer.added.md new file mode 100644 index 000000000..3ebc4ce5b --- /dev/null +++ b/changelog.d/native-original-immigration-transfer.added.md @@ -0,0 +1 @@ +Add a retained paired ACS immigration transfer that uses the complete original ASEC donor and the same native graph parent's original survey allocation, with explicit consumed controls and unchanged source weights. diff --git a/changelog.d/native-other-disability-completion.added.md b/changelog.d/native-other-disability-completion.added.md new file mode 100644 index 000000000..da87de8b1 --- /dev/null +++ b/changelog.d/native-other-disability-completion.added.md @@ -0,0 +1 @@ +Add an optional development graph for other-disability amount completion from full-original ASEC DESIGN donors to original ACS recipients, preserving unknown source states and exact clone transport. Exact nullable source reports remain in a declared artifact; default enrichment and release eligibility are unchanged. diff --git a/changelog.d/native-other-disability-context-replay.fixed.md b/changelog.d/native-other-disability-context-replay.fixed.md new file mode 100644 index 000000000..4c78d6842 --- /dev/null +++ b/changelog.d/native-other-disability-context-replay.fixed.md @@ -0,0 +1 @@ +Compare disability continuation manifest Frames and mass ledgers symmetrically while preserving strict full-context equality with the retained receiving population. diff --git a/changelog.d/native-other-disability-qualification.fixed.md b/changelog.d/native-other-disability-qualification.fixed.md new file mode 100644 index 000000000..393bf28a0 --- /dev/null +++ b/changelog.d/native-other-disability-qualification.fixed.md @@ -0,0 +1 @@ +Recheck the other-disability adapter's implementation after its final source-owner validation, refusing semantic or qualifier changes before returning observations. diff --git a/changelog.d/native-primary-family-source.added.md b/changelog.d/native-primary-family-source.added.md new file mode 100644 index 000000000..83755b09e --- /dev/null +++ b/changelog.d/native-primary-family-source.added.md @@ -0,0 +1 @@ +Add source-qualified Census primary-family, own-child, household-size, and separate resident-spouse-pair diagnostics with explicit unknown and group-quarters states. diff --git a/changelog.d/native-puf-original-carried-outputs.fixed.md b/changelog.d/native-puf-original-carried-outputs.fixed.md new file mode 100644 index 000000000..6a6622404 --- /dev/null +++ b/changelog.d/native-puf-original-carried-outputs.fixed.md @@ -0,0 +1 @@ +Accept the receiving structural version as the owner of carried arm-one PUF outputs in the original placement cut while proving the carried values are unchanged. diff --git a/changelog.d/native-puf-original-host.added.md b/changelog.d/native-puf-original-host.added.md new file mode 100644 index 000000000..555f4fedd --- /dev/null +++ b/changelog.d/native-puf-original-host.added.md @@ -0,0 +1 @@ +Add opt-in original-arm PUF55 application and conservative placement to the survey enrichment host, preserving the actual selected terminal and independent source, artifact, population, and storage checks. Keep the default path unchanged; full host and release qualification remain separate from bounded component verification. diff --git a/changelog.d/native-puf-original-placement-artifact-replay.fixed.md b/changelog.d/native-puf-original-placement-artifact-replay.fixed.md new file mode 100644 index 000000000..0b24b167f --- /dev/null +++ b/changelog.d/native-puf-original-placement-artifact-replay.fixed.md @@ -0,0 +1 @@ +Persist the input population versions instead of the in-process population seal in the original PUF placement artifact so the required replay reconstructs it byte-for-byte. diff --git a/changelog.d/native-puf-original-placement.added.md b/changelog.d/native-puf-original-placement.added.md new file mode 100644 index 000000000..301e3f638 --- /dev/null +++ b/changelog.d/native-puf-original-placement.added.md @@ -0,0 +1 @@ +Add conservative original-arm PUF55 development placement helpers and typed graph declarations. Preserve known and preexisting values, leave mixed-known units unresolved, and permit five new person values only on complete eligible singletons. The three tax-unit outputs and all 55 upstream application dependencies are explicit; checked-host integration and release qualification remain separate. diff --git a/changelog.d/native-puf-original-source-version.fixed.md b/changelog.d/native-puf-original-source-version.fixed.md new file mode 100644 index 000000000..f6edba787 --- /dev/null +++ b/changelog.d/native-puf-original-source-version.fixed.md @@ -0,0 +1 @@ +Open a private keep-all version for the original PUF arm's recipient and fixed-input nodes so they no longer join the financial version closure and change the authenticated parent's keys. diff --git a/changelog.d/native-row-ceilings.changed.md b/changelog.d/native-row-ceilings.changed.md new file mode 100644 index 000000000..16d60b926 --- /dev/null +++ b/changelog.d/native-row-ceilings.changed.md @@ -0,0 +1 @@ +Raise the seven US native-build row-count ceilings a full-source build meets to four times their measured full-source counts, keeping every refusal code and expression, and re-pin the one implementation digest that moves. diff --git a/changelog.d/native-runtime-helper-classification.fixed.md b/changelog.d/native-runtime-helper-classification.fixed.md new file mode 100644 index 000000000..fe22733a4 --- /dev/null +++ b/changelog.d/native-runtime-helper-classification.fixed.md @@ -0,0 +1 @@ +Classify the shared prior-year income names and logical H5 readback verifier in the exhaustive US runtime inventory, preserving the checks for unreviewed modules and population-treatment boundaries. diff --git a/changelog.d/native-soi-agi-size-targets.added.md b/changelog.d/native-soi-agi-size-targets.added.md new file mode 100644 index 000000000..c5a4ff679 --- /dev/null +++ b/changelog.d/native-soi-agi-size-targets.added.md @@ -0,0 +1 @@ +Adopt the narrow SOI Table 1.1 cross-period size-of-AGI compiler from #960 for native integration, preserving source identities and control-year alignment with additional invented exclusion and aging tests. No feed, target-surface or dataset pins change. diff --git a/changelog.d/native-spm-amount-source.added.md b/changelog.d/native-spm-amount-source.added.md new file mode 100644 index 000000000..c56205172 --- /dev/null +++ b/changelog.d/native-spm-amount-source.added.md @@ -0,0 +1 @@ +Add a source-only SPM amount projection that borrows existing current ASEC money and original membership owners, preserves repeated unit values and unresolved zeros, and leaves missing observations explicit. diff --git a/changelog.d/native-spm-complete-unit-projection.added.md b/changelog.d/native-spm-complete-unit-projection.added.md new file mode 100644 index 000000000..a651e06f1 --- /dev/null +++ b/changelog.d/native-spm-complete-unit-projection.added.md @@ -0,0 +1,4 @@ +Add a pure annual SPM source-input projection that verifies complete original +SPM-unit and household membership through both native clones. Preserve nullable +source roles and require explicit OUTSIDE scope before applying a Boolean +storage placeholder; refuse changed years, ancestry or existing output columns. diff --git a/changelog.d/native-spm-country-source-inputs.fixed.md b/changelog.d/native-spm-country-source-inputs.fixed.md new file mode 100644 index 000000000..776c7124f --- /dev/null +++ b/changelog.d/native-spm-country-source-inputs.fixed.md @@ -0,0 +1 @@ +Honor PolicyEngine-US's declared dataset source inputs in the live adapter and import-free metadata index, including SPM independence roles and annual universe status despite household fallback formulas. Reject malformed or conflicting declarations, retain formula-owned output guards, and omit source observations from generated defaults. diff --git a/changelog.d/native-spm-enrichment-graph.added.md b/changelog.d/native-spm-enrichment-graph.added.md new file mode 100644 index 000000000..8cc467290 --- /dev/null +++ b/changelog.d/native-spm-enrichment-graph.added.md @@ -0,0 +1 @@ +Add an explicitly configured three-node survey enrichment fragment for retained SPM roles and annual universe scope, with complete-unit transport, replay verification and final source/output validation. diff --git a/changelog.d/native-spm-retained-source.added.md b/changelog.d/native-spm-retained-source.added.md new file mode 100644 index 000000000..fa2d3a9de --- /dev/null +++ b/changelog.d/native-spm-retained-source.added.md @@ -0,0 +1 @@ +Add retained-owner qualification of nullable ACS and ASEC SPM roles and explicit annual scope without granting dataset or release certification. diff --git a/changelog.d/native-ss-full-original-basis.added.md b/changelog.d/native-ss-full-original-basis.added.md new file mode 100644 index 000000000..fb5890af5 --- /dev/null +++ b/changelog.d/native-ss-full-original-basis.added.md @@ -0,0 +1 @@ +Add an optional full-original ASEC Social Security report basis with native IDs, original DESIGN weights, source literals and unchanged selected-survey descriptions, leaving donor selection and component modeling separate. diff --git a/changelog.d/native-ss-report-completion.added.md b/changelog.d/native-ss-report-completion.added.md new file mode 100644 index 000000000..49226937b --- /dev/null +++ b/changelog.d/native-ss-report-completion.added.md @@ -0,0 +1 @@ +Add an optional source-qualified Social Security report fragment with full-original ASEC DESIGN donors, real categorical fit/probability artifacts, reason-constrained report completion, and a no-fit empty-recipient path. It preserves source unknowns and does not write canonical beneficiary inputs or qualify a release. diff --git a/changelog.d/native-survey-development-handoff.added.md b/changelog.d/native-survey-development-handoff.added.md new file mode 100644 index 000000000..49a89093b --- /dev/null +++ b/changelog.d/native-survey-development-handoff.added.md @@ -0,0 +1 @@ +Add a checked native survey development handoff in the maintained US release builder, preserving complete Frame checkpoints and source/model flags while reporting missing inputs and outstanding release qualifications without cloning or fabricating pool receipts. diff --git a/changelog.d/native-survey-development-inputs.added.md b/changelog.d/native-survey-development-inputs.added.md new file mode 100644 index 000000000..54a69dab9 --- /dev/null +++ b/changelog.d/native-survey-development-inputs.added.md @@ -0,0 +1 @@ +Add an opt-in graph producer for four source-qualified ASEC development inputs, preserving source unknowns and exact original clone mappings before PUF application. diff --git a/changelog.d/native-survey-household-domains.added.md b/changelog.d/native-survey-household-domains.added.md new file mode 100644 index 000000000..b9564213e --- /dev/null +++ b/changelog.d/native-survey-household-domains.added.md @@ -0,0 +1 @@ +Add a source-owned domain projection for the exact survey allocation and raw clone0/1 prefix, retaining group-quarter exclusions, rational design anchors and live verification custody without claiming downstream matching or release authority. diff --git a/changelog.d/native-survey-race-hispanic-inputs.added.md b/changelog.d/native-survey-race-hispanic-inputs.added.md new file mode 100644 index 000000000..b45253d6b --- /dev/null +++ b/changelog.d/native-survey-race-hispanic-inputs.added.md @@ -0,0 +1 @@ +Add optional source-qualified race and Hispanic-origin graph inputs on original survey persons and existing clones, preserving source literals, allocation histories and unsupported ACS race categories as explicit unknowns. diff --git a/changelog.d/native-survey-sex-inputs.added.md b/changelog.d/native-survey-sex-inputs.added.md new file mode 100644 index 000000000..ee0e309b3 --- /dev/null +++ b/changelog.d/native-survey-sex-inputs.added.md @@ -0,0 +1 @@ +Add opt-in native sex inputs to survey enrichment. The graph exposes qualified ASEC sex/allocation and ACS sex observations, binds nullable `is_female` once per original person, then copies it through the existing clone mapping without filling unknowns or changing weights. diff --git a/changelog.d/native-tail-invented-matching.added.md b/changelog.d/native-tail-invented-matching.added.md new file mode 100644 index 000000000..32995c810 --- /dev/null +++ b/changelog.d/native-tail-invented-matching.added.md @@ -0,0 +1,5 @@ +Add an invented-only native AGI donor matching contract that uses the maintained +selector, explicit income bases and known recipient roles to declare unique +compatible whole-household support expansions. The contract preserves the +approved equal-half household mass policy and explicitly defers actual source +qualification, monetary placement and geography. diff --git a/changelog.d/native-tail-structural-expansion.added.md b/changelog.d/native-tail-structural-expansion.added.md new file mode 100644 index 000000000..33c2cdc3c --- /dev/null +++ b/changelog.d/native-tail-structural-expansion.added.md @@ -0,0 +1,5 @@ +Add an invented-only native tail graph EXPAND that copies complete selected +households, preserves lineage and source anchors, and splits parent IMPORTANCE +weights exactly in half. Add an opt-in executor check for a sole stored weight +entity on cold and cached paths. Actual donor/domain qualification, monetary +placement and release integration remain separate requirements. diff --git a/changelog.d/native-usual-hours-proposals.added.md b/changelog.d/native-usual-hours-proposals.added.md new file mode 100644 index 000000000..006494317 --- /dev/null +++ b/changelog.d/native-usual-hours-proposals.added.md @@ -0,0 +1 @@ +Add pure native-keyed ACS usual-hours proposals with explicit age15 empirical and under15 modeled-zero policies, preserved source/allocation evidence, and invented validation. Native source and release qualification remain separate. diff --git a/changelog.d/native-verify-once.changed.md b/changelog.d/native-verify-once.changed.md new file mode 100644 index 000000000..60d5f6687 --- /dev/null +++ b/changelog.d/native-verify-once.changed.md @@ -0,0 +1 @@ +verify-once: Verify native sources once per run instead of once per accessor use and once per executed node. A run that the run-end re-derivation refuses now evicts every store object it published, so nothing it wrote survives under the source identity it started with; a run that leaves by any other exception gets the same re-derivation on its way out, keeping its work when its sources held still and evicting it otherwise. A process that stops without unwinding, after a rewrite that left every stat field identical, is not covered. diff --git a/changelog.d/native-veterans-benefits.added.md b/changelog.d/native-veterans-benefits.added.md new file mode 100644 index 000000000..ec11292c4 --- /dev/null +++ b/changelog.d/native-veterans-benefits.added.md @@ -0,0 +1 @@ +Add an opt-in native veterans’ benefits amount route using qualified full-original ASEC donors, preserved receipt/allocation unknownness, original-person ACS draws and explicit canonical clone attachment. diff --git a/changelog.d/native-workers-compensation.added.md b/changelog.d/native-workers-compensation.added.md new file mode 100644 index 000000000..53d87684b --- /dev/null +++ b/changelog.d/native-workers-compensation.added.md @@ -0,0 +1 @@ +Add an opt-in native workers compensation amount group using authenticated ASEC receipt evidence, preserving unresolved reporting status and sharing one ACS draw across support clones. diff --git a/changelog.d/other-disability-filter-reconstruction.fixed.md b/changelog.d/other-disability-filter-reconstruction.fixed.md new file mode 100644 index 000000000..e9c1bc1ce --- /dev/null +++ b/changelog.d/other-disability-filter-reconstruction.fixed.md @@ -0,0 +1 @@ +Materialize the disability donor and receiving keep-all FILTER Frames before independently checking their structural population patches. diff --git a/changelog.d/person-status-tax-filter-ownership.fixed.md b/changelog.d/person-status-tax-filter-ownership.fixed.md new file mode 100644 index 000000000..5c3729b63 --- /dev/null +++ b/changelog.d/person-status-tax-filter-ownership.fixed.md @@ -0,0 +1 @@ +Validate descriptive survey-status provenance on the retained pre-tax population while preserving the tax FILTER's current ownership, full-population seals and independent tax reconstruction. diff --git a/changelog.d/poverty-comparison-only.changed.md b/changelog.d/poverty-comparison-only.changed.md new file mode 100644 index 000000000..c9e665ce1 --- /dev/null +++ b/changelog.d/poverty-comparison-only.changed.md @@ -0,0 +1 @@ +Clarify that survey poverty is a comparison diagnostic and cannot guide calibration, informal tuning, candidate selection or release gates based on benchmark proximity. Preserve independent measurement and implementation correctness checks. diff --git a/changelog.d/pr893-wheel-fixture-contracts.fixed.md b/changelog.d/pr893-wheel-fixture-contracts.fixed.md new file mode 100644 index 000000000..b2831932e --- /dev/null +++ b/changelog.d/pr893-wheel-fixture-contracts.fixed.md @@ -0,0 +1 @@ +Gate pool orchestration tests that require US engine metadata, preserve source-enrichment checkout rejection in wheel tests, and isolate invented UK calibration code-pin fixtures while retaining refusal before I/O. diff --git a/changelog.d/pre-geography-survey-financial.added.md b/changelog.d/pre-geography-survey-financial.added.md new file mode 100644 index 000000000..c33b0e25e --- /dev/null +++ b/changelog.d/pre-geography-survey-financial.added.md @@ -0,0 +1 @@ +Add an explicitly issued pre-geography US survey-financial variant for completing the initial clone spine before assigning geography. Preserve the atomic path and bind PUF consumers to the two exact retained financial-owner variants. diff --git a/changelog.d/property-completion-host.added.md b/changelog.d/property-completion-host.added.md new file mode 100644 index 000000000..d6d8c1b15 --- /dev/null +++ b/changelog.d/property-completion-host.added.md @@ -0,0 +1 @@ +Add opt-in property completion diagnostics to the existing source projection, with a private typed row artifact, allowlisted aggregate graph receipts and independent replay checks. Preserve default options and numerical property behavior. diff --git a/changelog.d/property-completion-routing.added.md b/changelog.d/property-completion-routing.added.md new file mode 100644 index 000000000..b7e842805 --- /dev/null +++ b/changelog.d/property-completion-routing.added.md @@ -0,0 +1 @@ +Add deterministic original-person property completion routing, independent interest/dividend knownness, exact clone identity diagnostics and original DESIGN support summaries. The pure operation assigns no amounts and grants no source or release authority; host artifact wiring remains separate. diff --git a/changelog.d/property-independent-observation-preservation.internal.md b/changelog.d/property-independent-observation-preservation.internal.md new file mode 100644 index 000000000..cb1fc6a60 --- /dev/null +++ b/changelog.d/property-independent-observation-preservation.internal.md @@ -0,0 +1 @@ +Add a focused invented-source regression confirming that joint property-donor exclusions preserve independently qualified ASEC interest and dividends on both initial clones and through the existing tax split. Production behavior is unchanged. diff --git a/changelog.d/property-model-receipts.added.md b/changelog.d/property-model-receipts.added.md new file mode 100644 index 000000000..de28e20d2 --- /dev/null +++ b/changelog.d/property-model-receipts.added.md @@ -0,0 +1 @@ +Add source-blind verification of the four property fit/apply receipt pairs against authenticated original donor/recipient branches, with complete model history checks and deterministic application replay without refitting. diff --git a/changelog.d/property-survivor-scope.fixed.md b/changelog.d/property-survivor-scope.fixed.md new file mode 100644 index 000000000..4b9258e1c --- /dev/null +++ b/changelog.d/property-survivor-scope.fixed.md @@ -0,0 +1 @@ +Exclude positive survivor receipts from the initial property donor bridge because its two qualified source slots do not cover the additional sources included in the published aggregate. Preserve visible route diagnostics and report the excluded design-weight mass. diff --git a/changelog.d/puf-observer-python313.fixed.md b/changelog.d/puf-observer-python313.fixed.md new file mode 100644 index 000000000..17e72fbcb --- /dev/null +++ b/changelog.d/puf-observer-python313.fixed.md @@ -0,0 +1 @@ +Make PUF graph test observer cleanup compatible with Python 3.13 by explicitly unregistering owned callbacks after disabling events; verify normal and exceptional cleanup and tool reuse. diff --git a/changelog.d/puf55-canonical-donor.added.md b/changelog.d/puf55-canonical-donor.added.md new file mode 100644 index 000000000..10e06ae48 --- /dev/null +++ b/changelog.d/puf55-canonical-donor.added.md @@ -0,0 +1 @@ +Add a canonical59 artifact adapter for the explicit PUF55 survey Social Security profile, retaining return identity and weights and validating the total-carrier convention. Record the scoped passing native seven-node age-development run. diff --git a/changelog.d/puf55-checked-output.added.md b/changelog.d/puf55-checked-output.added.md new file mode 100644 index 000000000..c5fe0d86c --- /dev/null +++ b/changelog.d/puf55-checked-output.added.md @@ -0,0 +1 @@ +Retain PUF55 output handles for source, producer, store, and complete-population rechecks before downstream consumption, revoking a handle after a failed check without rerunning imputation. diff --git a/changelog.d/puf55-survey-host.added.md b/changelog.d/puf55-survey-host.added.md new file mode 100644 index 000000000..f0e410b85 --- /dev/null +++ b/changelog.d/puf55-survey-host.added.md @@ -0,0 +1 @@ +Add the PUF55 host over the combined ACS/ASEC support-clone population, with source-qualified Social Security routes, complete population preservation and required graph replay. diff --git a/changelog.d/puf55-survey-social-security.added.md b/changelog.d/puf55-survey-social-security.added.md new file mode 100644 index 000000000..ba625b57e --- /dev/null +++ b/changelog.d/puf55-survey-social-security.added.md @@ -0,0 +1 @@ +Add an explicit 55-output PUF profile that preserves survey Social Security components and requires a separate total as its ninth conditioning predictor. Validate canonical donor projection and whole-Population attachment/replay while retaining source identities, weights and Social Security unknownness. diff --git a/changelog.d/qrf-observed-matrix-conditioning.added.md b/changelog.d/qrf-observed-matrix-conditioning.added.md new file mode 100644 index 000000000..b1eeeceac --- /dev/null +++ b/changelog.d/qrf-observed-matrix-conditioning.added.md @@ -0,0 +1 @@ +Add an opt-in QRF matrix graph kernel that applies every fitted target, preserves raw draws, and uses identity-bound qualified values to condition later targets. A separate typed conditioning artifact and versioned state record draw, observation, matrix, and producer identities without changing the legacy application protocol. diff --git a/changelog.d/reform-claim-identity.changed.md b/changelog.d/reform-claim-identity.changed.md new file mode 100644 index 000000000..6d4ff055c --- /dev/null +++ b/changelog.d/reform-claim-identity.changed.md @@ -0,0 +1,10 @@ +Add optional benchmark publishers and executable reform definitions to schema-v1 +reform validation reports, ported from PR629 at +`a0441178c9a74bf55dd5aeea62ff9039092c3432`. Preserve scores, years, signs, model +calls, and the `microcosm` payload key; the eight shipped configs gain only the +229 publisher fields from that source. + +Record the actual baseline and reform worlds for simulated comparisons, +including each OBBBA stack step and its merged initial counterfactual when used. +Configured OBBBA repeal definitions remain distinct from those executed +comparisons. Copy definitions so report mutations cannot change input specs. diff --git a/changelog.d/scorer-hdf-engine-gating.fixed.md b/changelog.d/scorer-hdf-engine-gating.fixed.md new file mode 100644 index 000000000..fadf8dfc9 --- /dev/null +++ b/changelog.d/scorer-hdf-engine-gating.fixed.md @@ -0,0 +1 @@ +Mark the four scorer HDF loader tests as requiring the US engine, preserving engine-free installed-wheel source-I/O coverage and execution of the loader tests in the US engine matrix. diff --git a/changelog.d/shared-atomic-geography.added.md b/changelog.d/shared-atomic-geography.added.md new file mode 100644 index 000000000..4eed5d1a9 --- /dev/null +++ b/changelog.d/shared-atomic-geography.added.md @@ -0,0 +1 @@ +Add shared graph operators for atomic-area assignment and geographic derivation. Country declarations choose observed constraints, stable draw identities, sampling stages and versioned mapping relations. Preserve observed locations, reject ambiguous support, and verify mapping consistency on cloned or pruned populations. Existing country builds retain their current geography paths pending native support integration. Normalize US block mappings, qualify observed ACS/ASEC geography against source receipts, and compose shared assignment before the combined-survey clone, with an inherited-mapping gate afterward. diff --git a/changelog.d/signed-income-reconciliation.added.md b/changelog.d/signed-income-reconciliation.added.md new file mode 100644 index 000000000..428496f2c --- /dev/null +++ b/changelog.d/signed-income-reconciliation.added.md @@ -0,0 +1,2 @@ +Add a deterministic weighted projection onto a signed total with explicit +nonnegative component bounds, adjustment diagnostics and floating-point checks. diff --git a/changelog.d/source-io-wheel-extra.fixed.md b/changelog.d/source-io-wheel-extra.fixed.md new file mode 100644 index 000000000..9e8232f55 --- /dev/null +++ b/changelog.d/source-io-wheel-extra.fixed.md @@ -0,0 +1 @@ +Declare an engine-free HDF source extra and install it in the isolated wheel test environment. Verify an invented pandas HDF round trip before source tests, while retaining the base wheel import boundary and excluding the US rules engine. diff --git a/changelog.d/ss-beneficiary-convention.added.md b/changelog.d/ss-beneficiary-convention.added.md new file mode 100644 index 000000000..71508af20 --- /dev/null +++ b/changelog.d/ss-beneficiary-convention.added.md @@ -0,0 +1,6 @@ +Add a development-only Social Security report-lot allocation core with an +explicit opt-in singleton ASEC retirement convention. Preserve unknowns and +report zeros as evidence, refuse unresolved or duplicate lots at the consumer +subset boundary, and keep report classifier scores separate from beneficiary +incidence. Source qualification, graph attachment and engine compatibility +remain separate requirements. diff --git a/changelog.d/store-verified-bytes.fixed.md b/changelog.d/store-verified-bytes.fixed.md new file mode 100644 index 000000000..d4215726c --- /dev/null +++ b/changelog.d/store-verified-bytes.fixed.md @@ -0,0 +1 @@ +Return the exact verified graph-store byte payload, preventing a file replacement between verification and return from supplying unchecked bytes. diff --git a/changelog.d/survey-final-observation-seal.fixed.md b/changelog.d/survey-final-observation-seal.fixed.md new file mode 100644 index 000000000..bfa186d97 --- /dev/null +++ b/changelog.d/survey-final-observation-seal.fixed.md @@ -0,0 +1 @@ +Recheck the complete detached survey population observations after final source and artifact I/O, so late mutations cannot escape through the values retained for downstream graph composition. diff --git a/changelog.d/survey-social-security-source.added.md b/changelog.d/survey-social-security-source.added.md new file mode 100644 index 000000000..11d76e2dd --- /dev/null +++ b/changelog.d/survey-social-security-source.added.md @@ -0,0 +1 @@ +Add current survey Social Security source qualification with explicit reporting units, question-universe unknownness, allocation literals and component mapping judgments. diff --git a/changelog.d/uk-atomic-household-lineage.added.md b/changelog.d/uk-atomic-household-lineage.added.md new file mode 100644 index 000000000..b40a2a605 --- /dev/null +++ b/changelog.d/uk-atomic-household-lineage.added.md @@ -0,0 +1 @@ +Add a pure UK household-lineage adapter that derives stable atomic-geography keys from explicit source identities and selection/expansion records, preserving exact IDs and refusing ambiguous ancestry without granting source authority or changing the full-build host. diff --git a/changelog.d/us-adapter-spm-context.fixed.md b/changelog.d/us-adapter-spm-context.fixed.md new file mode 100644 index 000000000..792a851b1 --- /dev/null +++ b/changelog.d/us-adapter-spm-context.fixed.md @@ -0,0 +1 @@ +Forward explicit SPM settings to the US adapter's metadata system as well as its simulations, preserving the selected measurement context and the behavior when settings are omitted. diff --git a/changelog.d/us-asec-retirement-annuity-niu.fixed.md b/changelog.d/us-asec-retirement-annuity-niu.fixed.md new file mode 100644 index 000000000..0ddba37c7 --- /dev/null +++ b/changelog.d/us-asec-retirement-annuity-niu.fixed.md @@ -0,0 +1 @@ +Match published annuity NIU literals to the authenticated money owner's normalized zero and NIU status while preserving raw retirement source values. Reject mismatched dollar-zero and NIU encodings before qualification. diff --git a/changelog.d/us-atomic-financial-composition.added.md b/changelog.d/us-atomic-financial-composition.added.md new file mode 100644 index 000000000..a28385f6c --- /dev/null +++ b/changelog.d/us-atomic-financial-composition.added.md @@ -0,0 +1 @@ +Compose survey atomic geography and current financial imputation in a checked twenty-node graph, retaining geography through cloning and financial attachment. Add native Delaware population-only source normalization and complete support readback controls. diff --git a/changelog.d/us-canonical-household-roles-review.fixed.md b/changelog.d/us-canonical-household-roles-review.fixed.md new file mode 100644 index 000000000..6405cf35f --- /dev/null +++ b/changelog.d/us-canonical-household-roles-review.fixed.md @@ -0,0 +1 @@ +Correct the household-role graph test's receipt key, add Boolean-storage and actual source-coordinate/roster refusal controls, and document operative source/artifact capacity limits. diff --git a/changelog.d/us-common-survey-housing.added.md b/changelog.d/us-common-survey-housing.added.md new file mode 100644 index 000000000..1b73ad67b --- /dev/null +++ b/changelog.d/us-common-survey-housing.added.md @@ -0,0 +1,5 @@ +Connect independent housing-survey observations to participation inputs in the +common ACS–ASEC enrichment graph. Preserve source knownness and modeled origin, +fit unresolved receipt at household grain with original design weights, and +route assistance to the reference person's SPM unit. Leave program awards and +SPM valuation with the country model. diff --git a/changelog.d/us-completion-a50-us-schema-fixture.fixed.md b/changelog.d/us-completion-a50-us-schema-fixture.fixed.md new file mode 100644 index 000000000..1a9272a90 --- /dev/null +++ b/changelog.d/us-completion-a50-us-schema-fixture.fixed.md @@ -0,0 +1 @@ +Construct the completion custody tests' ordinary parent with the shared full US schema, invented group sentinel columns and an explicit empty metadata context. Check emptiness through the mapping interface. Preserve the original generic fixture and private child-support frames, and verify that ordinary US routes reject undeclared private-support metadata. diff --git a/changelog.d/us-completion-support-custody.fixed.md b/changelog.d/us-completion-support-custody.fixed.md new file mode 100644 index 000000000..3921953ce --- /dev/null +++ b/changelog.d/us-completion-support-custody.fixed.md @@ -0,0 +1 @@ +Keep child donor/recipient support observations and attached manifest views under their declared private schema during combined survey completion. Preserve full Population custody, permanent refusal, and the strict US receiving/source boundary. diff --git a/changelog.d/us-current-acs-income-anchor-source.added.md b/changelog.d/us-current-acs-income-anchor-source.added.md new file mode 100644 index 000000000..d8d6fce32 --- /dev/null +++ b/changelog.d/us-current-acs-income-anchor-source.added.md @@ -0,0 +1,3 @@ +Qualify original ACS property and retirement income anchors through the retained +survey source owner, preserving literal missingness, allocation, age universe and +signed values while verifying raw identities and adjusted amount bits. diff --git a/changelog.d/us-current-asec-dividend-source.added.md b/changelog.d/us-current-asec-dividend-source.added.md new file mode 100644 index 000000000..a7d453233 --- /dev/null +++ b/changelog.d/us-current-asec-dividend-source.added.md @@ -0,0 +1 @@ +Qualify current ASEC dividend observations and survivor source routes from the retained survey owner, preserving ambiguous zeros, allocation-codebook conflicts and source flags without assigning tax treatment or selecting donors. diff --git a/changelog.d/us-current-asec-income-routing-source.added.md b/changelog.d/us-current-asec-income-routing-source.added.md new file mode 100644 index 000000000..b4d469ed8 --- /dev/null +++ b/changelog.d/us-current-asec-income-routing-source.added.md @@ -0,0 +1 @@ +Qualify the current ASEC pension/annuity, retirement distribution, net property, farm and other-income source families into a typed reporting/routing projection, keeping printed totals, receipt universes, account and category routing, and allocation provenance distinct from any modelled taxable, private or residual component. diff --git a/changelog.d/us-current-asec-property-basis.added.md b/changelog.d/us-current-asec-property-basis.added.md new file mode 100644 index 000000000..908dff17d --- /dev/null +++ b/changelog.d/us-current-asec-property-basis.added.md @@ -0,0 +1 @@ +Add a pure ASEC property donor basis with separate reported and component totals, explicit retirement-slot routing derivations, exact identity alignment and original-design-weight exclusion diagnostics. diff --git a/changelog.d/us-current-asec-retirement-basis.added.md b/changelog.d/us-current-asec-retirement-basis.added.md new file mode 100644 index 000000000..d7da7254e --- /dev/null +++ b/changelog.d/us-current-asec-retirement-basis.added.md @@ -0,0 +1,3 @@ +Add a pure ASEC retirement candidate ledger with explicit measurement assumptions, source routing and knownness, conservative accounting intervals, preserved unresolved survivor/distribution scope, and original DESIGN-weight support diagnostics. It produces no fiscal inputs or source authority. + +Require exact applicable distribution aggregate balance and readable off-route evidence before admitting candidate intervals; preserve account evidence and signed residuals separately from interval availability. diff --git a/changelog.d/us-current-asec-retirement-detail-source.added.md b/changelog.d/us-current-asec-retirement-detail-source.added.md new file mode 100644 index 000000000..799cbc4b2 --- /dev/null +++ b/changelog.d/us-current-asec-retirement-detail-source.added.md @@ -0,0 +1 @@ +Qualify current ASEC pension, disability and survivor source details with exact original-person joins, preserved unknownness and unallocated aggregate comparisons. New literal fields remain separate from retained money; no retirement regularity, taxability or ACS component mapping is inferred. diff --git a/changelog.d/us-current-property-income-sources.added.md b/changelog.d/us-current-property-income-sources.added.md new file mode 100644 index 000000000..ca64a9350 --- /dev/null +++ b/changelog.d/us-current-property-income-sources.added.md @@ -0,0 +1 @@ +Compose source-qualified original ASEC property donor and ACS adult-anchor recipient branches, preserving DESIGN weights, full origin axes, knownness diagnostics and final source-lifetime checks without fitting or attaching tax leaves. diff --git a/changelog.d/us-current-survey-health-coverage.added.md b/changelog.d/us-current-survey-health-coverage.added.md new file mode 100644 index 000000000..5443947b8 --- /dev/null +++ b/changelog.d/us-current-survey-health-coverage.added.md @@ -0,0 +1,4 @@ +Add a source-qualified health coverage graph fragment for the common US survey +frame and its PUF clones. Preserve literal source and allocation codes, explicit +unknowns, and incoming amount fields; keep broader ACS concepts as documented +gaps instead of manufacturing narrower coverage flags. diff --git a/changelog.d/us-current-survey-household-roles.added.md b/changelog.d/us-current-survey-household-roles.added.md new file mode 100644 index 000000000..0951f4950 --- /dev/null +++ b/changelog.d/us-current-survey-household-roles.added.md @@ -0,0 +1 @@ +Add a source-qualified household reference-person fragment for the combined ACS/ASEC graph, preserving group-quarters unknownness and refusing unsupported or conflicting canonical incumbents. Include exact-clone binding, private provenance, full-population replay verification and bounded source/graph test proposals; country-host and native execution remain separate. diff --git a/changelog.d/us-declared-fiscal-measurement.added.md b/changelog.d/us-declared-fiscal-measurement.added.md new file mode 100644 index 000000000..58d36e459 --- /dev/null +++ b/changelog.d/us-declared-fiscal-measurement.added.md @@ -0,0 +1 @@ +Add a candidate-only US graph measurement stage using declared target bindings, household-aligned sparse measures, explicit national/state/district support checks, and complete input-closure checks for optional model evaluation. diff --git a/changelog.d/us-fiscal-consumer-propagation.added.md b/changelog.d/us-fiscal-consumer-propagation.added.md new file mode 100644 index 000000000..0ee46bacb --- /dev/null +++ b/changelog.d/us-fiscal-consumer-propagation.added.md @@ -0,0 +1 @@ +Allow fiscal materialization, batched reform and source-amount helpers, and written-file diagnostics to receive explicit dataset, simulation and system constructors with copied SPM settings. Existing default constructors and county reform-validation settings remain unchanged; injected dependencies do not establish runtime approval or cache identity. diff --git a/changelog.d/us-fiscal-dense-calibration.added.md b/changelog.d/us-fiscal-dense-calibration.added.md new file mode 100644 index 000000000..497b5415f --- /dev/null +++ b/changelog.d/us-fiscal-dense-calibration.added.md @@ -0,0 +1 @@ +Add a numerical fiscal graph calibration stage using existing grouped Adam, explicit original weight and origin bounds, exact household alignment, and target and origin diagnostics without claiming source or release admission. diff --git a/changelog.d/us-fiscal-formula-metadata.added.md b/changelog.d/us-fiscal-formula-metadata.added.md new file mode 100644 index 000000000..94b1d5339 --- /dev/null +++ b/changelog.d/us-fiscal-formula-metadata.added.md @@ -0,0 +1 @@ +Allow fiscal materialization, reform and weight-attachment helpers to use explicitly supplied period-specific formula metadata while preserving the cached static metadata default. Explicit metadata is checked before target checkpoint reuse and does not establish consumer or release qualification. diff --git a/changelog.d/us-fiscal-leaf-policy.added.md b/changelog.d/us-fiscal-leaf-policy.added.md new file mode 100644 index 000000000..dee0da8d5 --- /dev/null +++ b/changelog.d/us-fiscal-leaf-policy.added.md @@ -0,0 +1 @@ +Add optional pinned fiscal input policies with strict producer defaults, complete-population collision checks and private-copy preparation. Literal assumptions remain non-executable pending checked parent integration; sampled traces cannot exempt inactive inputs. diff --git a/changelog.d/us-fiscal-target-snapshots.added.md b/changelog.d/us-fiscal-target-snapshots.added.md new file mode 100644 index 000000000..61514455b --- /dev/null +++ b/changelog.d/us-fiscal-target-snapshots.added.md @@ -0,0 +1 @@ +Allow host-owned target snapshots on the US dense fiscal calibration kernel while preserving result and cache identity. Align grouped snapshot epochs with completed optimizer updates and keep required cache replay silent. diff --git a/changelog.d/us-head-to-head-explicit-consumer.added.md b/changelog.d/us-head-to-head-explicit-consumer.added.md new file mode 100644 index 000000000..25b29849c --- /dev/null +++ b/changelog.d/us-head-to-head-explicit-consumer.added.md @@ -0,0 +1 @@ +Allow the US head-to-head scorer to use one explicit model consumer for historical H5 variable ownership, formula normalization, batched target calculations and the ephemeral by-origin battery. The existing default path remains available; explicit dependencies do not attest runtime qualification or authorize publication. diff --git a/changelog.d/us-head-to-head-shipped-weights.added.md b/changelog.d/us-head-to-head-shipped-weights.added.md new file mode 100644 index 000000000..c8df19e81 --- /dev/null +++ b/changelog.d/us-head-to-head-shipped-weights.added.md @@ -0,0 +1 @@ +Allow US head-to-head scorecards to compare each file's shipped weights unchanged. The explicit population-weight mode applies to both artifacts and appears in JSON and Markdown; the historical rescaling default and population-scale diagnostics remain available. diff --git a/changelog.d/us-household-role-import-contract.fixed.md b/changelog.d/us-household-role-import-contract.fixed.md new file mode 100644 index 000000000..c76791cb1 --- /dev/null +++ b/changelog.d/us-household-role-import-contract.fixed.md @@ -0,0 +1 @@ +Move the unchanged shared demographic requirements into an import-free contract, preserving historical composed-stage re-exports and source identities while preventing household-role qualification from eagerly loading the legacy composed runtime. Bind both shared declarations at existing source and graph live checks. diff --git a/changelog.d/us-launch-integration.fixed.md b/changelog.d/us-launch-integration.fixed.md new file mode 100644 index 000000000..170634a55 --- /dev/null +++ b/changelog.d/us-launch-integration.fixed.md @@ -0,0 +1 @@ +Preserve grouped calibration closing-state results alongside ordinary best-iterate selection, retain the combined US runtime facade and joint geography validation, and publish complete metadata for lazy graph snapshots. diff --git a/changelog.d/us-launch-store-grouped-validation.fixed.md b/changelog.d/us-launch-store-grouped-validation.fixed.md new file mode 100644 index 000000000..c341c76e2 --- /dev/null +++ b/changelog.d/us-launch-store-grouped-validation.fixed.md @@ -0,0 +1 @@ +Reject nonfinite stored JSON as a store-corruption error and normalize the deprecated calibration method alias before validating grouped constraints. diff --git a/changelog.d/us-national-atomic-support.added.md b/changelog.d/us-national-atomic-support.added.md new file mode 100644 index 000000000..085c1627a --- /dev/null +++ b/changelog.d/us-national-atomic-support.added.md @@ -0,0 +1,4 @@ +Add an opt-in national Census population-only source control covering the fifty +states and DC, with independent block/state reconciliation, complete CD119 and +PUMA joins, and full support serialization/readback. Record the accepted native +normalization separately from household assignment and dataset release checks. diff --git a/changelog.d/us-native-child-support.added.md b/changelog.d/us-native-child-support.added.md new file mode 100644 index 000000000..d6acf3261 --- /dev/null +++ b/changelog.d/us-native-child-support.added.md @@ -0,0 +1 @@ +Add an opt-in native child-support amount group using full-original qualified ASEC receipt donors and one zero-aware ACS-original draw shared by clones. Keep paid support observed-only, retain unresolved source statuses, and explicitly replace only carried canonical child-support columns with lossless supported storage. diff --git a/changelog.d/us-native-other-disability-source.added.md b/changelog.d/us-native-other-disability-source.added.md new file mode 100644 index 000000000..fe74af615 --- /dev/null +++ b/changelog.d/us-native-other-disability-source.added.md @@ -0,0 +1 @@ +Add a qualified ASEC other-disability source adapter that reuses the retired two-slot, non-workers-compensation arithmetic over the existing retirement-detail owner, keeps NIU, missing, under-15 and contradictory slots unknown instead of zero, retains conditional allocation provenance, and copies each original row to its clones without a redraw. diff --git a/changelog.d/us-other-disability-host.added.md b/changelog.d/us-other-disability-host.added.md new file mode 100644 index 000000000..50d39f22d --- /dev/null +++ b/changelog.d/us-other-disability-host.added.md @@ -0,0 +1,4 @@ +Add an opt-in US survey enrichment continuation for other-disability dollars, +with full original ASEC donors, explicit source/model knownness, inherited-cache +checks and optional downstream original-arm PUF placement. Release qualification +remains separate. diff --git a/changelog.d/us-person-status-graph.added.md b/changelog.d/us-person-status-graph.added.md new file mode 100644 index 000000000..615666a24 --- /dev/null +++ b/changelog.d/us-person-status-graph.added.md @@ -0,0 +1 @@ +Add an optional four-node graph fragment exposing qualified descriptive ACS/ASEC person status after full cloning and atomic geography, with exact clone binding and complete receiving-population verification. diff --git a/changelog.d/us-policyengine-h5-logical-readback.added.md b/changelog.d/us-policyengine-h5-logical-readback.added.md new file mode 100644 index 000000000..41c8200ad --- /dev/null +++ b/changelog.d/us-policyengine-h5-logical-readback.added.md @@ -0,0 +1,6 @@ +Add an opt-in PolicyEngine-US H5 export wrapper that compares persisted values, +integer IDs and memberships, row order, household weights and period with a +supplied retained Frame and common-parent contract. It preserves adapter defaults +and admits only complete SPM roles, explicit scope and complete-input codec +behavior; the receipt does not establish +source ancestry, calibration validity or release eligibility. diff --git a/changelog.d/us-policyengine-h5-string-storage.fixed.md b/changelog.d/us-policyengine-h5-string-storage.fixed.md new file mode 100644 index 000000000..47390b30e --- /dev/null +++ b/changelog.d/us-policyengine-h5-string-storage.fixed.md @@ -0,0 +1,3 @@ +Accept logically identical complete Arrow-backed strings during PolicyEngine H5 +readback and report storage normalization without changing source admission or +the shared HDF codec. diff --git a/changelog.d/us-population-seal-streaming.changed.md b/changelog.d/us-population-seal-streaming.changed.md new file mode 100644 index 000000000..60002223e --- /dev/null +++ b/changelog.d/us-population-seal-streaming.changed.md @@ -0,0 +1 @@ +Stream the US full-PUF placement's in-process population seal into its digest instead of expanding every part into one tuple first, preserving each part, its order, its length prefix and its bytes exactly. diff --git a/changelog.d/us-property-income-graph.added.md b/changelog.d/us-property-income-graph.added.md new file mode 100644 index 000000000..ac92df734 --- /dev/null +++ b/changelog.d/us-property-income-graph.added.md @@ -0,0 +1 @@ +Add a source-blind property-income model graph with original-design QRF fitting, joint draw columns, signed reconciliation and explicit diagnostics. diff --git a/changelog.d/us-property-tax-leaves.added.md b/changelog.d/us-property-tax-leaves.added.md new file mode 100644 index 000000000..fe4905618 --- /dev/null +++ b/changelog.d/us-property-tax-leaves.added.md @@ -0,0 +1,6 @@ +Add a deterministic three-node property tax rebase with explicit receiving +version, separate interest/dividend knownness, maintained fraction splits, +subtraction complements and typed numerical completeness evidence. Preserve +unknowns and retirement-account earnings; refuse receiving shapes that the +current FILTER operation cannot preserve. Country and PUF host integration +remains separate. diff --git a/changelog.d/us-puf55-canonical-and-output-seals.added.md b/changelog.d/us-puf55-canonical-and-output-seals.added.md new file mode 100644 index 000000000..0e6679803 --- /dev/null +++ b/changelog.d/us-puf55-canonical-and-output-seals.added.md @@ -0,0 +1,4 @@ +Add a source-bound PUF55 canonical donor graph node with cold/replay controls, +stable live-code identity checks, and finalized-candidate seals for the shared +two-route numerical finalizer. Complete native survey attachment and release +validation remain separate build steps. diff --git a/changelog.d/us-puf55-qualified-fixed-inputs.added.md b/changelog.d/us-puf55-qualified-fixed-inputs.added.md new file mode 100644 index 000000000..f152c9f43 --- /dev/null +++ b/changelog.d/us-puf55-qualified-fixed-inputs.added.md @@ -0,0 +1 @@ +Add explicit original-arm PUF55 recipient projections and opt-in, source-qualified fixed-input graph artifacts. Preserve current clone-one defaults and expose provisional ASEC mapping assumptions without claiming observed taxability or release eligibility. diff --git a/changelog.d/us-puf55-survey-ss-measurement.added.md b/changelog.d/us-puf55-survey-ss-measurement.added.md new file mode 100644 index 000000000..ebe159455 --- /dev/null +++ b/changelog.d/us-puf55-survey-ss-measurement.added.md @@ -0,0 +1,5 @@ +Add an authenticated filer/joint-spouse Social Security report-sum measurement +for PUF55 conditioning, with an explicit eight-predictor fallback for incomplete +reports and preservation of all survey Social Security cells and masks. Record +31 passing invented controls and the remaining recipient integration and +measurement limitations. diff --git a/changelog.d/us-reviewed-dependency-classification.fixed.md b/changelog.d/us-reviewed-dependency-classification.fixed.md new file mode 100644 index 000000000..1f7fe4d55 --- /dev/null +++ b/changelog.d/us-reviewed-dependency-classification.fixed.md @@ -0,0 +1 @@ +Adopt the reviewed exact-buffer store read and optional consumer-metadata adapter dependency contracts so US implementation manifests validate their current source. Retain scoped dependency tripwires and invalidate affected implementation identities rather than reusing earlier cache keys. diff --git a/changelog.d/us-source-string-seal-efficiency.changed.md b/changelog.d/us-source-string-seal-efficiency.changed.md new file mode 100644 index 000000000..a0c33268e --- /dev/null +++ b/changelog.d/us-source-string-seal-efficiency.changed.md @@ -0,0 +1 @@ +Reuse bounded, column-local encodings for repeated source strings during ASEC frame hashing while preserving the exact digest byte stream and checking all values on every validation. diff --git a/changelog.d/us-structural-id-precision.fixed.md b/changelog.d/us-structural-id-precision.fixed.md new file mode 100644 index 000000000..7422f618c --- /dev/null +++ b/changelog.d/us-structural-id-precision.fixed.md @@ -0,0 +1,3 @@ +Require exact integer identity for entity IDs and memberships even when the +selected PolicyEngine variable stores floats. Metadata validation rejects lossy +float32/float64 conversions without changing source IDs or ordinary amount rounding. diff --git a/changelog.d/us-survey-catalogue-digest.changed.md b/changelog.d/us-survey-catalogue-digest.changed.md new file mode 100644 index 000000000..96d1c8d13 --- /dev/null +++ b/changelog.d/us-survey-catalogue-digest.changed.md @@ -0,0 +1 @@ +Stream eligible ACS catalogue seals through bounded record JSON encoding while preserving canonical digest bytes, full-value fallback, and repeated source and owner validation. Bind the additional JSON provider calls into the existing preparation authority checks. diff --git a/changelog.d/us-survey-completion-host.added.md b/changelog.d/us-survey-completion-host.added.md new file mode 100644 index 000000000..ef1e61661 --- /dev/null +++ b/changelog.d/us-survey-completion-host.added.md @@ -0,0 +1,4 @@ +Add an explicit optional US child-property completion host with qualified +household roles, retained issued-parent custody, a separate completion version, +and child-verification-bound tax rebasing. Source implementation and acceptance +tests are authored; combined-host runtime acceptance remains unrun. diff --git a/changelog.d/us-survey-completion-seams.added.md b/changelog.d/us-survey-completion-seams.added.md new file mode 100644 index 000000000..9ea3f64a7 --- /dev/null +++ b/changelog.d/us-survey-completion-seams.added.md @@ -0,0 +1 @@ +Add a dedicated all-row survey completion receiving operation and an optional typed child-verification dependency for property tax rebasing. Default declarations and split calculations remain unchanged; source and complete-parent authority stay with the country host. diff --git a/changelog.d/us-survey-copy-and-identity.fixed.md b/changelog.d/us-survey-copy-and-identity.fixed.md new file mode 100644 index 000000000..76d9520fa --- /dev/null +++ b/changelog.d/us-survey-copy-and-identity.fixed.md @@ -0,0 +1 @@ +Keep frame metadata copying and pickling from changing producer identity on Python 3.14. Refresh ACS source pins after verified formatting-only changes, and reduce per-cell survey identity encoding overhead while preserving canonical bytes, mutation checks, and the current-wage projection. diff --git a/changelog.d/us-survey-current-amount-enrichment.added.md b/changelog.d/us-survey-current-amount-enrichment.added.md new file mode 100644 index 000000000..dab37b769 --- /dev/null +++ b/changelog.d/us-survey-current-amount-enrichment.added.md @@ -0,0 +1 @@ +Add a checked US survey enrichment host that preserves the PUF population while attaching source-qualified unemployment, medical costs and current health coverage. Keep reporting unknowns, original observations and source-to-clone provenance explicit. diff --git a/changelog.d/us-survey-diagnostic-options.fixed.md b/changelog.d/us-survey-diagnostic-options.fixed.md new file mode 100644 index 000000000..e8668b7e0 --- /dev/null +++ b/changelog.d/us-survey-diagnostic-options.fixed.md @@ -0,0 +1 @@ +Reconstruct the grouped calibration solver's closing-state selection options during survey diagnostic verification, preserving exact diagnostic comparison and late-mutation checks. diff --git a/changelog.d/us-survey-interest-conservation.fixed.md b/changelog.d/us-survey-interest-conservation.fixed.md new file mode 100644 index 000000000..b58d850b5 --- /dev/null +++ b/changelog.d/us-survey-interest-conservation.fixed.md @@ -0,0 +1 @@ +Retain the tax-exempt remainder when the survey financial graph splits observed ASEC or modeled ACS interest totals, preserving both parts on the original survey support before PUF enrichment. diff --git a/changelog.d/us-survey-person-status-source.added.md b/changelog.d/us-survey-person-status-source.added.md new file mode 100644 index 000000000..ffb592c55 --- /dev/null +++ b/changelog.d/us-survey-person-status-source.added.md @@ -0,0 +1 @@ +Add a retained-source qualifier for descriptive ACS/ASEC difficulty and enrollment observations, preserving literal codes, item universes, edit provenance and survey periods while leaving statutory and annual eligibility inputs unresolved. diff --git a/changelog.d/us-whole-series-selection-sites.changed.md b/changelog.d/us-whole-series-selection-sites.changed.md new file mode 100644 index 000000000..e14368e80 --- /dev/null +++ b/changelog.d/us-whole-series-selection-sites.changed.md @@ -0,0 +1 @@ +Take the whole series by slice at the remaining US in-process seal sites (atomic geography, input coverage, origin budget, current survey predictors and PUF55 recipients), preserving each seal's emitted bytes exactly. diff --git a/changelog.d/weights-copy-protocol.fixed.md b/changelog.d/weights-copy-protocol.fixed.md new file mode 100644 index 000000000..4a29fdf93 --- /dev/null +++ b/changelog.d/weights-copy-protocol.fixed.md @@ -0,0 +1 @@ +Reconstruct base `Weights` through validation during shallow copy, deep copy and pickle round trips, preserving exact values and read-only storage without changing class metadata. Preserve existing subclass copy protocols. diff --git a/changelog.d/worker-identity-current-lock.fixed.md b/changelog.d/worker-identity-current-lock.fixed.md new file mode 100644 index 000000000..66e497e13 --- /dev/null +++ b/changelog.d/worker-identity-current-lock.fixed.md @@ -0,0 +1 @@ +Update the primary QRF worker's approved lock identity for the reviewed source-I/O packaging metadata change while preserving strict rejection of other locks and the explicit legacy campaign boundary. diff --git a/docs/acs-spm-source-assembly.md b/docs/acs-spm-source-assembly.md new file mode 100644 index 000000000..d883d373f --- /dev/null +++ b/docs/acs-spm-source-assembly.md @@ -0,0 +1,179 @@ +# ACS SPM construction before source mapping + +`microcosm.build.acs_spm_source_assembly` supplies an explicit development +partition for an ACS source Frame whose SPM table contains only `spm_unit_id`. +It preserves tax units, all other memberships and tables, household DESIGN +weights, strata and the mass log. It does not regroup amounts or run a country +model. + +```python +from microcosm.build.acs_spm_partition import ACS_SPM_DEVELOPMENT_POLICY +from microcosm.build.acs_spm_source_assembly import AcsSpmSourceAssemblyOptions +from microcosm.build.us_runtime.acs_pums import build_acs_pums_unit_frame + +frame, metadata = build_acs_pums_unit_frame( + source, + spm_construction=AcsSpmSourceAssemblyOptions( + policy=ACS_SPM_DEVELOPMENT_POLICY, + minor_partner_role=True, + ), +) +``` + +The default `spm_construction=None` retains the existing constructor path. An +explicit request checks canonical assembler capability before opening source +archives and raises `UnsupportedAssembler` if unavailable or incompatible. The +locked calculator 1.0.0 lacks the required behavior; the passing synthetic test +environment uses exact calculator PR45 source +`bcf45768003bb79addfafb0e6d9c7d2d5e547d9d`, with `units.py` SHA256 +`ce0d328d856ca81862b4e80947b6f6269a89bbd842cbdbb1569411b319da5a33`. +This change does not adopt a dependency or change a lockfile. + +## Registry and evidence + +The pure `assemble_acs_spm_source` function returns a Frame, partition projection, +numeric identity registry, crosswalk, unit evidence and aggregate receipt. Its +required `id_ceiling` must be an exact nonnegative int64 integer, excluding +Booleans. The ACS constructor supplies the existing reviewed clone-safe ceiling. +Old IDs survive unchanged member sets; split components receive distinct IDs +above the selected old-ID maximum, ordered by native person keys. Overflow is an +error. + +Construct one registry from the whole selected roster. Pass the same registry +when applying the adapter to complete-household subsets. The registry binds +native `(SERIALNO, SPORDER)` keys, the supplied person and household identities, +raw relationship/age/marital/universe evidence, household counts, construction +policy and actual assembler hash. It rejects changed source records, incomplete +households, partition changes and identity collisions. Row order may differ; +identities do not become stable across different selection domains. These are +generated component identities, not published Census SPM identifiers. + +The adapter temporarily joins household TYPEHUGQ and normalizes SPORDER for the +partition helper. It does not change the original stored tokens. Uncertainty +in secondary relationships remains visible in every affected household's unit +evidence, including a unit containing an observed reference person. The returned +nullable roles retain their original rule/source labels. Relationship allocation +provenance remains unresolved. No engine `is_spm_independent_minor_role` or annual +`spm_unit_spm_universe_status` is inferred or written. + +The source constructor retains the aggregate receipt in its metadata and the +Frame's immutable metadata. The pure adapter returns the separate rowwise +projection for callers that need it; the constructor does not publish those rows +in its aggregate receipt. The receipt binds supplied tables, not authenticated +source files, and does not grant source issuance, consumer or release authority. + +## Fresh SPM inputs and captured native construction + +The hook runs after the one tax-unit construction and household source attachment, +before `map_acs_native_inputs`. Existing tenure mapping follows the new SPM +membership. Existing housing-participation routing awards only the actual +household head's SPM unit. Neither copies an old SPM monetary table. Any existing +SPM amount, flag or provenance column makes this adapter refuse; the dense-parent +regroup helper is a separate workflow. + +The captured housing and native coverage owners accept an explicit +`spm_construction` option. They invoke the evidence-returning constructor against +their private, authenticated archive captures. The public constructor retains its +two-value return contract. Native owners retain canonical bytes produced from +the actual membership, links, partition crosswalk, regrouping summary, registry, +native crosswalk and unit uncertainty tables, including declared column rosters +and dtypes. They bind those bytes to the captured archive identities, options, +assembler and implementation identities. A 256 MiB limit bounds the development +evidence payload after serialization; it is not an allocation preflight or a +full-source memory qualification. + +After capturing the actual evidence, the housing owner removes exactly the +`acs_spm_source_assembly` aggregate key from its mapped graph Frame and records +that transport in its receipt. All other metadata stays intact. The aggregate +receipt remains in the owned evidence, and direct constructor callers retain it +in Frame metadata. It becomes neither graph-context operator authority nor an +engine role. Native verification checks the retained bytes; the preparation's +nested seals also retain them. Missing or changed evidence refuses even though +the mapped Frame no longer carries that aggregate key. Defensive decoded views +are values, not new detached authority. + +A version 1 `selection-request.json` keeps its exact closed shape and imports no +optional assembler. Version 2 requires the additional closed field: + +```json +{ + "acs_spm_construction": { + "policy": "acs_spm_development_reconstruction_v1", + "minor_partner_role": true + } +} +``` + +Use the exported `ACS_SPM_DEVELOPMENT_POLICY` value for the policy field and +`microcosm.us.survey-population-request.v2` for the protocol, retaining the existing +declaration, fraction and seed fields. No mode is enabled by default. The request +bytes bind the construction option into source and graph identities. Existing +survey callers read that request; financial/completion and retention host APIs +are unchanged. Source preparation checks capability before opening population +archives. The native owner additionally requires the exact reviewed PR45 defining +file and verifies the loaded callable during native verification. No global +package dependency or lockfile changes. + +`AuthenticatedSurveyPopulationPreparation.acs_spm_construction_evidence` returns +a defensive view of the checked native records. Its IDs belong to the whole +selected native roster. The existing per-entity `receipt["origins"]` map relates +native person, household and SPM IDs to assembled IDs; cloning remains a later, +separate operation. No rowwise roles or universe status are inferred into model +columns. + +The isolated branch updates the accepted parser/housing bytes and reviewed +helper inventory. These revisions change source/implementation identities even +for default requests; old receipts are not claimed byte-identical. The earlier +first slice at `fec3f9033ab47b752b2d5f8890507bcf06ad3428` intentionally left native +admission blocked. Its separate review and test receipts remain historical +source-only evidence. Frozen native performance and retention checkouts are not +changed by this continuation. + +## Invented validation + +At the first-slice source revision `fec3f9033ab47b752b2d5f8890507bcf06ad3428`, +the focused five-file battery passed 199 tests on exact PR45. Absent-assembler +and installed-1.0.0 environments each passed 151 tests with 48 strict expected +failures restricted to `UnsupportedAssembler`; neither had skips or unexpected +failures. The new assembly file contributes 42 passing PR45 cases, or 23 passing +and 19 expected unsupported cases. The imported partition/probe tests and existing +ACS parser/housing-participation tests retain their assertions. + +Tests cover tax/non-SPM conservation, role sensitivity without membership drift, +GQ preservation, source refusals, ID overflow/collisions, native-key binding, +shuffled and complete-household chunk application, exact large integer IDs, +and real constructor/tenure/head-only participation functions. Guards recorded +zero country imports, network requests and population-file reads. All source +records are invented fixtures. Earlier unsuccessful attempts remain in the local +evidence; this battery does not replace historical PR962 helper-matrix or actual +512A reports. No actual population, 512B, full-source performance, calibration or +release qualification follows from these tests. + +## Native request and issuer validation + +The continuation passes 218 focused adapter/native tests on exact PR45, plus +95 existing ACS literal/native coverage regressions. The 19 new cases cover +actual invented archive issuance, exact options and capability refusal before +source access, the default path without importing the assembler, actual retained +rows and native-to-assembled identity maps, non-SPM/weight conservation, missing +or changed evidence, copied-token refusal, source mutation, candidate replay, +and cold/required graph replay. Changing the explicit option changes the graph +key and refuses reuse of the old required result. The existing `return_values` +path retains the checked preparation and its defensive evidence accessor. + +Absent-assembler and installed-1.0.0 environments each pass 163 tests with +55 strict expected failures restricted to `UnsupportedAssembler`. Of those, +the new native file contributes 12 passing refusal/default cases and seven +unsupported cases. No test is skipped; unexpected exceptions remain failures. +Final completed runs record zero country-import, network and unapproved-file +read guard events. All archives, HDF fixtures and graph populations in these +tests are invented. The implementation and caller tests do not initialize a +country model. + +Earlier failing fixture/assertion attempts, the graph metadata-boundary refusal +and a combined synthetic run stopped by its initial 120-second CPU limit remain +in the local evidence. The final focused runner uses a 600-second CPU ceiling +and segregated temporary fixture directories. These are development test +receipts, not source-scale memory or runtime measurements. No actual population +application, country512B, engine role delivery, annual-universe qualification, +dependency adoption or release claim follows from this continuation. diff --git a/docs/asec-usual-hours-observations.md b/docs/asec-usual-hours-observations.md new file mode 100644 index 000000000..fed00661d --- /dev/null +++ b/docs/asec-usual-hours-observations.md @@ -0,0 +1,65 @@ +# ASEC usual-hours observations + +`current_asec_usual_hours.recode_asec_usual_hours` interprets original March 2025 +work-history literals across published ages 0-85. It returns source observations +without source admission, imputation, weights, graph execution or Frame mutation. + +The [2025 Census person dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +defines `HRSWK` on printed page 6C-17 (PDF page 38) as usual weekly hours during +weeks worked. Its universe is positive `WKSWORK`. Zero is NIU, and 99 represents +99 hours or more. Printed page 6C-20 (PDF page 41) distinguishes initial `WORKYN`, +the temporary/part-time follow-up `WTEMP`, and final recode `WRK_CK`. The work +questions cover people aged 15 and over; the follow-up applies to initial no +answers. Both pages were visually checked. The downloaded PDF SHA-256 is +`5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. + +These definitions support three distinct outcomes: + +| Source evidence | Result | +| --- | --- | +| Positive hours and weeks, coherent final work history | Preserve source hours and raw top code | +| Age 15+, zero hours/weeks and coherent final nonwork | Zero-hours completion labeled as source nonwork | +| Under 15 with NIU history, or adult with entirely NIU history | Unresolved hours with distinct universe/unanswered labels | + +The recoder rejects contradictory histories and invalid codes. An initial no +answer alone does not establish final nonwork. A NIU follow-up alone is not an +explicit no. Shared history and allocation-code checks keep the separate age-15 +donor recoder aligned without disguising an older person as a teenager. + +Original allocation flags and supplement response status remain separate from +the result's provenance. The [Census variable metadata](https://api.census.gov/data/2025/cps/asec/mar/variables/FL_665.json) +includes complete supplement nonresponse as `FL_665=0`; accepting the code does +not mean a respondent supplied the value. A zero person weight does not erase a +source observation. Positive weight remains required by the separate empirical +donor qualification routine. + +The all-NIU adult outcome is a conservative missing-information classification, +not a claim that NIU is a complete valid adult answer. This behavior must be +audited against the full source before graph integration. No earnings value, +last-week hours or historical engine default fills a gap here. In particular, +under-15 zero imputation needs a separately explicit assumption and earnings +conflict checks. This routine is not the all-person release input producer. + +`propose_asec_usual_hours` adds a separate, explicit completion operation. It +requires the original wage, nonfarm self-employment and farm self-employment +literals. It preserves observed adult hours independently of earnings. For an +under-15 person with NIU work history, it produces zero only when the caller +selects `us_hours_under15_zero_completion_v1` and all three earnings fields are +known zero. Nonzero earnings, missing columns or unknown earnings refuse that +completion. An unanswered adult remains unresolved and cannot produce a complete +proposal. The child's result stays labeled `under15_explicit_modeled_zero`. + +An aggregate diagnostic of the complete current ASEC person member found +142,125 records: 72,460 with positive source hours, 41,986 with source nonwork, +and 27,679 under-15 records with NIU history. No work-history contradiction or +zero/missing-hours record with nonzero earnings appeared in those three fields. +The diagnostic source hash matches the maintained 2025 member pin: +`06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5`. +These are unweighted source counts, not population estimates. A source scan does +not issue source authority or qualify a release. + +The version-2 retained source owner binds this observation/completion code and +its protocols, preserves raw earnings alongside work history, and returns a +complete selected-original-person hours projection for both survey arms. +Transport to clones, graph integration and all-person engine input qualification +remain separate. Public release flags remain false. diff --git a/docs/calibration-target-snapshots.md b/docs/calibration-target-snapshots.md new file mode 100644 index 000000000..b48fd32a9 --- /dev/null +++ b/docs/calibration-target-snapshots.md @@ -0,0 +1,155 @@ +# Calibration target snapshots + +An opt-in observer exposes target estimates already computed during an Adam +solve. It also records the estimates on the weights actually returned. Use it +to inspect fit over time without adding another model evaluation or changing +the optimization steps. + +```python +from pathlib import Path + +from microcosm.calibrate import ( + TargetSnapshotCadence, + TargetSnapshotObserver, + TargetSnapshotWriter, + calibrate, +) + +observer = TargetSnapshotObserver( + sink=TargetSnapshotWriter(Path("runs/example/targets"), history_limit=256), + run_id="example", + cadence=TargetSnapshotCadence(every=25), +) +result = calibrate(frame, targets, method="adam", target_snapshots=observer) +``` + +The caller supplies its existing frame and targets. No observer is enabled by +default. The example cadence is a caller choice, not a measured country-build +default. A sink exception propagates, matching the existing progress callback. + +Each snapshot has an ordered target identity, target values, achieved estimates, +relative errors, sequence, phase and iterate labels. Duplicate target names +remain distinguishable by ordered position. Honest nonfinite intermediate +estimates are represented by nulls and counts. Intermediate float32 estimates +and final float64 estimates are labeled with their actual iterate semantics. +Budget probes and subsequent selection/refit phases share one emission sequence. +`epoch` counts completed optimizer updates: zero is the starting state. An +in-loop estimate precedes that iteration's update; its selected counterpart uses +the same convention. Cadence counts loss evaluations starting at one, so an +every-25 cadence can emit epoch 24. The selected budget-search snapshot retains +the winning probe's identity, even when another probe ran afterward. If computing +a relative error overflows, the error is null and `non_finite_rows` counts it; +finite operands remain visible. + +Metadata accepts bounded flat scalar mappings and checked string identifiers; +it does not accept nested payloads or record vectors. `best_retained` must be a +complete `{available, epoch, loss}` mapping in a serialized snapshot. The emitter +can fill that mapping when no best iterate exists; the public validator rejects +a serialized null in its place. Each delivered payload is detached from the +caller's metadata and subsequent snapshots. + +The local writer atomically replaces `latest.json`. History chunks become +visible only after their complete bytes are written and synced, and an existing +chunk cannot be overwritten. Retention and dropped-history counts are explicit. +Use a fresh run directory unless intentionally adopting existing history. +Atomic history publication requires a filesystem that supports hard links and +directory synchronization. Unsupported filesystem errors propagate to the caller; +the writer does not expose partially written history as a fallback. + +## The grouped US solver + +The grouped/fixed-zero Adam path (`grouped_upper_bounds`, optionally +`grouped_preserve_zeros`) is instrumented at the same two seams and with the +same guarantees: the in-loop snapshot reads the exact float32 estimate tensor +the epoch's loss was computed from, and the closing `selected` snapshot reads +the same float64 `problem.estimates` the final diagnostics use. Enabling the +observer on a grouped run adds no matrix evaluation, no model evaluation and no +RNG advance, and returns bit-identical weights and trajectory. + +Two grouped specifics a consumer must read correctly: + +- Grouped Adam is a **closing-state** algorithm. It never runs the retain-best + rule, so every grouped snapshot carries + `best_retained = {"available": false, "epoch": null, "loss": null}` and the + closing snapshot is stamped `epoch == epochs`. That is not the same as an + ordinary run whose retain-best rule was switched off, so grouped snapshots + also carry bounded `selection` labels — `rule: "closing_state"`, + `constraint_mode: "grouped_upper_bounds"` and `grouped_preserve_zeros` — + which mirror the run's `options["iterate_selection"]`. +- In-loop snapshots are **pre-update loss evaluations**, the same convention the + ungrouped Adam loop uses. They are deliberately not the post-update projected + accepted vector that the private `_post_projection_observer` proof seam + reports after the next completed update: those are different vectors, and + producing target totals for the accepted one would require an extra matrix evaluation. + That private seam carries record-length weights, household IDs, the group map + and the absolute-bound vector; none of it reaches a snapshot, and the public + codec rejects record-level key names outright. + +Grouped runs keep every existing guard: fixed zeros stay in full-population +coordinates, the accepted-weight byte equality and ordered-household-ID checks +still run *after* the closing snapshot sink, and the unsupported grouped modes +(scalar cap, conserved mass, prox, L0/exact-k, gate initialization) are still +refused before the optimizer is constructed. Snapshot support does not turn any +of them into a permitted numeric path. + +## US fiscal host + +The actual dense fiscal kernel accepts the shared observer through an optional, +keyword-only constructor argument: + +```python +from microcosm.build.us_runtime.graph_fiscal_dense_calibration import ( + FiscalDenseCalibrationKernel, +) +from microcosm.calibrate import TargetSnapshotCadence, TargetSnapshotObserver + +observer = TargetSnapshotObserver( + sink=host_owned_sink, + run_id="fiscal-run", + candidate_id="candidate-under-review", + cadence=TargetSnapshotCadence(every=25), +) +kernels.register(FiscalDenseCalibrationKernel(target_snapshots=observer)) +``` + +The host owns the sink and any local writer directory. The kernel retains the +observer only on its instance and forwards it to the existing `calibrate` call. +Omitting the argument or passing `None` disables observation. Observer values, +callbacks and paths are absent from node parameters, implementation identity, +cache keys, graph artifacts and receipts. Changing this source implementation +changes its fingerprint normally; changing observer configuration does not. + +The solver binds snapshot identity to the actual compiled target order and +values from the fiscal measurement. The existing final matrix/target, household +ID, weight and measurement checks run after the sink completes. Sink exceptions +propagate, as with the ordinary progress callback. Each delivered dictionary is +detached: mutating its aggregate metadata or target rows cannot change the next +payload or optimizer result. This does not make an arbitrary callback safe to +mutate unrelated live application state. + +A required graph cache hit does not execute the optimizer and emits no target +snapshots, even when a new observer is supplied. Hosts should retain the previous +run's diagnostics and history with their identities; a replay must not be shown +as a new optimizer run. Complete-population and source admission remain the +country host's responsibility. + +This implements the solver, local storage and dense fiscal-host portions of +[issue 908](https://github.com/PolicyEngine/microcosm/issues/908). Staging upload, +a dashboard consumer and native cadence/performance acceptance remain separate +work. See [the fiscal-host integration evidence](../experiments/fiscal-target-snapshot-host-20260912.md). + +The reviewed repair branch passed 307 calibration tests before the final strict +null correction. All 52 snapshot tests then passed after that one-line correction, +including passive solver parity tests. Independent review closed the metadata, +detachment, partial-publication and codec findings. See the +[invented benchmark evidence](../experiments/908-target-snapshot-bench-receipts.md) +for overhead measurements; those are not native US or UK benchmarks. + +Fable's later PR review identified a mixed epoch convention, a missing winning +budget-probe label and a relative-error overflow case. Four new counterexamples +failed first; all 56 snapshot tests and all 311 calibration tests then passed +after those corrections. The 26 affected build identity and graph parity checks +also passed after recalculating the calibration source pins; fit and simulation +pins were preserved. These changes affect diagnostics only. The public writer +still validates each incoming payload because callers can supply serialized +snapshots independently of an observer. diff --git a/docs/categorical-probability-artifacts.md b/docs/categorical-probability-artifacts.md new file mode 100644 index 000000000..7e660e480 --- /dev/null +++ b/docs/categorical-probability-artifacts.md @@ -0,0 +1,48 @@ +# Categorical probability artifacts + +`microcosm.fit.categorical.fit_categorical` fits a real scikit-learn +`HistGradientBoostingClassifier` with directly supplied typed DESIGN weights. +It does not resample donors. Every declared class must have strictly positive +finite weighted support; missing or extra labels refuse. At least two classes +are required because the supported estimator's binary probability interface +does not provide a one-column single-class result. There is no invented mass +or fallback. The caller owns donor selection, scientific encoding and source +qualification. + +Inputs use a unique ordered int64 entity index and finite float64 predictors. +The label Series must have the same index. `CategoricalConfig` exposes the +supported iteration, learning-rate, tree-size, regularization and bin controls. +All constructor parameters are serialized, including the seed, disabled early +stopping, absent class weighting and absent categorical encoding. Probabilities +are taken from the actual fitted estimator and reordered by its `classes_` into +the declared class order. Application neither fits recipients nor draws random +values. + +The immutable fitted artifact binds ordered feature/class schemas, exact donor +matrix, labels, DESIGN weights, source projection hash, configuration, seed, +implementation and runtime identity. It contains the actual serialized fitted +estimator. Its loader is explicitly for trusted local producers/content stores: +the versioned envelope and expected digest detect corruption, but cannot make +an untrusted pickle safe. No fitted mutable object is returned to callers. + +`graph_categorical.categorical_fit_node` consumes a donor Slice and typed source +projection. `categorical_probability_node` consumes its model/metadata pair, +the maintained typed recipient matrix, and a separate recipient source +projection. The nodes return artifacts only. Exact graph declarations, +producer-key pairing, platform, content hashes and training metadata are checked +before prediction. `read_probabilities` checks the caller's expected bindings, +recipient IDs/order, class order and probability values. Those descriptive +checks cannot grant survey-source authority. + +This adapter changes no QRF behavior and attaches no Social Security amounts. +A future SS host must use the authenticated full-original source basis, +explicitly approved predictors and imported component order, then apply the +reason mask and conservation operator separately. The interpretation of +conditional category probabilities as shares, source-window transport, +beneficiary ownership and real-data statistical acceptance remain separate +development/release decisions. + +The tiny invented tests exercise the real weighted estimator, typed graph +cold/required replay, class order, dependency invalidation and refusals. They +do not establish calibration, source admission, a full host replay or release +acceptance. diff --git a/docs/child-property-income-graph.md b/docs/child-property-income-graph.md new file mode 100644 index 000000000..4d08d945b --- /dev/null +++ b/docs/child-property-income-graph.md @@ -0,0 +1,98 @@ +# Child property-income graph candidate + +This source-only fragment connects the source-qualified ASEC age 15–17 donor +projection to the real paired empirical model and original-coordinate draws. +It depends on separately reviewed numerical, source-qualifier and generic graph +adapter revisions. Its tests are authored and unrun. It supplies no production +support default and is not connected to the default financial, tax or PUF host. + +Six declared nodes expose the complete model-support schema: + +1. `child_property.donors` creates a private donor support population. Original + source coordinates are separate from its minted index IDs. The explicit + original-household design-weight column enters the fit; unit index weights + exist only to satisfy the support Frame contract. +2. `child_property.recipients` creates a private population containing every + selected original recipient, its exact coordinates, source age, component + knownness, eligibility and reason. Keeping this population nonempty permits + zero eligible children without a sentinel or an empty weight vector. It + emits the separate recipient projection and complete scenario artifact. +3. `child_property.fit` is the reusable typed empirical-fit node. Full-source + donors are independent of target sampling and clone expansion. The donor + projection excludes the selected preparation digest; changing target + sampling can change custody keys but not the fitted paired donor law. +4. `child_property.draw` is the reusable keyed-draw node. Its declared Boolean + eligibility input selects the original recipients internally. The full + source coordinate plus `child-property-v1` and `pattern`/`donor` suffixes + determine uniforms. Clone IDs, row order and scenario hashes do not. +5. `child_property.attach` reconstructs the real model and draws from the + retained source projections, verifies exact typed artifacts, and fills only + jointly unknown eligible O/D cells on both initial clones. It writes + `child_property_completion_status` and `child_property_imputed`; a model zero + has status `imputed_zero`, never source nonreceipt. Existing source-knownness, + all nonrecipient amounts, other columns, groups and geography are preserved. +6. `child_property.verify` requalifies the source again and compares all table + values/dtypes/missingness, effective weights and strata. It emits typed + verification evidence that still requires the host's complete-population + check, including owners and the ledger. + +`ChildPropertyBoundary` requires the actual authenticated preparation, retained +receiving `Population`, explicit options, typed host edges and exact artifact +pins. Its `require_parent` callback must recheck the host's issued parent and +return that exact Population. The fragment checks the callback's code, defaults +and closure identity, retained source issuance, complete source projections, +configuration and parent physical seals before and after callbacks. The real +source qualifier executes on every fresh check; serialized projections alone +cannot create a boundary. A detected execution/materialization failure revokes +the boundary for future checks. + +The country host must call `verify_materialized` on both cold and required +replay. Required cache hits skip kernel bodies. This method requires the actual +receiving population, both support populations, all verification-node input +`ArtifactValue`s and its output verification `ArtifactValue`. It re-executes the +real empirical fit and keyed draw independently of those artifacts, compares +all three complete populations with maintained replay equality, repeats source +and parent verification, and checks final output seals. Caller-owned artifact +descriptors and the verification output are detached before the first owner +borrow, then compared after the first and final borrows; their complete fields +and exact support-mapping keys must remain unchanged. The reconstructed artifact +and verification contracts are checked again after the final callback. The returned document +is descriptive evidence; it is not an issued financial, tax-complete or release +handle. + +Root-owned host integration remains explicit: insert after the complete survey +clones, atomic geography and current-property attachment, before property-tax +rebasing/gates and PUF. The receiving version must have a structural base so +the two amount writes are declared rewrites. Supply the real preparation, +allocation, frame-context and ordering edges/pins; retain the actual parent +owner in the callback and in the common host's live/source/replay seals. Bind +the actual run manifest's producer keys, implementation identities and source +identities when resolving artifact descriptors. This fragment reconstructs +payloads and checks typed paired producers; it does not independently issue +the common host's parent or validate an arbitrary claimed run manifest. Bind +the checked completion evidence into the tax gate without changing its +all-person completeness requirement. No unresolved adult class is completed by +this child candidate. The disabled/default path must remain unchanged. + +Tests separate pure descriptive fixtures from actual owner execution. Pure +fixtures test the real numerical law and graph patch, exact large IDs, clone +joins, zero stress, zero eligible children, known-incumbent refusal, exact +artifact bindings and drift anywhere in the receiving state. Actual-owner tests +reuse tiny authenticated survey fixtures whose sole teenager donor is absent +from the selected target sample. They run an invented receiving prefix and the +actual six-node fragment, including cold/required replay and first/final-owner +mutation controls for receiving values, artifact replacement, support-key +addition/removal, and verification payload/producer metadata. Refusal permanently +revokes the boundary. They do not claim actual financial/tax/PUF host acceptance. + +The accepted receiving Frame profile has no explicit links; the maintained +whole-population comparison refuses nonempty links rather than dropping them. + +Source projections and draw/completion JSON are private and bounded at 64 MiB; +selected originals are capped at 600,000. The generic model also has its 64 MiB +limit. JSON size checks follow object construction. No measured bytes-per-row, +peak memory or national capacity is claimed. This first implementation repeats +source requalification and model reconstruction at validation boundaries; a +separately reviewed sizing ladder and possible compact artifact representation +are required before a large candidate. Source-only tests cannot establish that +runtime performance or scientific donor support is adequate for release. diff --git a/docs/ci-file-sharding.md b/docs/ci-file-sharding.md new file mode 100644 index 000000000..eaa213790 --- /dev/null +++ b/docs/ci-file-sharding.md @@ -0,0 +1,77 @@ +# CI file sharding + +The fast `rest` inventory runs across six independent GitHub Actions matrix jobs +per Python version, the installed-wheel inventory across eight, and the engine +`us-am` inventory across four. The initial +partition uses sorted file paths round-robin. It balances file counts, not +measured runtime; an expensive single module can still dominate a job. + +Files are the smallest scheduling unit. All tests in a module, its module-scoped +fixtures, cold/required pairs and mutation controls stay in the same pytest +invocation. `us-am:build` and `us-am:other-shards` remain separate processes +inside each of the four `us-am` jobs, preserving import isolation. The US `us-qs` and +UK process divisions also remain unchanged. + +The environment tiers keep their original scope: + +- Fast covers the existing inventory except the explicit `ENGINE_ONLY` list. +- Engine groups partition the full inventory, with the existing country change + filters and US/UK extras. Both Python versions stay unchanged. +- Wheels partition the complete inventory, including `ENGINE_ONLY` files. Their + engine markers/import guards still skip when appropriate. Each matrix job + builds every real wheel, installs into a clean constrained venv, checks import + and spec identity, installs the same source extras, performs the HDF smoke and + confirms engine absence before its portion of the full suite. + +No tests are renamed, newly marked, skipped or replaced by sharding. Only the +partition regression file is added. The independent seed-diagnostic job and +the required `ci-ok` aggregation are unchanged. More jobs repeat setup and +wheel-building work; shorter wall time is not guaranteed by this configuration. +Every sharded pytest invocation prints its 25 slowest tests (`--durations=25`) +so an imbalance shows in the job log. `rest` went from four to six jobs and +`wheels` from four to eight on 2026-09-14 after one `rest` job and both `wheels` +2/4 jobs hit the six-hour runner limit; the partition is by file count, so a +few heavy modules dominate a job and the durations report is what shows which. + +## Selection and verification + +`tools/ci_test_groups.py` owns the file partition. For example: + +```sh +python3 tools/ci_test_groups.py --list rest --shard 1/6 +python3 tools/ci_test_groups.py --list us-am:build --shard 2/4 +python3 tools/ci_test_groups.py --list wheels --shard 8/8 +``` + +The inventory includes only `packages//tests/test_*.py` at that exact +depth. Git's explicit `:(glob)` pathspec and a matching path-depth check keep +nested test files out of the flat inventory. A separate tracked-file scan makes +`--verify` reject nested tests instead of silently dropping their coverage. + +`INDEX/COUNT` is 1-based, with `1 <= INDEX <= COUNT <= 64`. Selection sorts +the files and takes `files[INDEX-1::COUNT]` after the group/process filter. +Duplicate inventory entries, invalid ranges and an inventory too small to fill +every requested shard fail. Omitting the option preserves the original full +group selection. `wheels` is deliberately separate from the fast classifier. + +The workflow uses a checked command substitution and rejects empty output +before turning it into a bash array. It does not hide selector failures inside +process substitution, which can otherwise produce an empty pytest invocation +and inadvertently run the entire suite. + +```sh +python3 tools/ci_test_groups.py --verify +python3 -I -B -S packages/microcosm-build/tests/test_ci_test_groups.py +``` + +These commands need only the standard library and Git; they do not import +Microcosm, collect data tests, build wheels or run a microsimulation. The first +proves the nonempty, disjoint unions across groups, processes and configured +shards and rejects nested tests. The second exercises invalid/empty/duplicate +selection and verifies the actual workflow matrix covers each tier exactly +once while retaining the Python versions, process isolation and wheel boundary. + +Adding a file can shift subsequent round-robin assignments. The coverage proof +must still pass on the current checkout. Future tuning should use separately +reviewed module-level timing evidence; file counts are not timing evidence and +must not justify weaker checks or longer timeouts. diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md new file mode 100644 index 000000000..a4351354a --- /dev/null +++ b/docs/current-survey-puf59-progress.md @@ -0,0 +1,72 @@ +# PUF59 and current survey predictor implementation status + +The PUF59 profile fits and transfers 56 person outcomes and three tax-unit outcomes. Six detailed mortgage fields remain owned by the separate SCF producer. The historical full65 default remains available. PUF59 conditions on the source-specific PUF2015 filing-status class and capped return-size proxy, plus six monetary predictors. Its self-employment predictor includes ordinary and SSTB Schedule C income. Return-level Boolean outputs use an explicit incidence capacity of one; that is not a physical person count. + +The local PUF59/full65 regression run passed 128 ordinary tests. Those tests execute real weighted QRF fits, sequential target conditioning, draws, finalization, whole-Population preservation and required cache replay. They include excluded mortgage incumbent preservation and source-specific predictor checks. The run took 93.9 seconds and peaked at 622.7 MB RSS. These are invented mechanism fixtures, not held-out fit-quality or launch acceptance. + +Current survey financial completion passed 26 ordinary tests, including 15 ASEC demographic projection controls. The financial graph uses original ASEC design weights to fit interest, dividends and capital-gains outcomes for native ACS people. It preserves ACS observed wage/self-employment amounts and source-owned age-15 universe zeros. A draw is shared with its clone by source identity. The graph retains typed preparation, allocation and context evidence while separating the donor branch from survey allocation, avoiding a structural dependency cycle. Cold execution and required replay passed with the actual fitted model artifacts. The combined run took 121.4 seconds and peaked at 524.0 MB RSS. + +The source projection records the distinct survey observation, income and price periods. ASEC sex and state come from the retained original person and household sources, with allocation flags and unknownness preserved. Attaching those demographic projections to the graph is a separate pending integration step. The current financial model conditions on age and two earnings fields; sex/state conditioning and held-out fit quality remain explicit modeling work. + +Social Security handling remains a launch blocker. The historical PUF59 donor transports its total through the retirement component with modeled zero carrier columns for the other components. Those zeros do not establish absent recipient benefits. Its reconciliation code can treat unknown recipient components as zero and use equal component shares. The explicit `PUF55_SURVEY_SS` profile now excludes these four outcomes from PUF enrichment and conditions the remaining 55 outcomes on an additional Social Security total predictor. It preserves existing survey component fields, including unknownness. This profile is not yet wired into the native build. + +The new profile requires a known, finite, nonnegative `tax_unit.puf_conditioning_social_security_total` supplied by an upstream source owner. It does not infer that total from unknown components or move it between entity grains. All eight historical PUF59 predictors retain their order, with the total appended as predictor nine. The profile has a distinct train/apply phase and retains return-level incidence-capacity validation. Historical full65/PUF59 defaults and their 65/59 outcome contracts remain unchanged. + +Eleven new controls pass with real 55-target QRF fits, finalization, raw-draw validation and required cache replay. They check exact Social Security incumbent bit patterns, including signed zero and NaN, and refuse changes to the conditioning total after fitting. The run took 28.5 seconds and peaked at 631.6 MB RSS; receipt SHA256 is `8d5beacf15d8eb7bf568705d4e12bd71d810f3e748571b669a6ec44921ebf0cf`. All 128 historical profile cases also pass against the identical production module. An earlier combined run retained a failure in the new test's comparison of Boolean storage types; the corrected eleven-case run preserves exact values and missingness while treating the requested PUF Boolean output's nullable storage explicitly. Native recipient total qualification, beneficiary completion and whole-population PUF55 attachment remain separate pending checks. The native donor projection now passes as described below. + +The additive current-source qualifier now verifies original ASEC total, reason and allocation literals against the live survey preparation and retained original member. It also binds the ACS 2024 issuance and SSP price adjustment. Both surveys preserve under-15 question-universe unknownness. ASEC may record combined family payments on one person's report; a reason code is not a separately observed component amount or an assignment to individual beneficiaries. See [Social Security source semantics](survey-social-security-source.md) for the source contract, judgment boundaries and unresolved model work. + +The qualifier and numerical basis pass 52 controls: 50 numerical/literal cases and two actual maintained source issuances on invented original inputs. They include reordered source identities, ambiguous reasons, allocation flags, exact requalification, unknownness and no input mutation. The corrected run took 67.6 seconds and peaked at 428.9 MB RSS; the receipt SHA256 is `2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. An earlier run's two source tests exposed an incorrect capture-budget argument; the corrected caller uses the source owner's existing decoded-body bound while retaining exact CSV byte and digest checks. No native Social Security completion or beneficiary allocation has been accepted. + +The combined current source also passes all **225 component integration controls**: 34 shared geography cases, 52 survey Social Security cases, 11 PUF55 cases and all 128 historical full65/PUF59 cases. The exact combined run completed in 205.6 seconds at 687.8 MB peak RSS with no skips or unexpected access refusals. Before/after/current source, model-source and code-resource checks all agree; the maintained checkout matches the tested source map. Receipt SHA256: `63ef0ad0602a66d40f0736490d4c1e29f57723ba938c58d317802b387f947a87`. This remains an invented-input component integration, not a native population certification. + +The consolidated source passes a 314-case selected integration and identity suite, including the 128 profile cases, 26 financial/demographic cases, two canonical/growth wrappers, five metadata regressions and 153 identity controls. A subsequent diagnostic-option reconstruction correction passes all 16 current seven-node age-development tests, including late-mutation refusals. All source, model-source and resource pins survived external postchecks for each run. See the [review guide](us-launch-review.md) for exact revisions, receipts and exclusions. + +The new canonical59-to-PUF55 donor adapter also passes the maintained invented-source construction and artifact wrapper on 64 and 2,048 returns. It verifies the supplied artifact identity and the exact Social Security carrier convention before projecting the grown total into predictor nine. All 55 retained outputs, source RECIDs and weights survive unchanged. Internally rehashed artifacts with altered carrier semantics refuse. The corrected wrapper passed in 9.4 seconds at 486.9 MB peak RSS; receipt SHA256: `db3e9d8aa91863973de5d3201efcbabe31cbc1067c8c3f7fd72be79d887ffc27`. The initial knownness-column mismatch is preserved as a failed control; the fix supplies the owner's exact nonstructural column roster. This adapter grants no source authority. A subsequent separately guarded native projection passes for all 207,692 returns, retaining every target, source RECID, weight and incidence capacity and appending the modeled Social Security total as predictor nine. It took 11.05 seconds at 1.43 GB peak RSS. The original canonical artifact remained unchanged; source/control/model/code checks agree, with no unexpected access refusals. Receipt SHA256: `ce09455947f0efbca16014a23baa19037ac6450876c639568af6584c81bfc45c`. The earlier attempt passed its value assertions but lacked collection accounting and was refused by the guard; that failure is preserved. This native projection ran in memory and produced metadata only. It did not fit a recipient model or attach results to the survey population. + +The genuine seven-node survey age-development run now passes at the unchanged 1/1000 sampling fraction. It prepares 1,584 households and 3,464 people (ACS: 1,529 households/3,324 people; ASEC: 55/140), then retains 3,168 household records and 6,928 person records after the support clone. Export/readback, final source-owner and target checks completed under the frozen `a9895d8a5` source; the final manifest was written only after those checks. All 473 source/control/runtime and 12 code-resource postchecks agree, with no unexpected access refusals. The run took 4,268.8 seconds and peaked at 8.26 GB RSS. Receipt SHA256: `2aa76b5daca7bb0df1ac85d4db23d6b5482073d31d437418d46442ef01b002ca`. This run does not include the subsequent Social Security, PUF55 or atomic-geography integration. + +This is accepted native age-development evidence, not a national or congressional-district release, native enrichment fit-quality result or country-engine result. Complete enrichment, geography, calibration and release-export acceptance remain pending. + +The PUF55 profile now also passes three invented whole-Population attachment +controls. They execute all 55 fitted outcomes through the actual graph, typed +artifact loader and materialized population verifier, followed by Frame storage +and required replay. Every non-owned field, Social Security bit pattern and +unknown cell, membership, design anchor and household weight remains intact. +Two negative cases reject changed Social Security values and unknown-to-zero +conversions even when predictor matrices remain identical. The run took 21.03 +seconds at 631.3 MB peak RSS; receipt SHA256: +`ad8519d867279ebce9ee78bf8f495f3799afb45ec1b3fcda1be649f8a129af30`. +All 482 source/control, 5,983 model-source and 12 code-resource postchecks agree, +with no skips or unexpected access refusals. A subsequent independent review +required a direct finalizer comparison, because the attachment and its verifier +share a column-construction helper. The strengthened three-case run also passes: +all 55 attached outputs agree with the direct finalizer after independent entity +ID alignment, on cold and required replay. It took 20.73 seconds at 630.5 MB; +receipt SHA256 is +`e3a4d548f0209cfacd7d9e3beddb63c1b46204a3168eb9869d2b2eafe9cf2291`. +All 486 source/control, 5,983 model-source and 12 code-resource postchecks agree. +Native recipient qualification, measurement comparability and actual survey +attachment remain pending. + +The observed-geography source qualifier passes 11 invented original-source +controls. It binds current ASEC state and ACS state/PUMA to literal source +household keys, preserves unknown locations, and keeps draw keys stable across +sampling fractions. The initial ten-case run correctly refused a smaller sample +with no positive ASEC support; its corrected fixture supplies positive support +before issuance. Review also added a receipt-digest check against mutation of +the detached ASEC projection. The passing run took 182.50 seconds at 430.3 MB; +receipt SHA256 is +`1900801b1bbcea89a3747f656410e500b1328519694c3351100562bba61cab08`. +The 484 source/control, 5,983 model-source and 12 code-resource postchecks agree, +with no skips or unexpected refusals. + +The observed-geography graph node separately passes 14 controls over the same +kind of invented original sources. Actual three-node execution and required +replay preserve inputs, weights and unknown geography. Declaration, source +receipt, typed-artifact and final-mutation cases refuse invalid output. The run +took 319.44 seconds at 439.1 MB; receipt SHA256 is +`28a6b5c58a258c7b3129a48054a3360647c788f8b76d781cc3bc8fc35a73f997`. +All 486 source/control, 5,983 model-source and 12 code-resource postchecks agree. +Full geography-before-clone composition, native block-source integration and +complete calibration remain separate work. diff --git a/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json b/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json new file mode 100644 index 000000000..79974b85d --- /dev/null +++ b/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json @@ -0,0 +1,447 @@ +{ + "schema": "microcosm.us.puf2015_target2024_growth/1", + "input_money_year": 2015, + "output_money_year": 2024, + "output_profile": "puf59_scf_mortgage_v1", + "ordered_targets": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "tax_exempt_interest_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "non_sch_d_capital_gains", + "taxable_private_pension_income", + "taxable_ira_distributions", + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", + "alimony_income", + "alimony_expense", + "salt_refund_income", + "charitable_cash_donations", + "charitable_non_cash_donations", + "real_estate_taxes", + "home_mortgage_interest", + "investment_interest_expense", + "investment_income_elected_form_4952", + "student_loan_interest", + "educator_expense", + "qualified_tuition_expenses", + "casualty_loss", + "unreimbursed_business_employee_expenses", + "traditional_ira_contributions_desired", + "self_employed_pension_contributions_desired", + "rental_income", + "estate_income", + "farm_income", + "farm_operations_income", + "farm_rent_income", + "miscellaneous_income", + "partnership_income", + "s_corp_income", + "partnership_self_employment_net_earnings", + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", + "sstb_self_employment_income_would_be_qualified", + "business_is_sstb", + "qualified_bdc_income", + "qualified_reit_and_ptp_income", + "sstb_self_employment_income_before_lsr", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "unadjusted_basis_qualified_property", + "w2_wages_from_qualified_business", + "domestic_production_ald", + "unrecaptured_section_1250_gain", + "health_savings_account_ald" + ], + "money_fields": { + "employment_income_before_lsr": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "w2_wages_from_qualified_business": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "sstb_w2_wages_from_qualified_business": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "self_employment_income_before_lsr": { + "positive_family": "business_profit", + "negative_family": "business_loss", + "positive_factor": "1.222517420922031096142334753675412822692", + "negative_factor": "1.718898116890934040328182671660914217539", + "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition." + }, + "sstb_self_employment_income_before_lsr": { + "positive_family": "business_profit", + "negative_family": "business_loss", + "positive_factor": "1.222517420922031096142334753675412822692", + "negative_factor": "1.718898116890934040328182671660914217539", + "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition." + }, + "taxable_interest_income": { + "positive_family": "taxable_interest", + "negative_family": "taxable_interest", + "positive_factor": "2.599750625636417399355437523686408804619", + "negative_factor": "2.599750625636417399355437523686408804619", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "tax_exempt_interest_income": { + "positive_family": "tax_exempt_interest", + "negative_family": "tax_exempt_interest", + "positive_factor": "0.9295395824924337172788943063597248917385", + "negative_factor": "0.9295395824924337172788943063597248917385", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "qualified_dividend_income": { + "positive_family": "qualified_dividends", + "negative_family": "qualified_dividends", + "positive_factor": "1.409891045424454325551259680558160347719", + "negative_factor": "1.409891045424454325551259680558160347719", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "non_qualified_dividend_income": { + "positive_family": "nonqualified_dividends", + "negative_family": "nonqualified_dividends", + "positive_factor": "2.483795459689869824933006427197633652403", + "negative_factor": "2.483795459689869824933006427197633652403", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "non_sch_d_capital_gains": { + "positive_family": "non_schedule_d_distributions", + "negative_family": "non_schedule_d_distributions", + "positive_factor": "1.120350061633613230084419890940102481805", + "negative_factor": "1.120350061633613230084419890940102481805", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "taxable_private_pension_income": { + "positive_family": "taxable_pensions", + "negative_family": "taxable_pensions", + "positive_factor": "1.327588034973940867230285994253866499704", + "negative_factor": "1.327588034973940867230285994253866499704", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "taxable_ira_distributions": { + "positive_family": "ira_distributions", + "negative_family": "ira_distributions", + "positive_factor": "1.510871073557979130794209158828995676924", + "negative_factor": "1.510871073557979130794209158828995676924", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "alimony_income": { + "positive_family": "alimony_income", + "negative_family": "alimony_income", + "positive_factor": "1.542033706590010885068081742929926580179", + "negative_factor": "1.542033706590010885068081742929926580179", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "salt_refund_income": { + "positive_family": "state_tax_refund", + "negative_family": "state_tax_refund", + "positive_factor": "0.9296526775720357466732616002244135476521", + "negative_factor": "0.9296526775720357466732616002244135476521", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "short_term_capital_gains": { + "positive_family": "short_current_gain", + "negative_family": "short_current_loss", + "positive_factor": "2.042865587775988756504710527637911842775", + "negative_factor": "1.738139193325432816803529458382722438642", + "assumption": "Current-year gross gains/loss components per all return proxy for signed net PUF amount; carried losses are excluded." + }, + "long_term_capital_gains_before_response": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "long_term_capital_gains_on_collectibles": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "unrecaptured_section_1250_gain": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "social_security_retirement": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_disability": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_dependents": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_survivors": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "rental_income": { + "positive_family": "rental_royalty_profit", + "negative_family": "rental_royalty_loss", + "positive_factor": "1.476663829685885394438774259073237903799", + "negative_factor": "1.690793641559507564235995133214906035185", + "assumption": "Related rental/royalty reporting-return mean proxy; published total family includes farm rent and does not exactly equal E25850/E25860 component definitions." + }, + "estate_income": { + "positive_family": "estate_profit", + "negative_family": "estate_loss", + "positive_factor": "1.476331638358104936021886589313386353228", + "negative_factor": "1.582810027948011789708036602150526302352", + "assumption": "" + }, + "farm_operations_income": { + "positive_family": "farm_profit", + "negative_family": "farm_loss", + "positive_factor": "1.444720208215047581256304812994467826979", + "negative_factor": "1.728383897337776476443532509645504817056", + "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income." + }, + "farm_income": { + "positive_family": "farm_profit", + "negative_family": "farm_loss", + "positive_factor": "1.444720208215047581256304812994467826979", + "negative_factor": "1.728383897337776476443532509645504817056", + "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income." + }, + "farm_rent_income": { + "positive_family": "farm_rent_profit", + "negative_family": "farm_rent_loss", + "positive_factor": "1.629452810459537374541656575493532799084", + "negative_factor": "1.150378978298943195689490419607276484593", + "assumption": "" + }, + "miscellaneous_income": { + "positive_family": "other_property_gain", + "negative_family": "other_property_loss", + "positive_factor": "1.738242164922618767195190248113271033066", + "negative_factor": "1.411645744283618865226902643798237455207", + "assumption": "Canonical source E01200 is other property gains/losses, not Table1.4 other income." + }, + "partnership_income": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "s_corp_income": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "partnership_self_employment_net_earnings": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "qualified_bdc_income": { + "positive_family": "nonqualified_dividends", + "negative_family": "nonqualified_dividends", + "positive_factor": "2.483795459689869824933006427197633652403", + "negative_factor": "2.483795459689869824933006427197633652403", + "assumption": "Modeled BDC component follows its parent nonqualified dividend pool; no independent source observation." + }, + "qualified_reit_and_ptp_income": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real proxy for the combined modeled REIT/PTP output, which mixes nonqualified-dividend and pass-through components. No claim that this is a pure dividend pool." + }, + "alimony_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "charitable_cash_donations": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "charitable_non_cash_donations": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "real_estate_taxes": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "home_mortgage_interest": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "investment_interest_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "investment_income_elected_form_4952": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "student_loan_interest": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "educator_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "qualified_tuition_expenses": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "casualty_loss": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "unreimbursed_business_employee_expenses": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "traditional_ira_contributions_desired": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "self_employed_pension_contributions_desired": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "domestic_production_ald": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "health_savings_account_ald": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "unadjusted_basis_qualified_property": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "sstb_unadjusted_basis_qualified_property": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + } + }, + "return_incidence_fields": [ + "business_is_sstb", + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", + "sstb_self_employment_income_would_be_qualified" + ], + "source_pins": { + "NATIONAL-GROWTH-EXTRACT.json": "775d89d3fe7a3b1eb9a09c50f6077d41bf838cdb80d9e4aaa9f0ab80ada4e0c8", + "INDEX-VALUES.json": "221f3786a462fe105d3da15df3624de5ad3e3f921b4d0205e096067685fd59b3", + "extract_national.py": "b6c3a8e4fea52653c01d801d6efe9620cdba4a489d6d7ae8c95cf00ee6dfbe1b", + "ACQUISITION.json": "e9a2c34c2ca06a229b2a790ed2987f25d31996653490f342b4b54af37d5f098c" + }, + "sensitivity_cpi_factor": "1.323487344789614247079323424058189918867", + "provenance": "All growth applications are modeled distribution transport from observed national series; source observed/derived/modeled flags remain separate.", + "operation_order": "Decode source statistical2015 \u2192 source canonical transformations \u2192 QBI2015 model (all16 outputs) \u2192 this growth transform once \u2192 2024 canonical donor. Monetary recipient predictors remain 2024.", + "noninterference": "RECID, S006, raw FLPDYR/FLPDMO, raw money, count source fields and actual survey membership are separate inputs and are not accepted or mutated by this transform.", + "sensitivities": "cpi_only uses exactly the same source cohort, zeros, signs and return incidence; no new policy evaluation or calibration authority." +} diff --git a/docs/evidence/puf2015-target2024/INDEX-VALUES.json b/docs/evidence/puf2015-target2024/INDEX-VALUES.json new file mode 100644 index 000000000..5dd19e557 --- /dev/null +++ b/docs/evidence/puf2015-target2024/INDEX-VALUES.json @@ -0,0 +1,8 @@ +{ + "schema": "microcosm.public_index_value_transcript/1", + "retrieved_date": "2026-09-09", + "retrieval": "Official URLs successfully read with web retrieval; exact values transcribed and independently checkable. These are not hashes of original publisher HTML/PDF bytes. Direct SSA urllib acquisition returned HTTP403 before writing any document.", + "awi": {"url": "https://www.ssa.gov/oact/cola/AWI.html", "series": "National Average Wage Index", "2015": "48098.63", "2024": "69846.57", "measurement": "Wages subject to federal income tax and deferred compensation; average-worker index, used as an explicit proxy for PUF wage amount growth."}, + "cola": {"url": "https://www.ssa.gov/oact/cola/colaseries.html", "cash_timing_url": "https://www.ssa.gov/cola/", "december_payable_percent": {"2015": "0.0", "2016": "0.3", "2017": "2.0", "2018": "2.8", "2019": "1.6", "2020": "1.3", "2021": "5.9", "2022": "8.7", "2023": "3.2"}, "timing": "December benefits are received in January of the following year. Product covers January2016 through January2024 cash increases relative to calendar2015 cash benefits. This transports a fixed entitlement; it does not forecast the recipient mix or new awards."}, + "cpi": {"url": "https://www.bls.gov/cpi/tables/supplemental-files/historical-cpi-u-202412.pdf", "series": "CPI-U, US city average, all items, annual average, 1982-84=100", "table": "Historical CPI-U index averages, annual average column", "2015": "237.017", "2023": "304.702", "2024": "313.689", "pages_one_based": [4,5], "monthly_2024_check": ["308.417","310.326","312.332","313.548","314.069","314.175","314.540","314.796","315.301","315.664","315.493","315.605"]} +} diff --git a/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json b/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json new file mode 100644 index 000000000..e24f8d37d --- /dev/null +++ b/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json @@ -0,0 +1,4439 @@ +{ + "schema": "microcosm.us.puf_observed_growth_sources/1", + "scope": "exact headers and national row only; no AGI rows used", + "tables": { + "15in14ar.xls": { + "url": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "sha256": "9f41bee263f5a8bb971df39a9673b55136c94954131a95bfd83b73a22801ec89", + "sheet": "TBL14", + "year": 2015, + "title": "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, \nby Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 9, + "all_returns": { + "cell": "B9", + "value": 150493263, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns" + ] + }, + "families": { + "taxable_interest": { + "count": { + "cell": "H9", + "value": 42636696, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I9", + "value": 95881223, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "tax_exempt_interest": { + "count": { + "cell": "J9", + "value": 5827038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K9", + "value": 61871455, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Amount", + "Amount" + ] + } + }, + "ordinary_dividends": { + "count": { + "cell": "L9", + "value": 27607044, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M9", + "value": 260252720, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "qualified_dividends": { + "count": { + "cell": "N9", + "value": 25755976, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "O9", + "value": 203187788, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Amount", + "Amount" + ] + } + }, + "state_tax_refund": { + "count": { + "cell": "P9", + "value": 20256512, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Q9", + "value": 31110732, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "alimony_income": { + "count": { + "cell": "R9", + "value": 414420, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "S9", + "value": 10077086, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "business_profit": { + "count": { + "cell": "T9", + "value": 18791200, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 391975736, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "business_loss": { + "count": { + "cell": "V9", + "value": 5935725, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 60161435, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "non_schedule_d_distributions": { + "count": { + "cell": "X9", + "value": 4323250, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 11563203, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "other_property_gain": { + "count": { + "cell": "AD9", + "value": 1063576, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 33036562, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "other_property_loss": { + "count": { + "cell": "AF9", + "value": 1153117, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 21093508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "ira_distributions": { + "count": { + "cell": "AH9", + "value": 14159018, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 253213041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "taxable_pensions": { + "count": { + "cell": "AL9", + "value": 28199160, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AM9", + "value": 689991999, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_profit": { + "count": { + "cell": "AZ9", + "value": 6768234, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA9", + "value": 103058883, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_loss": { + "count": { + "cell": "BB9", + "value": 4531666, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC9", + "value": 46245560, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_rent_profit": { + "count": { + "cell": "AV9", + "value": 394273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AW9", + "value": 5205330, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_rent_loss": { + "count": { + "cell": "AX9", + "value": 101041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 710953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "estate_profit": { + "count": { + "cell": "BH9", + "value": 630256, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 32453002, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "estate_loss": { + "count": { + "cell": "BJ9", + "value": 57802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 5033199, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_profit": { + "count": { + "cell": "BL9", + "value": 520802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 13533867, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_loss": { + "count": { + "cell": "BN9", + "value": 1278825, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 27497651, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "other_income_profit": { + "count": { + "cell": "BX9", + "value": 6121770, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 46769290, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "other_income_loss": { + "count": { + "cell": "BZ9", + "value": 332708, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 6693961, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "partnership_s_corp_profit": { + "count": { + "cell": "BD9", + "value": 6044409, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE9", + "value": 755622761, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "partnership_s_corp_loss": { + "count": { + "cell": "BF9", + "value": 2699817, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG9", + "value": 126618186, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + } + }, + "23in14ar.xls": { + "url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "sha256": "b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af", + "sheet": "TBL14", + "year": 2023, + "title": "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, \nby Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 9, + "all_returns": { + "cell": "B9", + "value": 160602107, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns" + ] + }, + "families": { + "taxable_interest": { + "count": { + "cell": "T9", + "value": 55260238, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 313812674, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "tax_exempt_interest": { + "count": { + "cell": "V9", + "value": 6893860, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 66091992, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Amount", + "Amount" + ] + } + }, + "ordinary_dividends": { + "count": { + "cell": "X9", + "value": 33718460, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 504224981, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "qualified_dividends": { + "count": { + "cell": "Z9", + "value": 31106344, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AA9", + "value": 336070296, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Amount", + "Amount" + ] + } + }, + "state_tax_refund": { + "count": { + "cell": "AB9", + "value": 3109096, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AC9", + "value": 4311977, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "alimony_income": { + "count": { + "cell": "AD9", + "value": 183582, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 6686429, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "business_profit": { + "count": { + "cell": "AF9", + "value": 21411713, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 530380019, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "business_loss": { + "count": { + "cell": "AH9", + "value": 9067174, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 153441387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "non_schedule_d_distributions": { + "count": { + "cell": "AJ9", + "value": 3209131, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AK9", + "value": 9340820, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "other_property_gain": { + "count": { + "cell": "AP9", + "value": 945122, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AQ9", + "value": 49567896, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "other_property_loss": { + "count": { + "cell": "AR9", + "value": 775387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AS9", + "value": 19448929, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "ira_distributions": { + "count": { + "cell": "AT9", + "value": 16694154, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AU9", + "value": 438147938, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "taxable_pensions": { + "count": { + "cell": "AX9", + "value": 29541284, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 932130236, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_profit": { + "count": { + "cell": "BL9", + "value": 6346357, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 138609251, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_loss": { + "count": { + "cell": "BN9", + "value": 3196282, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 53570273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_rent_profit": { + "count": { + "cell": "BH9", + "value": 325953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 6811206, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_rent_loss": { + "count": { + "cell": "BJ9", + "value": 67356, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 529586, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "estate_profit": { + "count": { + "cell": "BX9", + "value": 648583, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 47892046, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "estate_loss": { + "count": { + "cell": "BZ9", + "value": 36592, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 4898828, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_profit": { + "count": { + "cell": "CB9", + "value": 484162, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CC9", + "value": 17656301, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_loss": { + "count": { + "cell": "CD9", + "value": 1258869, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CE9", + "value": 45444491, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "other_income_profit": { + "count": { + "cell": "CN9", + "value": 7468434, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CO9", + "value": 60816508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "other_income_loss": { + "count": { + "cell": "CP9", + "value": 318839, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CQ9", + "value": 12213628, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "partnership_profit": { + "count": { + "cell": "BP9", + "value": 3266345, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ9", + "value": 452077038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "partnership_loss": { + "count": { + "cell": "BR9", + "value": 2103634, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BS9", + "value": 168884100, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "s_corp_profit": { + "count": { + "cell": "BT9", + "value": 4138040, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BU9", + "value": 797450861, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "s_corp_loss": { + "count": { + "cell": "BV9", + "value": 1415974, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BW9", + "value": 89543715, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + } + }, + "15in14acg.xls": { + "url": "https://www.irs.gov/pub/irs-soi/15in14acg.xls", + "sha256": "7e128eb06e14371a15baa72060e91d74e4c55ba8596610ddd047c9ef168cbbd1", + "sheet": "Sheet1", + "year": 2015, + "title": "Table 1.4A Returns with Income or Loss from Sales of Capital Assets\nReported on Form 1040, Schedule D: Selected Items, by Size of\nAdjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 10, + "families": { + "short_term_gain": { + "count": { + "cell": "F10", + "value": 3900677, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "G10", + "value": 38164189, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Amount", + "Amount" + ] + } + }, + "short_term_loss": { + "count": { + "cell": "H10", + "value": 7820586, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I10", + "value": 250592206, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Amount", + "Amount" + ] + } + }, + "long_term_gain": { + "count": { + "cell": "BJ10", + "value": 11735160, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK10", + "value": 733313255, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Amount", + "Amount" + ] + } + }, + "long_term_loss": { + "count": { + "cell": "BL10", + "value": 6375392, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM10", + "value": 312723716, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Amount", + "Amount" + ] + } + }, + "short_assets_gain": { + "count": { + "cell": "J10", + "value": 3923409, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K10", + "value": 22196394, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_assets_loss": { + "count": { + "cell": "L10", + "value": 6269981, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M10", + "value": 68624843, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_other_gain": { + "count": { + "cell": "AZ10", + "value": 231076, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA10", + "value": 5948464, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "short_other_loss": { + "count": { + "cell": "BB10", + "value": 336867, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC10", + "value": 1980907, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_gain": { + "count": { + "cell": "BD10", + "value": 483349, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE10", + "value": 17194108, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_loss": { + "count": { + "cell": "BF10", + "value": 725964, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG10", + "value": 16056081, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_carryover": { + "count": { + "cell": "BH10", + "value": 2000156, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI10", + "value": 171105171, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Amount", + "Amount" + ] + } + }, + "long_assets_gain": { + "count": { + "cell": "BN10", + "value": 8440065, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO10", + "value": 282493545, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_assets_loss": { + "count": { + "cell": "BP10", + "value": 4441176, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ10", + "value": 62343837, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_other_gain": { + "count": { + "cell": "DD10", + "value": 2177243, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DE10", + "value": 226671889, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "long_other_loss": { + "count": { + "cell": "DF10", + "value": 296456, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DG10", + "value": 2562723, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_gain": { + "count": { + "cell": "DH10", + "value": 1721361, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DI10", + "value": 216227585, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_loss": { + "count": { + "cell": "DJ10", + "value": 552552, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DK10", + "value": 10694019, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_distributions": { + "count": { + "cell": "DL10", + "value": 9733033, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DM10", + "value": 62496866, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Amount", + "Amount" + ] + } + }, + "long_carryover": { + "count": { + "cell": "DN10", + "value": 4371225, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DO10", + "value": 291699768, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Amount", + "Amount" + ] + } + } + } + }, + "23in14acg.xls": { + "url": "https://www.irs.gov/pub/irs-soi/23in14acg.xls", + "sha256": "5b35c2f007f16503c0dcf6cc085d3a6b049ca523d99b7d7ae29afdbdd2d02e70", + "sheet": "Sheet1", + "year": 2023, + "title": "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets\nReported on Form 1040, Schedule D: Selected Items, by Size of\nAdjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 10, + "families": { + "short_term_gain": { + "count": { + "cell": "F10", + "value": 6215519, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "G10", + "value": 70482448, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Amount", + "Amount" + ] + } + }, + "short_term_loss": { + "count": { + "cell": "H10", + "value": 11370982, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I10", + "value": 673910232, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Amount", + "Amount" + ] + } + }, + "long_term_gain": { + "count": { + "cell": "BJ10", + "value": 11888516, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK10", + "value": 971279947, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Amount", + "Amount" + ] + } + }, + "long_term_loss": { + "count": { + "cell": "BL10", + "value": 11230240, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM10", + "value": 450065138, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Amount", + "Amount" + ] + } + }, + "short_assets_gain": { + "count": { + "cell": "J10", + "value": 7140149, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K10", + "value": 58927145, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_assets_loss": { + "count": { + "cell": "L10", + "value": 8377604, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M10", + "value": 135549749, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_other_gain": { + "count": { + "cell": "AZ10", + "value": 174491, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA10", + "value": 8619769, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "short_other_loss": { + "count": { + "cell": "BB10", + "value": 401474, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC10", + "value": 3833687, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_gain": { + "count": { + "cell": "BD10", + "value": 495780, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE10", + "value": 28464220, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_loss": { + "count": { + "cell": "BF10", + "value": 560758, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG10", + "value": 16759596, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_carryover": { + "count": { + "cell": "BH10", + "value": 5151589, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI10", + "value": 543302365, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Amount", + "Amount" + ] + } + }, + "long_assets_gain": { + "count": { + "cell": "BN10", + "value": 8891406, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO10", + "value": 405281941, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_assets_loss": { + "count": { + "cell": "BP10", + "value": 9496262, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ10", + "value": 149558927, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_other_gain": { + "count": { + "cell": "DD10", + "value": 1979380, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DE10", + "value": 316779349, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "long_other_loss": { + "count": { + "cell": "DF10", + "value": 338620, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DG10", + "value": 5211519, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_gain": { + "count": { + "cell": "DH10", + "value": 1219800, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DI10", + "value": 282281744, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_loss": { + "count": { + "cell": "DJ10", + "value": 766861, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DK10", + "value": 22526709, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_distributions": { + "count": { + "cell": "DL10", + "value": 9487999, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DM10", + "value": 44095015, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Amount", + "Amount" + ] + } + }, + "long_carryover": { + "count": { + "cell": "DN10", + "value": 5622923, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DO10", + "value": 349944357, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Amount", + "Amount" + ] + } + } + } + } + }, + "factors": { + "taxable_interest": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "H9", + "value": 42636696, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I9", + "value": 95881223, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "T9", + "value": 55260238, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 313812674, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "2.525269343625908636956987730976528139607", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.599750625636417399355437523686408804619" + }, + "tax_exempt_interest": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "J9", + "value": 5827038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K9", + "value": 61871455, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "V9", + "value": 6893860, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 66091992, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "0.9029088360274333448808012169263853497016", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "0.9295395824924337172788943063597248917385" + }, + "ordinary_dividends": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "L9", + "value": 27607044, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M9", + "value": 260252720, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "X9", + "value": 33718460, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 504224981, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.586285145427430620786783837166923431342", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.633071660126895406016322292262784767590" + }, + "qualified_dividends": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "N9", + "value": 25755976, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "O9", + "value": 203187788, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "Z9", + "value": 31106344, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AA9", + "value": 336070296, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.369498520263452278862567470282453558367", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.409891045424454325551259680558160347719" + }, + "state_tax_refund": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "P9", + "value": 20256512, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Q9", + "value": 31110732, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AB9", + "value": 3109096, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AC9", + "value": 4311977, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "0.9030186910014518713848306957259555062394", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "0.9296526775720357466732616002244135476521" + }, + "alimony_income": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "R9", + "value": 414420, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "S9", + "value": 10077086, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AD9", + "value": 183582, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 6686429, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.497855374161636196047724476262267688168", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.542033706590010885068081742929926580179" + }, + "business_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "T9", + "value": 18791200, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 391975736, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AF9", + "value": 21411713, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 530380019, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.187493036701270108472919624578501757792", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.222517420922031096142334753675412822692" + }, + "business_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "V9", + "value": 5935725, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 60161435, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AH9", + "value": 9067174, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 153441387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.669652726148833347538733957583542565766", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.718898116890934040328182671660914217539" + }, + "non_schedule_d_distributions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "X9", + "value": 4323250, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 11563203, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AJ9", + "value": 3209131, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AK9", + "value": 9340820, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.088252710423015210712466518141315463440", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.120350061633613230084419890940102481805" + }, + "other_property_gain": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AD9", + "value": 1063576, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 33036562, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AP9", + "value": 945122, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AQ9", + "value": 49567896, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.688442578911762234575993608257254511052", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.738242164922618767195190248113271033066" + }, + "other_property_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AF9", + "value": 1153117, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 21093508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AR9", + "value": 775387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AS9", + "value": 19448929, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.371202948062275806522918206792748706765", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.411645744283618865226902643798237455207" + }, + "ira_distributions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AH9", + "value": 14159018, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 253213041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AT9", + "value": 16694154, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AU9", + "value": 438147938, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.467585531705808482641269279807429143993", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.510871073557979130794209158828995676924" + }, + "taxable_pensions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AL9", + "value": 28199160, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AM9", + "value": 689991999, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AX9", + "value": 29541284, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 932130236, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.289553441251142788324750319651443404751", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.327588034973940867230285994253866499704" + }, + "rental_royalty_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AZ9", + "value": 6768234, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA9", + "value": 103058883, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BL9", + "value": 6346357, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 138609251, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.434358304667835504133977902598222238470", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.476663829685885394438774259073237903799" + }, + "rental_royalty_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BB9", + "value": 4531666, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC9", + "value": 46245560, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BN9", + "value": 3196282, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 53570273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.642353427026338423846026443645930519505", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.690793641559507564235995133214906035185" + }, + "farm_rent_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AV9", + "value": 394273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AW9", + "value": 5205330, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BH9", + "value": 325953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 6811206, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.582769973612852083106490319603270854083", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.629452810459537374541656575493532799084" + }, + "farm_rent_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AX9", + "value": 101041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 710953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BJ9", + "value": 67356, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 529586, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.117421316799902418041369349372073484912", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.150378978298943195689490419607276484593" + }, + "estate_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BH9", + "value": 630256, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 32453002, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BX9", + "value": 648583, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 47892046, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.434035630420548027555129084975779987827", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.476331638358104936021886589313386353228" + }, + "estate_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BJ9", + "value": 57802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 5033199, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BZ9", + "value": 36592, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 4898828, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.537463478591264240530009559622650667952", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.582810027948011789708036602150526302352" + }, + "farm_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BL9", + "value": 520802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 13533867, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CB9", + "value": 484162, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CC9", + "value": 17656301, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.403329848619305835091312061082920777637", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.444720208215047581256304812994467826979" + }, + "farm_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BN9", + "value": 1278825, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 27497651, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CD9", + "value": 1258869, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CE9", + "value": 45444491, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.678866744726831887395787683833365558775", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.728383897337776476443532509645504817056" + }, + "other_income_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BX9", + "value": 6121770, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 46769290, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CN9", + "value": 7468434, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CO9", + "value": 60816508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.065879634464584874641519579136196913601", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.097317105419594110775195552571543585636" + }, + "other_income_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BZ9", + "value": 332708, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 6693961, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CP9", + "value": 318839, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CQ9", + "value": 12213628, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.903940252832176698123261573865488200400", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.960095811549227363973941096035769795063" + }, + "short_current_gain": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "short_assets_gain", + "short_other_gain", + "short_pass_through_gain" + ], + "amounts_thousands_usd": [ + 45338966, + 96011134 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.984338725063739328074935070060878814103", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.042865587775988756504710527637911842775" + }, + "short_current_loss": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "short_assets_loss", + "short_other_loss", + "short_pass_through_loss" + ], + "amounts_thousands_usd": [ + 86661831, + 156143032 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.688342557388515472795249540239320768338", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.738139193325432816803529458382722438642" + }, + "long_current_gain": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "long_assets_gain", + "long_other_gain", + "long_pass_through_gain", + "long_distributions" + ], + "amounts_thousands_usd": [ + 787889885, + 1048438049 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.246932856321931105568396313543555115004", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.283710381837894882785950440755768785471" + }, + "long_current_loss": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "long_assets_loss", + "long_other_loss", + "long_pass_through_loss" + ], + "amounts_thousands_usd": [ + 75600579, + 177297155 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "2.197568722014698372473641296575486125444", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.262384673681395979556694950087192296750" + }, + "partnership_s_corp_profit": { + "method": "combined_dollars_per_all_return", + "source": { + "count": { + "cell": "BD9", + "value": 6044409, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE9", + "value": 755622761, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023_components": [ + { + "count": { + "cell": "BP9", + "value": 3266345, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ9", + "value": 452077038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + { + "count": { + "cell": "BT9", + "value": 4138040, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BU9", + "value": 797450861, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + } + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "observed_2015_2023_factor": "1.549554109090802831376907083449129785317", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.595257264233857504616939193376066035754" + }, + "partnership_s_corp_loss": { + "method": "combined_dollars_per_all_return", + "source": { + "count": { + "cell": "BF9", + "value": 2699817, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG9", + "value": 126618186, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023_components": [ + { + "count": { + "cell": "BR9", + "value": 2103634, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BS9", + "value": 168884100, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + { + "count": { + "cell": "BV9", + "value": 1415974, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BW9", + "value": 89543715, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "observed_2015_2023_factor": "1.912533224223542778513579904641714927631", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.968942227400735507640732147170530268700" + }, + "nonqualified_dividends": { + "method": "residual_dollars_per_ordinary_dividend_return", + "observed_2015_2023_factor": "2.412636223005660751243227924409122950293", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.483795459689869824933006427197633652403", + "source_families": [ + "ordinary_dividends", + "qualified_dividends" + ] + } + }, + "modeled_bridge": { + "method": "constant real conditional amounts after 2023", + "cpi_2023": "304.702", + "cpi_2024": "313.689" + }, + "capital_carryover_reconciliation_thousands_usd": { + "short": { + "2015": { + "net_including_carried_loss": -212428017, + "carried_loss": 171105171, + "current_year_net_by_removing_carryover": -41322846, + "current_year_gross_component_gain": 45338966, + "current_year_gross_component_loss": 86661831, + "current_year_net_from_component_difference": -41322865, + "reported_less_component_net_residual": 19, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + }, + "2023": { + "net_including_carried_loss": -603427784, + "carried_loss": 543302365, + "current_year_net_by_removing_carryover": -60125419, + "current_year_gross_component_gain": 96011134, + "current_year_gross_component_loss": 156143032, + "current_year_net_from_component_difference": -60131898, + "reported_less_component_net_residual": 6479, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + } + }, + "long": { + "2015": { + "net_including_carried_loss": 420589539, + "carried_loss": 291699768, + "current_year_net_by_removing_carryover": 712289307, + "current_year_gross_component_gain": 787889885, + "current_year_gross_component_loss": 75600579, + "current_year_net_from_component_difference": 712289306, + "reported_less_component_net_residual": 1, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + }, + "2023": { + "net_including_carried_loss": 521214809, + "carried_loss": 349944357, + "current_year_net_by_removing_carryover": 871159166, + "current_year_gross_component_gain": 1048438049, + "current_year_gross_component_loss": 177297155, + "current_year_net_from_component_difference": 871140894, + "reported_less_component_net_residual": 18272, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + } + } + }, + "notes": [ + "Reporting-return means mix intensive-margin growth and changing composition. This is a distribution transport assumption, not panel growth.", + "Positive/loss magnitudes are distinct series. Source zeros and sign states are retained.", + "No total calibration or reporting-incidence claim; source design weights are unchanged." + ] +} diff --git a/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json b/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json new file mode 100644 index 000000000..23975da76 --- /dev/null +++ b/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json @@ -0,0 +1,34 @@ +[ + { + "url": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "bytes": 83456, + "sha256": "9f41bee263f5a8bb971df39a9673b55136c94954131a95bfd83b73a22801ec89", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "15in14ar.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "bytes": 115712, + "sha256": "b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "23in14ar.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/15in14acg.xls", + "bytes": 64000, + "sha256": "7e128eb06e14371a15baa72060e91d74e4c55ba8596610ddd047c9ef168cbbd1", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "15in14acg.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/23in14acg.xls", + "bytes": 74752, + "sha256": "5b35c2f007f16503c0dcf6cc085d3a6b049ca523d99b7d7ae29afdbdd2d02e70", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "23in14acg.xls" + } +] diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 75804e57c..60a86419d 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "911b21d13cc214601cfb537c538fed2851e5166e8f72de5ebfce5f2918af43d3", + "documentation_sha256": "50a0658251980f47cdbc560300d264afbb32124bb8c1d9720a11ba7e5c73d95b", "field_usage": { - "authored_normative_field_count": 32400, + "authored_normative_field_count": 32458, "claim_count": 49, "claims": [ { @@ -220,8 +220,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 97, - "pointer_sha256": "cabaaa3d96c534f5ec37f51811c3f549d0ed20791e2892c868888c0da6d7d80e", + "pointer_count": 149, + "pointer_sha256": "42d438eba3d0f58fc66aa1f8ef9f58a2f5d600d1a8ec0128ba4accf4a14c0c07", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/transfer_execution", @@ -274,8 +274,8 @@ ], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 24488, - "pointer_sha256": "6382a3ed07016414c31462b8bedb24590d92b792405a89c927d85fa1bd2a8347", + "pointer_count": 24481, + "pointer_sha256": "379a4db6baac065f52386aa438fc6c0be051b141bf87d4a220175a28e4ba9e84", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/producer_graph", @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1712, - "pointer_sha256": "a46d5e657d919db23ee7752d844bd8bbe59c15321dec07102837452d7994d8dd", + "pointer_count": 1725, + "pointer_sha256": "84e499b0b557642a21470dd7bbd2877eb7d096de1b54928b978db11c52945098", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -736,8 +736,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 826, - "pointer_sha256": "7ff2d5d1c2fd8026d17a57244f969dc0e9625a9e47304b693ad15df329041282", + "pointer_count": 827, + "pointer_sha256": "525ed8621baa2c6e0c0970a6cec85b3b250f8aadfe93750a08deb0d9f240f15e", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_protocol", @@ -772,21 +772,21 @@ "verifier": "vintages" } ], - "configuration_field_count": 42180, - "consumed_field_count": 42180, + "configuration_field_count": 42239, + "consumed_field_count": 42239, "generation0_effect_counts": { - "legacy_behavior": 38494, - "no_generation0_effect": 3686 + "legacy_behavior": 38552, + "no_generation0_effect": 3687 }, "mode_counts": { - "compiler_semantic": 27717, + "compiler_semantic": 27711, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14006 + "legacy_behavior": 14071 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "21b6fe2f6b96307f8ec7d20d4162ee2e0702f363bb28ebd3fa47a4cf7fc12ed4", - "resolved_binding_field_count": 9780, + "pointer_inventory_sha256": "b02ebb9faa246da327ef335355e34fb52f8032b61d4f20dbdc349b486de8057b", + "resolved_binding_field_count": 9781, "unused_field_count": 0 }, "inventory_coverage": { @@ -810,7 +810,7 @@ "primary_targets": 65, "producer_authored_outputs": 92, "producer_compiled_outputs": 227, - "producer_inputs": 2744, + "producer_inputs": 2743, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, @@ -1074,11 +1074,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + "schedule_sha256": "75ea2d719acf50c6a37708e6401455edeb0c47be28326a5d241aeb06e1fdb2cb" }, "failures": [], "observed": { - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + "schedule_sha256": "75ea2d719acf50c6a37708e6401455edeb0c47be28326a5d241aeb06e1fdb2cb" }, "status": "covered" }, @@ -1250,14 +1250,14 @@ "expected": { "edges": 71, "nodes": 38, - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", + "schedule_sha256": "75ea2d719acf50c6a37708e6401455edeb0c47be28326a5d241aeb06e1fdb2cb", "waves": 6 }, "failures": [], "observed": { "edges": 71, "nodes": 38, - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", + "schedule_sha256": "75ea2d719acf50c6a37708e6401455edeb0c47be28326a5d241aeb06e1fdb2cb", "waves": 6 }, "status": "covered" @@ -1275,11 +1275,11 @@ ], "expected": { "relation": "source rows preserved exactly", - "rows": 2744 + "rows": 2743 }, "failures": [], "observed": { - "rows": 2744 + "rows": 2743 }, "status": "covered" }, @@ -1338,12 +1338,12 @@ ], "expected": { "nodes": 38, - "sha256": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e" + "sha256": "f922946015d0f4d25f6780a6aa01bcd574c16471d8c5b94ac2062aa05eab6675" }, "failures": [], "observed": { "nodes": 38, - "sha256": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e" + "sha256": "f922946015d0f4d25f6780a6aa01bcd574c16471d8c5b94ac2062aa05eab6675" }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "6abbfad73014ce3bac9fbd549f16e4a5e645a3b0fbd450b56e186876f3f0dd95" + "sha256": "c0785687d5c6054e5e0687bbbde2f0fd29ae030ab5bf446a65db7561b395eb75" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "6abbfad73014ce3bac9fbd549f16e4a5e645a3b0fbd450b56e186876f3f0dd95" + "sha256": "c0785687d5c6054e5e0687bbbde2f0fd29ae030ab5bf446a65db7561b395eb75" }, "status": "covered" }, @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "dad44808366198b8826823b837c7b592c9ecee2299adbd9f8a20251dd6d0b00a", - "protocol_sha256": "5f6d3b527060bc6f15f1871c64a42994d4882ee5962f3350ea8a71a83ffcd6bf" + "map_sha256": "acab6b8da7caa45597b75312b0e5fd008fb4ba7e91e699f22bd21d3fdb71c814", + "protocol_sha256": "6d673c335d43aa7a8da8feb34fd74e702a4f3337a825c7f76fed3f685530cd6f" }, "failures": [], "observed": { - "map_sha256": "dad44808366198b8826823b837c7b592c9ecee2299adbd9f8a20251dd6d0b00a", - "protocol_sha256": "5f6d3b527060bc6f15f1871c64a42994d4882ee5962f3350ea8a71a83ffcd6bf" + "map_sha256": "acab6b8da7caa45597b75312b0e5fd008fb4ba7e91e699f22bd21d3fdb71c814", + "protocol_sha256": "6d673c335d43aa7a8da8feb34fd74e702a4f3337a825c7f76fed3f685530cd6f" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "5f6d3b527060bc6f15f1871c64a42994d4882ee5962f3350ea8a71a83ffcd6bf", + "implementation_sha256": "6d673c335d43aa7a8da8feb34fd74e702a4f3337a825c7f76fed3f685530cd6f", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "5f6d3b527060bc6f15f1871c64a42994d4882ee5962f3350ea8a71a83ffcd6bf", + "implementation_sha256": "6d673c335d43aa7a8da8feb34fd74e702a4f3337a825c7f76fed3f685530cd6f", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "a87278445d928ea20e15935a8cc07312d6e819e34409b04e8c814f17910d828f", - "stages": 37 + "sha256": "2fe81ec636df4125ce56725ef13c81ffbe942d9e8fd04474969905dbeac4edd3", + "stages": 38 }, "failures": [], "observed": { - "sha256": "a87278445d928ea20e15935a8cc07312d6e819e34409b04e8c814f17910d828f", + "sha256": "2fe81ec636df4125ce56725ef13c81ffbe942d9e8fd04474969905dbeac4edd3", "stages": 38 }, "status": "covered" @@ -1840,11 +1840,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2" + "sha256": "c7168518b68ab3ef28eba3086e8447ea48e7b73004dc7dbe78450176a50feaa4" }, "failures": [], "observed": { - "sha256": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2" + "sha256": "c7168518b68ab3ef28eba3086e8447ea48e7b73004dc7dbe78450176a50feaa4" }, "status": "covered" }, @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "b1d5c65254b51897b27f0f5e44e344781ce9e01e305aaf1bfc042cc32bc417dc" + "sha256": "2a1ff0f7339372eed74bc22951fdb98a88ce2c42536daea7f9e14b436095dfa2" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "b1d5c65254b51897b27f0f5e44e344781ce9e01e305aaf1bfc042cc32bc417dc" + "sha256": "2a1ff0f7339372eed74bc22951fdb98a88ce2c42536daea7f9e14b436095dfa2" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "5b65545ed6f351b30e94ab1405e13d2c7f692ae0c8d4959badcfe9bb9d52d455" + "sha256": "c8a93e840f21d9fa47d74bed321b4b6b76fd1b34ffe76ca162e40c7de044b2d2" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f830589331dad583398485de5724dace0714d8526fb63b37f81cd783b1cb8f30" + "spec_sha256": "2dfa51b89e740c742437168cb07f3084a2ecd7f90ad61eb13d3b12b69f315e15" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f830589331dad583398485de5724dace0714d8526fb63b37f81cd783b1cb8f30" + "spec_sha256": "2dfa51b89e740c742437168cb07f3084a2ecd7f90ad61eb13d3b12b69f315e15" }, "status": "pass" } diff --git a/docs/fiscal-leaf-policy.md b/docs/fiscal-leaf-policy.md new file mode 100644 index 000000000..823603506 --- /dev/null +++ b/docs/fiscal-leaf-policy.md @@ -0,0 +1,114 @@ +# Fiscal input policies + +The declared fiscal measurement stage requires an input producer for every +leaf in its static model closure. Missing and nonfinite values still refuse +evaluation. The optional leaf policy records more precise intent without +weakening that default. + +An explicit policy can describe producer ownership or a proposed literal +scenario assumption. **Assumption-bearing executions currently refuse with +`FISCAL_MEASUREMENT_ASSUMPTION_PARENT_ADMISSION_UNSUPPORTED`.** No US assumption +resource is enabled, and no dataset column or producer requirement changes. + +## Exact policy identity + +`load_fiscal_leaf_policy(payload, expected_sha256=...)` accepts bounded, +canonical JSON bytes and an explicit SHA-256 pin. It performs no file or +network I/O. The same `FiscalLeafPolicy` goes to `fiscal_measurement_node` and +`FiscalMeasurementKernel`. Its pin is rechecked when used; the parsed document +is detached. Omitting the argument, or passing `None`, preserves the previous +all-producer declaration shape and runtime behavior. + +A policy has schema version 1, artifact kind +`microcosm.us.fiscal_leaf_policy`, one positive integer `period`, an exact +ordered list of model `roots`, and an `entries` mapping covering the static +closure exactly. Missing entries and stale entries refuse. Each entry's +entity must agree with the independently derived model metadata. + +| Kind | Required entry fields | Behavior | +| --- | --- | --- | +| `producer` | `kind`, `entity`, `producer` | The declared input column must exist and every cell must be known and finite. | +| `assumption` | `kind`, `entity`, `value`, `interpretation`, `period`, `roots`, `affected_roots`, `affected_programs`, `rationale`, `reviewer`, `source_issue`, `reform_sensitivity` | Validated and inspectable; execution remains unsupported. | +| `inactive` | No supported shape | Always refused. | + +An assumption's interpretation is explicitly `baseline_behavior` or +`scenario_parameter`. Neither category means the program is insensitive to +the choice. The affected roots must include every declared root whose static +closure contains that leaf, in declaration order. Scope must exactly match +the policy's period and roots; program names and review fields cannot be +empty. These fields describe claimed review and intent. They do not prove +reviewer authorization, establish a source observation, or make an observed +attribute an acceptable assumption. + +V1 supports finite Boolean, signed 64-bit integer and floating-point literals +with explicit engine type checks. Strings, enums, null, containers and an +`engine default` reference refuse. The actual engine metadata and whether a +default is available (with its literal value when present) are recorded +beside the proposed value. The host never chooses a value by default lookup. +The exact policy and engine/default record enter `model_contract`, so policy, +scope, ownership, review or engine-default changes alter the declared node and +its cache key. The same declaration is embedded in a successful measurement +artifact. In this version, only producer policies can produce that artifact. + +A column constant at an engine default remains admissible as a producer at +this boundary. The label does not establish observation or modeled variation. +The source owner and existing release input-coverage/variation gates retain +that responsibility. Behavioral take-up and filing propensities are not +automatically classified as innocuous scenario settings. No prior wages are +added by this infrastructure. + +## Complete-parent admission still required + +The graph kernel receives only its declared input slices. It cannot prove a +column is absent from the complete parent when a caller omitted that column +from the slice. Declaration checks reject an assumption name present on any +declared entity, but this is insufficient to enable execution. + +The future country host must retain its independently checked complete +population and use `validate_fiscal_leaf_policy_population(frame, policy)` +before projection, around relevant I/O, and during cache admission. The +validator rejects any incumbent assumption name on any entity, including +wholly unknown or partially unknown columns. Passing a projection or a +caller-authored column roster is not absence proof. The validator itself +does not authenticate or issue a parent. + +`private_fiscal_assumption_frame(frame, policy)` is a detached preparation +helper for that future integration. It repeats the collision check and +creates engine-only tables. It preserves the supplied frame's values, +membership, weights, strata and metadata. The current fiscal kernel does not +call it. A future host must validate the exact policy against the admitted +engine, retain source and complete-parent checks before and after I/O, bind +the engine/default record, evaluate only the private copy, and keep assumption +columns out of the population and measurement adapter. A cached artifact +cannot supply missing source or parent authority. + +## Why inactivity is unsupported + +`validation_input_coverage._provision_input_leaves` traces a single empty +one-person simulation. It observes the leaves read by that execution. A +conditional formula or `defined_for` branch may read additional leaves for +other households. Absence from that trace cannot establish universal +inactivity, even at the same period and roots. + +Future inactivity support needs exhaustive engine-, period- and root-scoped +dependency proof, including conditional paths and declared state roots. The +static closure remains conservative; neither it nor a sampled trace supplies +this exemption today. + +## Bounded implementation evidence + +The local source/mock test run passed 91 cases, including the existing +invented fiscal measurement, calibration and observer graphs. The new controls +check strict-default compatibility, malformed policies, exact scope and +closure coverage, type mismatches, private-copy isolation, omitted-column +collisions, late policy mutation, graph key changes and required replay with +a forbidden model call. There were no country engine imports, network +connections or subprocess operations; no native population or source data +was used. This is infrastructure validation, not actual engine-policy +admission or candidate fiscal validation. + +The source and selected test files were sealed before and after the run. +Receipt SHA-256: +`c7aed643333638460d7464091be0c068b910baba2eefbb76cfa1208af6f8265f`. +JUnit SHA-256: +`54215b02c7a8a85488db48b2fee339b0ddbad907a0985502019c185b9a1c3adb`. diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md new file mode 100644 index 000000000..5793b0b1f --- /dev/null +++ b/docs/geography-assignment.md @@ -0,0 +1,261 @@ +# Geography assignment in the population graph + +Design decision, corrected 10 September 2026: finish constructing the full survey +multispine, including its initial support and PUF clones, then assign one small +census area to each resulting household. Derive larger geographies from that +area using identified, versioned mappings. Distinct clones may receive different +areas subject to their observed source constraints. The assigned location then +remains fixed through subsequent enrichment, calibration and analysis views. + +The earlier block-before-clone implementation and its passing controls remain +historical evidence. They do not demonstrate this corrected ordering. Geographic +assignment must use a stable post-clone identity, including a clone discriminator +where original-source identity alone is shared by multiple households. + +The corrected order now passes 70 controls on invented inputs, plus a separate +default-path compatibility check. The nine-node survey prefix expands the initial +clones before assignment; its nineteen-node financial and twelve-node age +extensions retain the assigned geography. Draw identity combines +`survey_geography_origin_key` with `household_support_clone_index`, so renumbering +households does not redraw their location. A typed validation artifact orders +financial enrichment after the geography gate. See the +[scoped acceptance](../experiments/us-postclone-geography-70-controls-20260910.json). +Native execution of this corrected order and population-quality acceptance +remain pending. The older results below retain their original source scope. + +## Declarative country capability + +Geography is a declarative country capability backed by shared graph operators. +A country declares its atomic area type, code system and vintage, source mapping +artifacts, observed-geography constraints and sampling-weight convention. The UK +declaration can select an area system by nation. Adding a country should not +require writing a country-specific assignment or derivation kernel. + +The shared assignment operator selects the atomic area using the declared +support and stable household identity. Shared lookup/join operations derive +larger geographies from identified mappings. If an input already has a qualified +atomic area, validate and retain it. Source acquisition and normalization may +still require publisher-specific adapters; they produce the common support and +mapping contracts, rather than owning a separate assignment algorithm. + +Sampling and derivation remain executable graph operations with typed inputs, +outputs and replay identity. A country configuration does not itself execute +them. Mapping edges retain their relation type, including exact nesting or an +explicit best-fit convention. Prefer existing graph lookup/join primitives where +they support these contracts; introduce only the shared behavior they lack. + +The existing country-spec geography declaration is a starting point, but its +legacy clone-and-assign contract and country-specific runtime references do not +yet implement this shared atomic-area contract. Preserve those compatibility +paths while the shared operators and country adapters receive their own checks. + +```mermaid +flowchart LR + S[Complete survey multispine including clones] --> A[Assign one small area per resulting household] + C[Observed survey geography constraints] --> A + L[Versioned small-area support] --> A + A --> D[Derive larger geographies] + M[Versioned geographic mappings] --> D + D --> E[Subsequent enrichment retains location] + E --> R[Rules and calibration] + R --> P[Pruned analysis file retains location] +``` + +## Country anchors + +| Country | Assigned household anchor | Derived geographies | +| --- | --- | --- | +| US | Census block, with census vintage | Block group, tract, county, state, PUMA and congressional district using the declared mapping/boundary vintages | +| England and Wales | 2021 Output Area | LSOA, MSOA, local authority, ward, constituency and region | +| Scotland | 2022 Output Area | Scottish statistical areas, council area, ward, constituency and region | +| Northern Ireland | 2021 Data Zone | Super Data Zone, local government district, constituency and nation | + +Northern Ireland's 2021 Data Zones replaced the 2011 Small Areas. Scottish +Output Areas use the 2022 census. These are different source systems; the graph +must retain the original area type and vintage rather than imply identical units. +See [NISRA census output geography](https://www.nisra.gov.uk/statistics/census-2021-results/census-output-geography) +and [Scotland's census general report](https://www.scotlandscensus.gov.uk/about/scotlands-census-2022-general-report/). + +Some larger boundaries do not nest exactly. England and Wales small-area estimates +use best-fit mappings for wards and parliamentary constituencies. The derived +edge must identify whether its mapping is exact, best-fit or another explicit +approximation, with its source and vintage. A mapping convention cannot be +presented as observed household location. See the +[ONS methodology](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationestimates/methodologies/smallareapopulationestimatesqmi). + +For US congressional districts, the Census Bureau's CD119 block equivalency +file assigns whole original 2020 tabulation blocks to districts. Colorado's +block `080010096072000` crosses the enacted CD07/CD08 boundary but is assigned +to CD08 for tabulation. Represent this as an official tabulation mapping, +not exact spatial containment. The same source distinguishes original 2020 +blocks from subsequently adjusted geometry and includes undefined `ZZ` areas. +See the [Census Bureau's CD119 documentation](https://www.census.gov/geographies/mapping-files/2025/dec/rdo/119-congressional-district-bef.html). +By contrast, the same-vintage tract-to-PUMA relation is nested, as described in +the [Census PUMA guidance](https://www.census.gov/programs-surveys/geography/guidance/geo-areas/pumas.html). + +## Required behavior + +- Respect observed source constraints, such as ACS PUMA and FRS region. Record + inferred geographic detail as modeled. Preserve observed source geography + separately from the assigned location. +- Bind each draw to a stable household identity, seed and assignment definition. + Reordering rows or selecting an existing household must not redraw its location. + Changed source support or boundary mappings produce a new identified revision. +- Assign household members consistently with their resulting household. Initial + clones have distinct draw identities and may receive different anchors. + Materialized county, district and other fields must agree with that household's + selected anchor. Source geography remains a separate observed constraint. +- Show assignment and derivation as separate operations in the graph. Expose the + source constraints, support, sampling weights, mapping conventions and judgment + annotations in the inspector. +- Refuse missing, ambiguous or incompatible mappings unless an explicit, + source-supported resolution rule is part of the assignment definition. +- Verify inheritance and mapping consistency again on every pruned export. + Engine input profiles and geographic build invariants are separate contracts. + +## Existing implementation and next changes + +The shared implementation now lives in `microcosm.build.atomic_geography` and +`microcosm.build.graph_atomic_geography`. It provides four country-neutral nodes: +`geography.support_import@1`, `geography.assign_atomic@1`, +`geography.derive@1` and `geography.gate@1`. Countries supply declarations and +normalized, pinned support files. The graph interfaces and legacy country +operators are unchanged. Country configuration and native support integration +remain necessary before these nodes can assign locations in a release build. + +The support contract uses deterministic NPZ bytes with no object arrays. It has +one unique string code per atomic area; every mapping column identifies its +source, vintage and relation, and every integer sampling-weight column identifies +its source and basis. Nonnegative weights have a bounded exact total. A source +adapter must resolve any nonfunctional crosswalk before admission; the shared +lookup never multiplies households through a many-to-many join. + +Assignment conditions on all nonmissing observed constraints jointly. Declared +stages can use different weights—for example, household counts to select a +constituency followed by population counts to select its small area. Each draw +uses the existing `keyed_uniform` protocol with stable source identity, the +declared stream, system, stage, assignment definition and support digest. +Selection uses an integer inverse CDF. Reordering, subsetting or adding unrelated +households preserves draws. The numerical declaration remains `platform_bitwise`; +cross-platform equivalence has not been established. + +The version-1 declaration lists `identity`, `stream`, three assignment `outputs` +(`area`, `system`, `basis`), and `systems`. Each system supplies its atomic +identity and source, a selector, observed constraints, an optional observed-area +column, sampling stages and derived layers. Geographic codes use nullable +strings throughout, retaining leading zeroes. An engine that needs integer +storage must declare that conversion separately. A system without a particular +layer produces missing values for that layer, which the integrity gate checks. +The graph stores the canonical declaration in its normative `definition` +parameter and exposes the stream separately; no graph parameter grammar changed. + +The 34 invented-data controls include real graph execution, cold and warm reuse, +source-change invalidation, exact lookup metadata, independent sampling-boundary +checks, three-system routing, observed-area retention, row/subset stability and +clone/prune validation. All passed under the isolated control runner on +9 September 2026. These checks establish shared operator behavior; they do not +admit a Census/ONS source or certify any population's geographic fit. The gate +checks mapping integrity, while the executor and build own structural lineage +and population-quality acceptance. The country's chosen identity still needs +verification across actual sampling rungs. + +The US development composed graph attaches geography after harmonization, but +`us_runtime/graph_geography.py` selects a joint tract/congressional-district cell +and emits PUMA, county and district. It does not assign a Census block. Its prior +national source and joint-support checks therefore do not establish block-first +acceptance. The separate atomic survey graph now provides that block assignment +and derivation connection before enrichment. Its population-only national block +support and source review are recorded below. + +The UK already has an area-based ladder in `uk_runtime/geography_ladder.py`. +Its sampler selects a constituency within FRS region using household counts, +then an OA within constituency using population; one selected ladder row supplies +the other geographies. It currently consumes a shared seeded random stream, so +stable household-keyed assignment under reordering and subsets still needs work. +The graph should expose the final OA/Data Zone assignment and the mapping +derivations explicitly, including that sampling convention. + +The Northern Ireland source builder currently infers Data Zone constituencies +using the modal active-postcode constituency. Review the official +[NISRA constituency aggregation and lookup resources](https://www.nisra.gov.uk/publications/census-2021-output-geography-information-papers) +before choosing or replacing that approximation. Source existence is not +acceptance of a particular lookup or its application to the population. + +The US block adapter in `us_runtime/atomic_block_support.py` now preserves each +supplied block, leading zeroes, population weight and the source-labeled +tract-to-PUMA and block-to-district mappings. Its declaration uses the shared +operators, preserves observed state/PUMA separately, and labels CD119 as +`official_tabulation`. Its sampling proxy is 2020 persons; this does not imply +household counts or support for subsequent construction in unpopulated 2020 +blocks. All 13 invented tests passed, including missing/inconsistent mapping +refusals and subset-stable assignment. Receipt SHA256: +`404deb7666b3ac9b8a654181228c545ab95c131f30a1fbe65a2e15d5a08a4c4c`. +These initial adapter tests do not establish native geographic fit. Subsequent +national support and native pilot results are recorded below. + +The earlier `us_runtime/graph_atomic_survey_clone.py` declared this sequence: +shared import, assignment, derivation and integrity gate, then the existing +combined-survey support clone and an inherited-mapping gate. The graph compiler +makes the clone depend on every member of its base version, including the +pre-clone gate. The post-clone gate never draws another location. An actual +executor control passes cold execution and required replay: four invented +households become eight, every location column is inherited, all entity rows +are copied, and each household weight is split equally across the pair. +Subset mapping verification also passes. Receipt SHA256: +`1bb75a5ee53edcbdd199a6d0163cbe75c32e54c80513cb3eb8d13021c3c2b1c3`. +A separate source-only Codex review found no actionable defects. + +The current survey path now qualifies observed state and ACS PUMA from the +retained original-source preparation. Eleven source controls and fourteen graph +controls passed, covering missing geography, source changes, replay and stable +household identity across sampling rungs. These controls use invented originals +through the actual source issuers. Their receipts are recorded in +`experiments/us-survey-geography-source-controls-20260909.json` and +`experiments/us-survey-geography-graph-controls-20260909.json`. + +The earlier `us_runtime/graph_atomic_survey_population.py` connected that projection +to block assignment, geographic derivation, the integrity gate and the combined +survey clone. It retains the raw allocation separately from the enriched +pre-clone population, even though both share the allocation version identifier. +The complete ten-node prefix passed five controls: fresh execution, required +replay, full geography inheritance, support-byte mismatch and late mutation +refusals. The test population contains six survey households before cloning and +twelve afterward. The exact tested revisions and remaining checks are recorded +in `experiments/us-atomic-survey-population-controls-20260909.json`. + +The earlier calibration budget and age runner consumed this optional prefix +and independently reconstructed it from the raw allocation. Eight budget +controls passed, as did the complete thirteen-node age graph through fresh +execution and required replay. A separate compatibility control verifies that +the existing predictor and PUF-host qualifiers still accept the default survey +prefix. See the corresponding `us-atomic-budget-semantic-8`, +`us-atomic-age-v2-1`, and `us-budget-predictor-compatibility-1` records in +`experiments/`. + +Replay can canonicalize the hidden backing values of missing numeric cells. +Persistent budget identities therefore bind logical Frame values, population +version, complete ownership, weight kinds, mass ledger and exact design-weight +bytes. They exclude only the helper receipt's physical population stamp. +Same-object mutation checks retain physical identity. The initial physical-stamp +replay failure remains recorded; the corrected budget controls pass. + +The byte-source adapter passed thirty-six invented-source controls through the +maintained PL, CD and PUMA parsers. This does not authorize acquisition of native +PL archives, which also contain housing segments. Native support preparation +will instead request only Census `P1_001N` block populations and independent +state totals, then join CD119 and tract-to-PUMA mappings. Delaware is the first +source control; national survey assignment requires complete admitted support +for the fifty states and DC. No native acquisition is accepted by these tests. + +`us_runtime/atomic_block_api_sources.py` implements that population-only response +adapter. All thirty-five invented controls pass, including independent state +totals, exact block preservation, missing mappings, malformed response fields, +byte limits and late source-record mutation. Request descriptors preserve the +repeated Census `in` parameters and omit credentials. The adapter does not infer +the response's origin from its contents; acquisition and publisher qualification +remain separate. Evidence: +`experiments/us-atomic-block-api-sources-35-controls-20260909.json`. + +Publisher-qualified native block support, national and district calibration, +and pruned export verification remain necessary for a release. Source integrity +checks and successful invented-data runs do not establish geographic fit. diff --git a/docs/graph-signed-reconciliation.md b/docs/graph-signed-reconciliation.md new file mode 100644 index 000000000..8cba5aceb --- /dev/null +++ b/docs/graph-signed-reconciliation.md @@ -0,0 +1,57 @@ +# Signed component reconciliation in the graph + +`microcosm.fit.graph_signed_reconciliation.signed_reconciliation_node` exposes +the existing weighted Euclidean projection as an ordinary graph calculation. +Its parameters name the anchor, input draws, output components, nonnegative +bounds, positive scales and numerical tolerances. The country model must +declare and justify those choices. The operator supplies no default scale or +tax interpretation. + +For the proposed ACS property-income family, the components are ordinary +interest, retirement-account interest, dividends and signed property income. +The first three have nonnegative bounds; property income can offset them. +A zero reported total therefore does not imply that every component is zero. +Retirement-account interest remains a separate survey component and is not +relabeled as taxable or tax-exempt interest. Source qualification, a defensible +ASEC-to-ACS measurement bridge, fitting and tax conversion remain separate +upstream/downstream operations; this numeric node alone does not complete them. + +The graph retains the anchor and raw draw columns. It adds the reconciled +components, each component's adjustment and bound activity, the sum residual +and the projection objective. A typed summary artifact records the complete +rule, row count, adjusted-row count, maximum absolute residual and bound counts. +It describes the numerical result and grants no source or release authority. +These declarations and columns make the rule inspectable in the shared graph +viewer without another viewer implementation. + +All input amounts must be finite float64 values. Unknown anchors or draws +refuse; the country host must declare how it selects or models those cases. +Inputs cannot share names with the new outputs, and no original observation +is overwritten. The kernel never reads survey origin, weights or an RNG and +returns no membership or weight changes. Its numerical contract is bitwise on +one platform; the explicit projection tolerances test constraints within each +run and are not a claim of cross-platform equality. + +## Verification on 12 September 2026 + +Twenty-four guarded tests pass, including direct comparison to the independently +tested pure projection; negative and zero-net anchors; exact IDs above 2^53; +row permutation; missing, infinite and incorrectly typed inputs; invalid +bounds/scales and column collisions; and declaration/context disagreement. +An actual two-node graph executes cold, retains the complete input Frame, +metadata and zero-weight record, then reopens the store under required replay. +The replay hits both nodes and preserves artifact identity. Changing scales +reuses the source node and invalidates the reconciliation node. + +The closed run used 3.69 seconds wall time and 417,775,616 bytes peak RSS. +All 1,000 source and eight owned hashes, plus 15 resource hashes, were unchanged; +there were no unexpected denied operations or child processes. These are +invented-data graph checks, not native Microcosm acceptance. Receipt SHA-256: +`7e0ed8fefc911b9d7969f48e0d2c382c189e40a0427a120a163a8ce9c8ffae01`. +JUnit SHA-256: +`914745d61c768ae4f445df408db89601d4a4268374ba4f0e6caba6085ea6be6c`. + +The first run preserved 22 passing tests and two fixture failures: the test +attempted an empty `Weights` object and constructed unsorted group IDs. +Correcting those fixture inputs produced the result above; production source +was unchanged between the two runs. Independent review is recorded separately. diff --git a/docs/joint-empirical-graph.md b/docs/joint-empirical-graph.md new file mode 100644 index 000000000..bccfee1a2 --- /dev/null +++ b/docs/joint-empirical-graph.md @@ -0,0 +1,118 @@ +# Joint empirical graph operations + +The reusable adapter fits a weighted paired empirical law and draws from it with +the maintained keyed-uniform protocol. It uses `joint_empirical.py`; it does not +fit QRF or independently sample two marginals. The source host selects and +authenticates donors and recipients, supplies an explicit support policy, and +independently verifies draws before attaching them to a survey population. + +This source implementation depends on the numerical operator introduced in +`aaa8b071ab14a97e514deb05d367f74a964abb29`. Its tests are authored but unrun. + +## Private support branches + +The 2026-09-13 root decision clarifies the initial child implementation plan: +independent private model-support CREATE branches are allowed so their exact +column schemas appear in graph Slices. They do not change the receiving survey +population, original spine, clones or geography. There is no new EXPAND operation. +The US source owner creates those branches from its retained qualified source. +The reusable adapter owns only the fit and draw operations. + +Side-table entity IDs must be exact unique int64 values. They identify rows when +results are joined back and are excluded from the declared random coordinates. +Original source coordinates occupy separately declared int/string columns. No +float or Boolean source coordinate is accepted. The host must validate both the +original coordinate and the original-person join; a matching side-table ID alone +does not confer source authority. + +## Fit contract + +`joint_empirical_fit_node` declares one all-row Slice with the two ordered target +columns, original donor and household key columns, and explicit weight column. +Targets and weights are nonnullable float64; original keys retain their integer +or string types. Donor keys must be unique, while household keys may repeat. +All numeric validation and support rules execute in the real numerical model. +The input weight kind is recorded as `explicit`; the host separately documents +and authenticates any original household design-weight provenance. + +The node consumes a typed `source_projection` artifact whose payload hash is an +explicit parameter. This is the full donor projection, distinct from the selected +recipient projection. The complete support document is a canonical JSON string +plus its SHA256 in normative parameters. No production support default exists. + +The node emits the numerical operator's exact private `model` bytes and a typed +`model_metadata` artifact with model/support/donor-source hashes, target names, +explicit weight kind and aggregate diagnostics. Model bytes contain paired donor +values and exact tagged private coordinates. Metadata contains no donor IDs. +Changing selected recipient sampling does not enter the numerical model bytes. + +## Draw contract + +`joint_empirical_draw_node` declares the original-recipient coordinate Slice. It +can also declare an exact nonnullable Boolean `eligibility_column`. The support +Frame then contains all real original recipients; the kernel selects eligible +coordinates internally, with candidate/draw counts in its receipt. An all-false +column produces a legitimate zero-row draw while retaining the nonempty support +Frame. This uses no sentinel person and does not relax Frame or Weights: a +row-masked empty Slice would itself require forbidden empty projected weights. +An empty direct DataFrame test is only a numerical context test, not this graph +path. Invalid or duplicate unselected coordinates are still refused. + +The node +consumes model and metadata from the same producer and the separate typed +recipient `source_projection`. It checks platform scope, artifact store identity, +the numerical model's complete plain-data encoding, support agreement, donor +source hash and recipient projection hash. + +Parameters include the complete stream `(algorithm, experiment_id, replicate, +base_seed)`, a coordinate suffix, the complete canonical transport document and +hash, and explicit source/support/scenario revisions. The US child candidate +uses suffix `('child-property-v1',)`. The adapter appends `pattern` and `donor` +separately and calls the real `keyed_uniform`. It declares `SeedSource.KEYED` and +does not consume the node-key-seeded sequential generator. + +Only the original source coordinates and explicit suffix enter the stream. +Side-table IDs, clone numbers, row position, batch boundaries, model node keys, +recipient sample size and scenario hashes do not. The scenario hash is a host +revision reference, not authority over its semantics; the actual transport and +stream parameters are independently normative. Scenario comparisons share +uniforms while remaining distinct content-addressed graph operations. + +The private `draws` artifact records exact tagged original coordinates, decimal +int64 side IDs, paired float64 values, patterns, positive donor references and +all model/support/source/scenario/transport bindings. Rows have canonical +coordinate ordering. Pattern zero has no sampled donor identity; it remains a +model zero rather than an observed nonreceipt. No survey columns are written. + +`read_joint_empirical_draw` checks exact supplied side IDs and original keys plus +the expected hashes and parameter documents. It returns values in the supplied +order as bytes-backed readonly arrays. This validates the serialization and join +contract, not the draw law: a host must reconstruct the actual model and keyed +draws from retained qualified source before accepting canonical attachment. + +## Verification and limits + +The invented tests cover actual numerical fit/draw calls, unequal-weight paired +probabilities, exact large IDs, integer/string distinctions, row permutations, +partitions, recipient additions, side-ID changes, common scenario uniforms, +typed artifact/producer/hash changes, malformed private joins, explicit zero +stress and empty recipient batches. A real four-node cold/required replay checks +both model-support Frames and scenario-specific draw keys. No test grants survey +authority, and no native fit or production support policy is claimed. + +The numerical model is capped at 64 MiB and 1,048,576 donors. Private draw JSON is +also capped at 64 MiB and 1,048,576 recipients; the byte check occurs after Python +object construction. These are ceilings, not measured national capacity. Before +large use, measure bytes per row, working-memory peaks and repeated validation +cost on small samples. A separately reviewed compact transport may be needed. +Implementation identity includes this adapter, the numerical model, weight +resolver, keyed-randomness module and its canonical serializer implementation, +with NumPy/Pandas platform-bitwise scope. Serialized metadata and draw-header +bindings compare canonical bytes, preserving integer/Boolean/float distinctions. + +The corrective source tests add valid changed-model/stale-metadata refusal, +noninteger encoded stream/count refusal, canonical source identity and actual +compiler-key changes for source/support/transport/stream revisions. The actual +four-node runtime test now covers all, some and zero eligible recipients with +cold/required replay and refusal of corrupted invented cache payloads. These +tests are authored and UNRUN pending a separately approved bounded runtime. diff --git a/docs/native-hours-source-owner.md b/docs/native-hours-source-owner.md new file mode 100644 index 000000000..bbdabbfb6 --- /dev/null +++ b/docs/native-hours-source-owner.md @@ -0,0 +1,109 @@ +# Native hours source ownership + +`qualify_current_survey_hours(preparation, age15_policy=..., under15_policy=...)` +connects the pure usual-hours proposals to the original ACS and ASEC source +records held by an authenticated survey preparation. Version 2 returns complete +selected-original-person hours for both arms. It preserves source observations +and labels explicit completion and empirical imputation separately. The source +qualifier does not modify a Frame; the graph fragment described below owns +attachment to the receiving population. + +The function captures the preparation's pinned ACS person archive and current +ASEC person member, checks their sizes and hashes, exhausts every CSV record +and archive member, and joins selected people by original survey coordinates. +Original ages must match the preparation. It derives the complete income-2024 +age-15 donor key set from the issued ASEC coverage before selection and requires +the captured member's donors to match that exact set. The production roster has +2,174 people; matching that count alone does not qualify a different roster. + +The age-15 empirical and under-15 zero-completion policies are explicit arguments. +The pure proposal module supplies their versioned names, seed, native-keyed draw +and interpretation of raw codes. Source allocation and response flags remain +distinct from modeled provenance, including valid `FL_665=0` supplement +nonresponse. Both future clones should receive the same original-person draw. +This source owner neither creates clones nor assigns hours to them. + +The returned `QualifiedSurveyHoursProposals` holds private raw tables, proposals, +original source owners and an immutable receipt. Call `validate()` before +borrowing the result and after final relevant I/O. It checks implementation, +retained source identities, exact projection objects, table storage and nested +proposal contents, including ASEC earnings and the combined person projection. +A copied object, arbitrary callback, matching detached table +or receipt alone does not replace the retained object. Raw records and donor +assignments are private evidence; public diagnostics should use aggregate counts +and provenance summaries. + +The receipt qualifies selected original-person hours and the ASEC own arm. It +deliberately leaves `all_person_engine_input_qualified`, `source_admission_issued` +and `release_eligible` false. The pure batches' scope flags remain false as well; +the enclosing live source owner supplies the checked relationship rather than +changing those flags. `person_hours` preserves the original-person index order +and includes the target, provenance and policy columns. The ACS `proposals` +and separate `asec_proposals` retain native keys and literals for inspection. + +ASEC hours use the [separate observation and completion rules](asec-usual-hours-observations.md). +The under-15 zero assumption requires explicit policy and known-zero source +earnings; it is never relabeled observed zero. Missing adult hours refuse complete +projection construction. Historical engine defaults, last-week hours and prior +wages are not substitutes for current source evidence. + +`borrow_cloned_hours_columns(qualified, receiving_people)` reuses the existing +exact two-clone attachment contract. It transports each original proposal and +its provenance to both descendants without redrawing hours. It checks original +IDs, native IDs, survey channels, complete clone pairs and output ownership. +It validates the retained source owner before and after borrowing, then checks +that source I/O changed neither the recipient table nor the borrowed output. +The returned Series are independent copies. This helper does not issue receiving +population authority: the graph host retains and authenticates its own receiving +owner and binds these columns to execution and replay. + +## Graph execution and replay + +`graph_current_survey_hours` declares four ordinary operations on the receiving +population. `survey_hours.source` retains the authenticated source projection in +a private artifact. `survey_hours.asec_recode` executes the ASEC recoder and +explicit under-15 completion. `survey_hours.acs_recode_impute` executes ACS +recoding, the original-keyed age-15 empirical draw and under-15 completion. +`survey_hours.attach` checks the two column artifacts against the independent +source-owner result and attaches hours, provenance and policy to both clones. + +The attachment reads only the four ancestry and survey-channel columns. It owns +`weekly_hours_worked_before_lsr`, `hours_provenance` and `hours_policy`; existing +ownership refuses before execution. Public graph parameters contain source +hashes, survey periods, policy identifiers and the seed. Raw literals, identifiers +and donor assignments remain in private artifacts. + +`graph_us_survey_enrichment` appends these operations after housing attachment. +Its final population is the hours attachment output. On cold execution and +required-cache replay it compares every hours artifact and intermediate +population with expectations derived from the independent retained source +owner. The host checks current source and implementation ownership after final +store I/O. Its receipt records the hours source and attachment hashes and both +policies, while release eligibility remains false. Prior wages are not inputs. + +## Verification boundary + +The tests use the real source issuers, preparation, private capture and owner +validation with tiny invented archives. A selected sample deliberately excludes +the sole age-15 donor household while the full donor cohort remains available. +Tests compare full versus selected proposals and cover explicit policy choices, +malformed/duplicate records, detached owners, replaced callbacks and receipts, +table and nested-proposal mutation, retained coverage mutation, changed seed, +and projection mutation during final source I/O. The small graph tests also run +cold and required-cache execution, check clone equality after reordering and +preserve existing tables, memberships and weights. They refuse changed private +artifacts, copied or revoked owners, output collisions and mutation during the +final host callback, and check that policy changes invalidate downstream keys. +The fixture's one-donor limit +is a private pre-issuance test seam; production retains the 2,174-key requirement. + +This change continues the native branch based on `fee4aac9d`, including pure +hours and response-status correction `4f34495e5`. Fresh main was inspected and +lacks these native APIs, so it is not an independent main-branch replacement. +Actual full-source owner and integrated-host execution, calibrated release +comparisons and publication remain separate. The hours fragment has been tested +without rerunning the financial and full PUF fits; that bounded test does not +establish native production qualification or David's hours-distribution gate. +A complete +all-age ASEC source diagnostic supports the recoder but does not replace the +retained source owner or qualify its production execution. diff --git a/docs/native-integration-20260923.md b/docs/native-integration-20260923.md new file mode 100644 index 000000000..35c1d561e --- /dev/null +++ b/docs/native-integration-20260923.md @@ -0,0 +1,392 @@ +# Native line integration on main — 2026-09-23 + +This record covers the `native-integration-20260923` lane for +[microcosm#956](https://github.com/PolicyEngine/microcosm/issues/956). +It preserves the native development stack, integrates current main, and keeps +survey poverty comparison-only. It records code integration, not an actual-data +run, artifact certification, publication approval, or a completed native release. + +## Superseded review stack + +The integration PR consolidates these earlier review branches. Their recorded +remote tips were each verified with `git merge-base --is-ancestor` against the +integrated lane head; this does not close or merge their PRs. + +| Earlier PR | Included branch and checked tip | Scope carried into integration | +| --- | --- | --- | +| [#893](https://github.com/PolicyEngine/microcosm/pull/893) | `microcosm-us-launch-integration-20260909`, `a402dfcc9` | Original US graph-native integration | +| [#935](https://github.com/PolicyEngine/microcosm/pull/935) | `native-verify-once`, `c6e38d6f1` | Verification reuse within a native run | +| [#945](https://github.com/PolicyEngine/microcosm/pull/945) | `native-scale-transport`, `ae3955cd9` | Receipt transport and population-retention work | +| [#949](https://github.com/PolicyEngine/microcosm/pull/949) | `native-row-ceilings`, `0319b1c73` | Reviewed native row-count ceilings | +| [#950](https://github.com/PolicyEngine/microcosm/pull/950) | `native-retention-seal`, `98b698593` | Content seals and observer detachment | + +The resume point was `c15661e5a`, on top of `native-release-integration-20260919` +at `47960af43` and the already integrated September 21 descendant branches. +Those completed merges were retained. Byte-transport work (`25d66f3c5`) and +verified-store-byte reuse (`78b1125f4`) were also already integrated. + +## Conflict record + +The earlier #950 merge (`9e9c885bc`) resolved two files: `pyproject.toml` +unions lint exceptions; `graph_atomic_survey_financial.py` combines the existing +compact profile with content seals for the default profile. Declared consumers +retain populations, the host owns their detachment, and the explicit +`_retain_every_node_population` option supports inspection. The detailed record +is in [the retention-seal note](us-native-retention-seal.md). + +The resumed merge of main `d9a248d0a` had 26 conflicted paths. Their previously +resolved text was inspected and preserved; the resolver was not rerun. The +following inventory lists every conflicted path exactly once. Paths in the +tables are relative to the repository root. + +| Paths | Resolution | +| --- | --- | +| `CLAUDE.md`; `docs/shared-constants.md` | Keep independent guidance from both parents. | +| `PROGRESS.md` | Preserve both historical journals and historicize the retention-seal currency claims. | +| `docs/uk-national-calibration-runbook-623.md` | Keep integration's provenance paragraph and main's driver guidance. | +| `packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py` | Preserve the lazy facade while including main's exports; adapt its export-inventory test. | +| `packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py` | Keep the lazy facade and add main's SPM independence-role exports, receipt-input constant, donor entry, and stage order. The merged public export set is the exact parent union: 917 unique names. | +| `packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_us.py`; `packages/microcosm-frame/tests/test_policyengine_us_metadata_index.py` | Retain integration's validated static source-input helpers and uncached declaration lookup; include main's SPM source ownership and aging changes. Keep metadata tests consistent with the declared source input. | +| `packages/microcosm-build/src/microcosm/build/us_runtime/medicaid_take_up.py` | Use main's `state_name` and `_substituted_hierarchy`, preserving the family target label; retain integration's replacement of neighboring-state metadata labels. Adapt the substitution-hierarchy test to that API. | +| `tools/build_us_fiscal_refresh_release.py`; `packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py` | Keep main's SPM composition gate and integration's formula-metadata-aware weight attachment. Move the committed-feed-pin check into shared `_compile_fiscal_release_target_registry`; preserve both test import surfaces and update the placement assertion. | +| `packages/microcosm-graph/src/microcosm/graph/executor.py` | Keep main's observer-detach behavior and documentation, alongside the integration branch's existing exact-float digest optimization. | +| `packages/microcosm-build/tests/test_uk_calibration_run.py`; `packages/microcosm-build/tests/test_us_acs_pums.py` | Union the independent test additions. | +| `tools/ci_test_groups.py`; `pyproject.toml` | Union shared-spec test registration and per-file lint exceptions. | +| `docs/evidence/spec-engine/us-f0-coverage.json`; `packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py` | Carry the merged coverage inventory and pin surface; derive final counts and hashes from the resulting code and resources. | +| `packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml`; `tools/generate_us_bundle_from_constants.py`; `packages/microcosm-build/tests/test_us_bundle_core_contracts.py`; `packages/microcosm-build/tests/test_us_spec_bundle.py` | Include main's source-stage additions and corresponding source manifest, frozen-resource digest, and stage-count assertions. Derive identities from merged resources. | +| `packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py` | Bind the approved-lock digest to the merged lockfile. | +| `packages/microcosm-build/tests/test_spec_engine_loader.py`; `packages/microcosm-build/tests/test_us_multispine_pool_tool.py`; `packages/microcosm-build/tests/test_us_spine_blindness.py` | Reconcile loader and country-spec identities and the reviewed runtime-module inventory; verify the resulting pins and counts against the merged tree. | + +The subsequent merge of newer main `6442030a2` had one conflict, again in +`packages/microcosm-build/tests/test_uk_calibration_run.py`. Both independent +fixtures were retained: integration's `invented_code_pin` and main's +`_fake_cgt_projection`/`_cgt_projection` support. + +`poverty-comparison-only-20260921` at `07b69be68` then merged cleanly. Its +comparison-only boundary remains explicit; poverty evidence does not supply a +population treatment or certify this integration. + +## Observed identity repairs + +The merged field ledger contains 42,239 distinct configuration pointers: +32,458 authored fields and 9,781 resolved bindings. Relative to the previous +42,180-field report, the validated claim changes are transfer execution +52, +producer graph -7, source stages +13, and resolved seed protocol +1. +The producer input inventory is 2,743 and the source-stage inventory is 38. +These are compiler inventories, not population coverage or release acceptance. + +Two independent normal `load_bundle("us")` computations agreed on the spec +identity `217008445ad8c84088d66af45ad441b36e4f1878f4ce15f5de584fa412e8c9b6`. +The loader's separate minimal golden fixture was also re-derived from its +unchanged construction. Counts and hashes were updated from these observations; +no validation condition was removed to accommodate the merged tree. + +The graph battery exposed stale accepted fingerprints in +`us_runtime/acs_native_coverage_binding.py`. Main's `59642af17` (#765) adds +optional `WKHP`/`WKL`/`FWKHP` preservation to `acs_pums.py` and annual-hours +mapping to `acs_inputs.py`. Review confirmed that missing hours remain absent, +invalid supplied codes refuse, and the existing age, relationship, and income +mappings remain intact. Only the two reviewed fingerprints were updated; +unreviewed source changes still refuse before native preparation issuance. + +The later hours-fragment retry exposed a semantic integration conflict. Main's +mapper emits `weekly_hours_worked_before_lsr` when `WKHP` is supplied, while +native graph preparation reserves that column for its cross-survey hours +completion owner. The new receipt also fails the native operator-free boundary. +The mapper now accepts an explicit `include_usual_hours=False` mode: it still +validates the hours, age, past-year-work and allocation codes and retains all raw +observations, but leaves the canonical output and receipt to the completion +owner. Authenticated housing preparation and the raw ACS graph codec use that +mode; the mapper's default behavior remains unchanged. Its reviewed source +fingerprints were refreshed. +The mapper, source-hours, graph-hours and implementation-inventory files passed +243 tests after this correction. The complete ACS PUMS file passed 30 tests, +including genuine archive cases checking graph-loader raw-hour preservation and +invalid-code refusal. None of the three modified producers belongs to the F0 +direct or QRF seed roster; the graph codec's static dependency contract is +unchanged. The measured portable specification identities above remain unchanged. + +The native implementation inventory also required main's `microcosm.frame.scaling` +module. It is included in the exact package roster and every stage that binds +that closed package, with its static dependency contract. The frame facade, +US adapter's static-aging imports, and SPM operator-output registry import have +reviewed dependency contracts. The runtime roster and dependency refusals remain +unchanged. All ten native stage identities change because they bind the inventory +JSON; this JSON is outside the F0 seed and specification hash inputs. + +The spine-blindness inventory had omitted eight integrated native modules. Their +review preserves distinct source qualification, fixture validation, and numerical +execution boundaries: + +| Modules | Reviewed boundary | +| --- | --- | +| `current_survey_household_domains.py` | `_project` authenticates selected origin/clone coordinates for domain projection. | +| `current_survey_other_disability_completion.py` | The qualifier defines original ASEC observations and ACS age-15-plus development-model applicability; `_receiver` checks origin/clone consistency. Unresolved ASEC observations remain unknown. Fitting and attachment remain scanned. | +| `graph_current_survey_other_disability_completion.py`; `graph_us_other_disability_host.py` | Fixed report-column and typed artifact/population selectors; explicit provenance reads remain guarded. | +| `graph_native_puf_tail_expand.py` | `provenance_columns`, `_validated_tables`, and `native_puf_tail_expand_nodes` validate invented fixture origins and declare columns. They do not authenticate actual source data; execution and weight splitting remain scanned. | +| `native_puf_tail_matching.py` | `_frame_inputs` validates fixture consistency; ranking remains scanned. | +| `native_puf_tail.py`; `survey_social_security_beneficiaries.py` | Invented donor selection/thinning and report-lot accounting respectively; both scan clean without exceptions. | + +The new guard recognizes only the listed top-level function bodies. Defaults, +decorators, other declarations, and unlisted functions remain scanned. Regression +cases reject renamed boundaries and injected provenance reads. The pool tool's +import graph was re-derived at 77 modules: its previous 73-module assertion was +already stale against the resume tree's 74, and main adds three SPM modules. + +Five existing modules also required reviewed dynamic-selector entries: ASEC CSV +byte/token indexing, retained immigration state tuples and fixed lineage keys, +state-graph artifact maps, and validated SNAP state/role estimate maps. Each new +entry is covered by injected-provenance regressions; none receives a module-wide +source-provenance exemption. + +The complete scan exposed unnecessary Cartesian expansion of dictionary values +while resolving dictionary iteration keys. Dependency selection now includes +ordinary keys and unpacked mappings for a root dictionary, while other expression +roots retain complete traversal. Dictionary values are still scanned separately. +A deterministic call-count regression covers the former `25**5` case; explicit +key, value, unpacking, and formatting cases verify continued provenance detection. + +## Enrichment population replay direction + +The first complete disability-host fixture reached the inherited population +comparison and refused `SURVEY_POPULATION_REPLAY_NONCANONICAL_NULL_BACKING`. +The replay comparator deliberately permits ContentStore normalization only on +its actual side. Review found two different comparison contexts that require +explicit ordering: + +- Inherited observer populations are cached materializations of a retained + parent. Both enrichment and disability continuation now use + `_compare_inherited_manifest_population`, with the retained parent manifest + as expected and the observed cached population as actual. +- Newly reconstructed expected values precede the current execution manifest. + Disability's `_compare_manifest_population` keeps that direction for its + final output checks, matching the existing enrichment output comparison. + +The strict comparator and the separate checks of full predecessor ownership, +design weights and custody remain unchanged. Real ContentStore roundtrips for +nullable integer and boolean columns reproduced the inherited-direction refusal +before correction. The complete population-context file then passed 24 tests +across both contexts, including refusal of changed present values, masks and +noncanonical backing data. The separate replay file passed 248 tests. The exact +column from the original long fixture was unavailable after temporary-store +cleanup and is not inferred from these small reproductions. No reviewed source +pin, F0 seed hash or implementation inventory contract required adjustment; +enrichment's normal runtime source hash binds the changed implementation. + +## Enrichment fixture member pins and default amount groups + +The survey-enrichment module fixture refused before geography with +`DEMOGRAPHIC_SOURCE_CONTRACT`. The wrapped refusal was `STUDENT_SOURCE_SIZE` +from the demographic owner's member snapshot. The fixture pins the coverage, +restoration and demographic owners to its rewritten 2022-2024 ASEC person +members. `add_hours_source_fields` then rewrites those members again but +re-pins only the coverage and restoration owners. This did not come from main +or #991. The same refusal reproduces on the pre-merge native tree `c15661e5a`. +It also reproduces at `ac15e9fb9`, the native commit that added the hours +helper to this fixture. That commit's parent reconstructs the demographic +source successfully. After the hours helper, the fixture now re-pins the +demographic owner to the final roster, as the immigration enrichment fixture +already does. A new fixture test fails without that line; it checks the +coverage pins against the final member bytes and that the restoration and +demographic owners match them. The other owners that pin these members +(for example `asec_student_controls`) are not covered by that test. + +Clearing that refusal exposed two assertions that the native line's +20 September opt-in amount routes had made stale. Those routes are workers +compensation, child support and veterans benefits. The assertions enumerated +every declared `values.GROUPS` entry, but the default call runs only +`unemployment` and `health_costs`, as `us-current-survey-amount-successor.md` +documents. A diagnostic run of the unchanged assertions found that only the +three opt-in outputs were absent from the population. They also remained +missing inputs. The node count (281) and input inventory (161) matched their +existing assertions. The tests now read the executed groups from the run +receipt and assert that default. They also require the unselected opt-in +outputs to stay absent and missing. No production module, source pin, +implementation inventory or F0 identity changed. The complete file passed +12 tests from a cold store in 1,054 seconds with an 0.87 GB maximum resident +set. + +## Immigration pair producer-key control + +The immigration enrichment file's +`test_genuine_pair_artifact_and_producer_controls` expected +`ARTIFACT_PRODUCER_KEY` after replacing only the retained pair artifact's +`producer_key`. The enrichment host's `context` passes each typed input through +the shared `graph_puf_detail_transfer.artifact` check before comparing +producers. That check requires `key == opaque_artifact_key(producer_key, +artifact)`. `ArtifactValue` stores `key` and `producer_key` as separate fields, +so the mutation always refused there with `DETAIL_ARTIFACT_IDENTITY`. + +The merge did not change the check order, and producer detection still works. +The test, the host's `context`, the shared check, `ArtifactValue` and +`opaque_artifact_key` are byte-identical on the native parent `c15661e5a` and +the merge. The assertion arrived with native commit `d33d71bee`, which is on +neither main nor poverty-comparison-only. Its check order was already the +current one. The survey-enrichment host test has re-derived the store key for +its producer-key case since `091261c0e`. + +The test now covers both refusals separately. The unmodified retained inputs +must pass first. A producer-key-only mutation must then refuse with exactly +`DETAIL_ARTIFACT_IDENTITY`. A foreign producer key whose store key is re-derived +from it passes that identity check and must refuse with exactly +`CURRENT_SURVEY_AMOUNTS_ARTIFACT_PRODUCER_KEY`. The two codes differ in form +because the two checks use different `require` helpers. The host module binds +`require = values.require`, which is `current_survey_amounts.require` and +prefixes every host refusal with `CURRENT_SURVEY_AMOUNTS_`. The shared check +binds `puf_detail_transfer.require`, which raises the bare reason. Both helpers +were already bound this way at `d33d71bee`. The sibling survey-enrichment test +matches the unanchored substring `ARTIFACT_PRODUCER_KEY`; this test anchors both +codes so that neither refusal can satisfy the other's assertion. No production +module, source pin, implementation inventory or F0 identity changed. + +## Merge of main `2bce66156` (26 September) + +This merge brings in the 19 main merges since `6442030a2`: + +- US release: the Route A remediation stack, PR-1 to PR-5 and a follow-up + (#1016, #1018, #1017, #1025, #1024, #1028). Also the tips return-count + exclusion at every vintage (#1029, d179), the national + state target + surface (#1005) and its source coverage (#1008), and the QRF tail-register + batching (#1004). Also the capital-gains tail clone on a PUF-support base + (#992), the stored-input contract (#1031, #1026) and the publish-now, + flip-later tag flow (#1015). +- US inputs and grading: the PUF imputation no longer predicts income from + itself (#1033, #982), and eCPS parity grades under the live engine names + (#994). +- UK: bus fares from the NTS (#954), per-line release pointers and the + 2024-25 national default (#966), and income anchors at the calibration year + (#1006). +- Shared: each packaged country spec loads once per process (#1010). + +It had eight conflicted paths. The earlier resolutions recorded above were not revisited; +each file below is listed once. + +| Paths | Resolution | +| --- | --- | +| `.github/workflows/test.yml` | Keep the native sharded matrix and diagnostic job. Main's only change since `6442030a2` adds `--durations=25` to the four pytest calls; the native text already carries it in each. | +| `packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py` | Keep the lazy facade. Add main's three fiscal-target exports (`US_FISCAL_TARGET_ALL_VINTAGE_SUPPORT_EXCLUSIONS`, `US_FISCAL_TARGET_EXCLUSION_VINTAGE_BYPASSES`, `us_fiscal_target_exclusion_receipt`) to the `fiscal_targets` export map; `__all__` merged cleanly. The public export set is the parent union: 920 names. | +| `packages/microcosm-build/src/microcosm/build/us_runtime/reform_validation.py` | Keep main's `_released_engine_state` helper and its two callers, which collect only after releasing an object that held engine state. Keep the native `default_simulate_factory` signature with explicit dataset and microsimulation constructors and SPM selection. | +| `tools/build_us_fiscal_refresh_release.py` | Carry the native explicit-consumer seams onto main's Route A structures; details below. | +| `packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py`; `docs/evidence/spec-engine/us-f0-coverage.json` | Re-derive from the merged tree (below). | +| `packages/microcosm-build/tests/test_spec_engine_loader.py`; `packages/microcosm-build/tests/test_us_multispine_pool_tool.py` | Re-derive the golden and live spec identities from the merged tree. | + +One further file had no textual conflict but pinned the facade size: +`packages/microcosm-build/tests/test_us_runtime_facade_union.py` now asserts +920 names and the union digest +`5b9df20773d016ae9cb5b8f9b3669e373ec1fe57a5b6ea2e66e4197e32a08b43`. Each +parent's `__all__` has 917 names, and their union is 920; the merged +facade's digest equals the digest of that union. + +### Fiscal release tool + +Main's Route A PR-4 (#1025, `f1fa9f465`) replaced the frame-based reform-validation +factory, `_batched_reform_validation_simulate_factory_from_frame`, with +`_HouseholdBatchedPostExportScorer`. The scorer reads the written H5 once, +binds its digest, scores each consumer in household batches, and records a +`post_export_scoring` block in both manifests. The native line (`0b476f984`, +`54f224f2f`) had added explicit consumer seams to the old factory: formula +metadata, a dataset constructor, a microsimulation constructor and an SPM +selection. The merge carries them onto the scorer: + +- The scorer accepts `formula_metadata`, `dataset_cls` and `spm` beside its + existing `microsimulation_cls`, `dataset_from_frame` and `load_frame`. + Supplied metadata checks the whole written frame before any batch dataset or + engine exists. `dataset_from_frame` and `dataset_cls` together refuse. +- An explicit SPM selection is copied once at construction. Every batch engine + and every reform system receives its own copy, and the manifest block records + the selection actually used. +- With no explicit seam, every engine, dataset and reform-system construction + keeps main's call shape: each engine declares `US_RELEASE_SPM_SELECTION`, + and a reform system is built from `reform=` alone. Some internal helper calls + pass the omitted seams on as explicit `None` keywords, which change no + behavior. `_main` supplies no seam. +- `_score_post_export_consumer`, `_write_reform_validation` and + `_write_demographics` forward explicit seams only to a scorer they open + themselves. An existing scorer refuses them. + +Main's Route A PR-5 (#1024, `a13324e90`) batched the target materializer's base +simulation. The native seams now reach each base batch as well as each JCT +reform batch. PR-2 (#1018, `c1b0a3204`) made a build commit mandatory with a +target-frame checkpoint path; two native checkpoint tests now supply one. The +native shared helpers remain: `_main` calls +`_compile_fiscal_release_target_registry` once, and it solves the dense and +L0 arms through `_calibrate_fiscal_support`. Exact-k keeps its ladder path. Main's exclusion receipt (Route A PR-1, +#1016, `8b228eccd`, written under the manifest receipt key by #1017) stays in +`_main`, and still replays the facts, period and crosswalk that compiler +used. The native survey handoff, the ACA frame-context fix and the +private calibration attachment bridge are unchanged. + +Three native test files that addressed the removed factory now address the +scorer. They keep their assertions: explicit seams reach every batch dataset, +engine and reform system, captured selections survive caller mutation, and +supplied metadata refuses before any engine work. +`test_us_fiscal_consumer_propagation.py` adds a refusal test for overrides on +an existing scorer, and a test that omitting every seam preserves the +release's calls and manifest selection. `test_us_fiscal_formula_metadata.py` +now expects four single-household batch datasets, two base and two reform, +where the unbatched base used one. `test_us_fiscal_refresh_builder.py`'s +receipt-replay test follows the compile into the shared helper. It asserts +that `_main` has no direct compile call, passes the crosswalk, and takes back +the ledger artifact whose facts were compiled. + +### Identity repairs for this merge + +Every value below was observed on the merged tree; none was hand-derived. +`tools/generate_us_bundle_from_constants.py --check` passes: the committed US +bundle equals the bundle generated from the merged constants. Its identity is +`2dfa51b89e740c742437168cb07f3084a2ecd7f90ad61eb13d3b12b69f315e15`, which +replaces both parents' values in the pool tool's live-constants assertion. +Before the repair, four inventory items failed: +primary predictor tuples, producer resource semantics, seed protocol and owner +map digests, and the stacked checkpoint base identity. Their observed values: + +| `EXPECTED_HASHES` key | Native | Main | Merged | +| --- | --- | --- | --- | +| `primary_tuples` | `987b501c…` | `fdf23da4…` | `fdf23da4…` (main's #1033 predictor block; the native line did not change it) | +| `late_resource_semantics` | `6c4f4ec0…` | `6abbfad7…` | `c0785687…` | +| `seed_protocol` | `b1797d9b…` | `5f6d3b52…` | `6d673c33…` | +| `seed_map` | `e04c4f86…` | `dad44808…` | `acab6b8d…` | +| `full_checkpoint` | `cfa87759…` | `b1d5c652…` | `2a1ff0f7…` | + +`late_schedule`, `source_manifest`, `authority` and `graph_nodes` keep the +native values, because main left them at the base values. After the repair, +`tools/spec_engine_coverage.py` reports 42,239/42,239 configuration fields and +41/41 inventory checks. The field count equals the previous integration's. +The loader's minimal golden vector binds the seed protocol implementation +digest, so it moves with it: +`f2047cb96d0d063f40c2d5eb5afa2f3821e5667e49968268d14829bc5e09ab5a`. + +## Failures fixed after the `2bce66156` merge + +CI on the earlier head and on the merge exposed failures beyond the two +residuals above. Each was reproduced or diagnosed before the fix, and none +relaxes a validation condition. + +| Failure | Cause | Fix | +| --- | --- | --- | +| `test_spec_seed_identity_lock.py` (LOCK) | The seed diagnostic still pinned the native `uv.lock` of `e47adc6db`; `worker_identity.py` already bound the current lock. | Re-pin `LOCK_SHA256` to the committed lock's digest. | +| 3.13 lanes: `test_us_child_property_verification_operation.py`, `test_us_current_survey_household_domains.py` | Their `sys.monitoring` observers called `clear_tool_id`, which exists only from 3.14, before `free_tool_id`, leaving the tool id claimed. | Zero local events, unregister the callback, check nothing remains, free the id (the pattern `test_us_graph_survey_puf55.py` already used). | +| `test_us_hours_worked.py`, `test_us_acs_local_release_tool.py` | The native summary gate required all three hours columns; main's #941 pool/ACS gate counts only the two it carries. | `us_hours_worked_gate_from_summary` takes the same `required_columns`, with the strict three-column default. A property test (skipped where hypothesis is absent) checks that the scope changes only which counted columns are required. | +| `test_us_plan.py` | Native `071f563b1` made the legacy immigration stage require `A_LFSR`, which no base-stage provider supplied, and left the plan's Pew citation at the 2024 report. | `A_LFSR` joins main's #720 Census person restoration as a separately declared post-#720 column. Its codes {0,1,2,3,4,7} were observed complete in all three pinned members. The donor citation now matches the stage's 2025 report. | +| `test_us_asec_census_person_columns.py` | The native health projection names `NOW_CAID`, recorded as "no build reader". | `NOW_CAID` carries a reason of its own. It is read only from the native projection's own pinned member, never the pooled H5, and a test fails if any other module names it. | +| `test_us_multispine_pool.py` | The ported #779 paired immigration transfer needs ACS `CIT`/`POBP`/`YOEP` on every ACS row; the port omitted #779's ACS loader carriage. | `acs_pums` reads the three as optional person columns, as #765 does for hours. Its reviewed fingerprint is re-pinned. The pool fixture carries them, and the transfer has #779's 35 predictors. | +| `test_us_population_input_coverage.py` | The native profile froze main's coverage manifest at 163 inputs; main now requires 167 (#959 role, #978 receipts). | Re-extract the profile in manifest order, re-pin the manifest digest, and version the identifiers (`us_release_167_v1`, `us_national_cd_165_v1`, `us_native_national_cd_163_v1`). | +| 3.14 lanes: PRODUCER_CHANGED | On Python 3.14 a class's `__annotate__` function closes over the live class namespace (the `__classdict__` cell), and the producer seal records closure cells by value. `test_us_native_calibration_attachment.py` deep-copies a mass log, and copying caches `__slotnames__` in `MassChangeRecord`'s namespace (`copyreg._slotnames`). That was the only moved entry. | Seal that cell by identity and by its contents minus `__slotnames__` only, so every other class attribute, dataclass fields included, stays sealed. The regression test requires a copy to pass, and an in-place field edit or a replaced namespace to refuse. It fails under both the original and an identity-only rule. | +| Wheels: UK seam tests | Main's newer seam tests drove the real `run_uk_calibration`, which pins a git commit, without the native invented pin. | They request the invented pin. Production stays strict. | +| Wheels: donor-receipt producer identity | The native sharded wheels group newly runs this checkout-bound check against installed shards. | Its success and foreign-module tests skip only when the producer's modules are not this checkout's files. | +| `test_us_fiscal_refresh_builder.py`, `test_us_fiscal_targets.py` | Main tests counted three solve calls in `_main` and main's pinned-feed target counts. | Count the shared solve helper. Main's counts plus exactly the 16 native #958 Table 1.1 size-of-AGI rows, observed by their rebase flag. Every Route A fence assertion holds. | + + +## Local Git recovery and validation record + +The sandbox could read the original linked-worktree Git administration but could +not write it. The lane prepared Git metadata under `.integration-resume/git` +inside its assigned workspace, retaining the original head and pending merge +state. Subsequent Git mutations use that lane-local metadata. The original +administration and other worktrees were left untouched. + +Final integration commit identifiers, targeted test results, and push/PR status +will be recorded by the lane owner after validation. Earlier journal test results +are historical and are not evidence that the final merged tree passed. No +actual-data native run, benchmark, certification, upload, or publication was +performed as part of this conflict documentation. diff --git a/docs/native-primary-family-source.md b/docs/native-primary-family-source.md new file mode 100644 index 000000000..682b70078 --- /dev/null +++ b/docs/native-primary-family-source.md @@ -0,0 +1,89 @@ +# Native source household classification + +`current_survey_primary_family.classify_primary_household` classifies one complete +original household. `current_survey_primary_family_source.qualify_current_survey_primary_family` +borrows the existing authenticated survey preparation and householder projection, +reads the pinned original members, and returns a detached household table and +descriptive receipt. It grants no source admission, population admission, graph +attachment, calibration, or release eligibility. Future graph integration must +requalify against the live source after relevant I/O and use the maintained exact +householder/source-to-clone transport. + +## Definitions and official source mappings + +The [2024 ACS subject definitions](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf) +define own children as never-married biological, adopted, or stepchildren of the +householder under 18 (printed page 89). A household has at most one primary +family, and its married-couple classification requires the householder and spouse +to reside together (page 90). A related subfamily does not add another family to +that count (page 91). This classifier follows that household concept, rather than +the broader employment-tabulation child universe. The spouse relationship includes +both opposite-sex and same-sex couples. + +The [2024 ACS PUMS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.txt) +maps `RELSHIPP` 20 to the householder, 21/23 to the spouse, and 25/26/27 to +biological/adopted/stepchildren. `MAR=5` establishes never-married status; +`AGEP` supplies age. `TYPEHUGQ=1` identifies housing units and 2/3 group quarters. +`NP` must match the complete retained roster. Spouse and unmarried-partner +relationships appearing together are unresolved. ACS does not publish the general +resident spouse-line pointer needed here to count secondary couples; that result +is explicitly unavailable, even when the primary couple is known. + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +maps `A_EXPRRP` 1/2 to the reference person, 3/4 to the spouse, and 5 to own +children (printed page 6C-2, PDF page 23). Code 6 is unnamed and stays unresolved. `A_MARITL=7` +supplies never-married status; `A_AGE` supplies age. `A_SPOUSE` is a line number +(page 6C-1), resolved reciprocally within `PH_SEQ` using `A_LINENO`. Missing, +out-of-household, self, asymmetric, and marital-inconsistent pointers stay unknown. +All valid resident pairs are counted separately from the primary family. + +ASEC has no original `TYPEHUGQ` field. `H_LIVQRT` 1–7 identifies housing units; +8–12 identifies other units (page 6A-2). `HRHTYPE` 9/10 denotes group quarters +(page 6A-3); disagreement between those classifications is unavailable. The +classifier also requires an interview (`H_HHTYPE=1`), `H_NUMPER` equal to roster +size, and the independently qualified reference person. Household-type marriage +codes are a consistency check, never a substitute for spouse evidence. + +The [CPS subject definitions](https://www.census.gov/programs-surveys/cps/technical-documentation/subject-definitions.html) +also require never-married status for own children under 18. CPS published counts +may include college-dorm residents associated with a family; this projection +describes retained resident household rosters and does not claim equality with +every CPS published family universe. + +## Outputs and unknowns + +The household table carries nullable integer columns: + +- `census_household_count` and `census_household_population`; +- `census_married_primary_family`; +- `census_married_primary_family_own_child_u18` and + `census_married_primary_family_own_child_u6` (at least one own child under 6); +- `census_primary_married_own_children_u18_count`; +- `census_household_size_1` through `_6`, and `_7_plus`; +- `census_all_resident_spouse_pairs`, a separately qualified ASEC measure. + +Separate household, primary-family, own-child, and spouse-pair status columns +distinguish qualified values, exclusions, and missing or inconsistent evidence. +Known nonhousing units contribute zero to household-family/size indicators; +their pair count is unavailable. Unresolved universe evidence produces unknown +metrics. A known housing unit with an incomplete roster retains a household count +of one, but has unknown size/family metrics and an explicit incomplete-roster +status. Size bins partition the complete qualified housing-unit cohort only. + +For a known nonmarried primary household, married-family child measures are zero. +For a married primary family, missing age or marital status of a potential own +child remains unknown. Grandchildren, in-laws, foster children, and unrelated +children do not become own children through age, tax dependency, SPM membership, +or co-residence. Native identifiers are handled as literal strings or exact Python +integers without float conversion. + +The receipt records exact readsets, source and householder receipt digests, +source-year origins, output digest, and status counts. Published values may include +Census allocation; no unallocated-observation claim is made. Original source +mutation invalidates the existing preparation, and the qualifier reconstructs +all metrics/statuses after its final owner/source I/O. Returned tables are +detached values; their presence is not retained live authority. + +This change adds no targets, tolerances, weighted estimates, status draws, or graph +nodes. It does not establish calibrated household composition or statistical +acceptance against a published table. diff --git a/docs/native-puf-tail-fixture-contract.md b/docs/native-puf-tail-fixture-contract.md new file mode 100644 index 000000000..cb11b3adb --- /dev/null +++ b/docs/native-puf-tail-fixture-contract.md @@ -0,0 +1,159 @@ +# Native PUF tail fixture contract + +`microcosm.build.us_runtime.native_puf_tail` implements the first pure slice of +native tail support. It accepts explicitly invented role projections only. It +does not admit actual PUF source rows, choose a modeled person allocation, +transfer values into a population, assign geography or qualify a release. + +The donor selector adapts the union-arm and AGI-only thinning algebra from +[`puf_agi_tail.py` at PR #964 commit f7df78b2a00421f9b90305a9b7db192444075ae0](https://github.com/PolicyEngine/microcosm/blob/f7df78b2a00421f9b90305a9b7db192444075ae0/packages/microcosm-build/src/microcosm/build/us_runtime/puf_agi_tail.py). +This attribution is an immutable algorithm reference, not a claim that the +native implementation has qualified equivalent actual source projections. It +participates in the selection identity deliberately: an edit to the reference +this algebra came from is a change of provenance and must move the digest. +Two behaviours deliberately diverge from that reference and are marked below: +the residual placement in exact cell rescaling, and the arithmetic that decides +the AGI floor. + +## APIs and ownership + +- `native_puf_tail_profile()` derives 52 outputs from `PUF55_SURVEY_SS` and the + existing three-field late-owner matrix. It preserves independent survey Social + Security and detailed SCF mortgage ownership. It does not use #964's wider + full65-minus-three projection. +- `declare_puf_tail_role_projection(...)` snapshots typed return/person cells, + an all-known witness and invented head/spouse/dependent roles into immutable + bytes. It requires physical person booleans, exact int64 identifiers, finite + numeric amounts stored at exactly int64 or float64, and float64 return DESIGN + weights. Unknown cells refuse; omitted cells do not become zero. Real + source/model provenance modes refuse. +- `projection_tables(...)` revalidates the complete snapshot before returning + detached tables. Public dataclasses and hashes are descriptive; neither is a + source-authority capability. A caller may declare invented values, never use + this contract to authenticate observed people. +- `select_native_puf_tail(...)` accepts two aligned, explicitly declared fixture + masks for capital-gains support and eligibility. It binds both masks and the + projection in a deterministic selection identity. It does not derive or + qualify the capital-gains quantiles. + +## Admitted cells + +Filing-status codes come from the repository's existing PUF filing-status +authority, `puf_support._FILING_STATUS_CODES`, whose integer-keyed inverse view +`puf_capital_gains_tail._FILING_STATUS_BY_CODE` carries the same five entries. +All five are admitted, including `5 = SURVIVING_SPOUSE`; this contract does not +maintain a second local enumeration. The admitted domain appears in the receipt. +It is a validation gate rather than a selection parameter -- an out-of-domain +code refuses the whole declaration -- so it is not folded into the selection +identity, whose filing-status content already travels in the projection digest. + +Money is stored at exactly int64 or float64. Narrower floats, narrower or +unsigned integers and non-native byte order refuse at declaration and again at +snapshot restore, rather than being reinterpreted at a precision this contract +has not demonstrated. Knownness is all-known only: a partially known +declaration refuses, so the knownness bytes are a constant all-true witness +that carries no information beyond the row count. Admitting partial knownness +would require threading the caller's real mask through the snapshot first; the +constant must not be read as measured knownness in the meantime. + +## The AGI arm + +The AGI arm uses an inclusive $5 million threshold on the declared 20-component +signed additive proxy. This is not calculated AGI. That 20-leaf list is the +donor-side build-period income locator and matches the reference selector's own +donor proxy; the seven-column `_RECIPIENT_AGI_PROXY_COLUMNS` in the sibling +`puf_capital_gains_tail` is a recipient-side SOI-band locator for a different +operator. Neither list substitutes for the other. + +The floor decision does not route through a float64 total, which is the second +deliberate divergence from the reference. Each row's int64 components are +summed as Python integers, exactly at any magnitude; its float64 components are +summed with `math.fsum`, which returns the correctly rounded sum of the doubles +it was given -- a stable summation, not universally exact real arithmetic. The +decision then compares that float part against the exact integer remainder of +the floor, which is an exact comparison, so an int64 component above 2**53 +cannot cross the floor by rounding. The float64 proxy total that the receipt +masses and the rank-decile order use is a diagnostic view of the same row; it +is labelled as such in the receipt and is not the threshold authority. + +AGI donors require exactly one head, at most one spouse, representable roles and +matching person monetary sums. Those person sums are reconciled over Python +scalars, so signed integer reconciliation cannot wrap or round; they are also +person-row-order independent, because an eligible donor has exactly one head, +at most one spouse and exactly zero dependent money, leaving at most two nonzero +values in any column. Any nonzero dependent money skips that donor, including a +donor in both arms; it is not folded into the head. Dependent Boolean flags are +explicitly ignored, matching the reference rule. + +`SelectedTailDonor.spouse_payload_nonzero` describes the declared cells and +nothing else: it reports whether any declared spouse cell of a role-screened +donor is nonzero. It is `None` for capital-gains-only donors, which are outside +the role-eligibility scope and were never examined -- unexamined is not the same +as examined and found zero. It never consults `filing_status_code`, and a joint +return with an all-zero spouse row is not a finding that the return needs no +spouse. Which household a donor becomes, and whether that household has a +spouse, remains entirely the later population owner's decision. + +## Arms, thinning and weights + +Capital-gains-only and both-arm donors remain unthinned and uncapped. AGI-only +support is capped at 3,000, using deterministic filing-status/rank-decile cells +and midpoint selection. Cell quotas assert their own postconditions -- the +quotas sum to the budget, every cell keeps at least one donor, and no cell keeps +more than it had. Those are inexpensive invariants; no counterexample to them is +known in this implementation. + +Each thinned cell preserves its float64 return-weight sum exactly. The residual +correction is applied to a largest weight in the cell, which is the first +deliberate divergence from the reference: the reference forces the residual onto +the last row, and a cell with dispersed weights can have a last row orders of +magnitude smaller than the correction, which either drives that weight +non-positive or leaves the correction unable to move the sum at all. Positivity, +finiteness and exact conservation stay mandatory; a cell mass that still cannot +be represented refuses rather than being approximated. + +Receipts report changes in proxy amount sums and weighted proxy sums; amount +mass is not asserted to be conserved, and those totals are float64 diagnostics, +not an exact amount reconciliation. Selected weights are IMPORTANCE support +weights at donor-return grain. They are not household population weights. +Conditional support selection does not retain DESIGN population semantics even +for a numerically unchanged weight, so every selected weight is IMPORTANCE; +each donor additionally carries a `weight_treatment` recording whether its +value was left unchanged or rescaled inside a thinned cell, and the receipt +totals both. A later owner reconciling design mass must read that field rather +than assuming the whole selection was resampled. + +Role/monetary eligibility applies to AGI-only and both-arm donors. CG-only +selection inherits the declared mask without claiming full52 person fidelity; +its distinct transfer semantics remain a later-owner responsibility. The receipt +reconciles every arm: AGI candidates equal selected plus skipped plus thinned, +capital-gains candidates equal selected plus skipped, and both-arm candidates +equal selected plus skipped. A capital-gains donor can only be skipped when it +is also an AGI candidate, because only the AGI arms are role-screened. + +Later population integration must resolve the scientific units and source +cohort behind donor-return-to-household normalization, and conserve host +household IMPORTANCE mass. No normalization formula is admitted by this slice. + +The current contract bounds declarations to 10,000 invented returns and +100,000 invented persons. Those two bounds, the 3,000 AGI-only cap and the +inclusive 5,000,000 floor are declared development bounds: this slice states +them, and does not derive them. A source run must establish their basis +separately. They do not authorize an actual source run. +The receipts always state `source_admission_issued=false` and +`release_eligible=false`. Return/person values remain local fixture evidence. + +## Qualification still required + +Actual source-to-role ownership, stable source/growth identities, a reviewed +person-allocation method, structural EXPAND, full-spine geography placement, +late-owner replay and actual qualification remain separate work. The native +return-only donor envelope and its technical person rows are not observed PUF +people. Do not relabel them or import the legacy wrapper's rows as authority. + +Administrative AGI distributions may supply independently admitted calibration +targets. Reform scores are diagnostics, not an optimization objective for this +selector. Survey poverty is comparison-only: do not tune weights, imputations, +take-up, candidate selection or target deferrals toward a survey poverty rate, +and do not gate release on proximity to that rate. Independent definition, +implementation and administrative-target correctness checks still apply. diff --git a/docs/native-puf-tail-invented-matching.md b/docs/native-puf-tail-invented-matching.md new file mode 100644 index 000000000..f1a919618 --- /dev/null +++ b/docs/native-puf-tail-invented-matching.md @@ -0,0 +1,73 @@ +# Invented native AGI tail matching + +`us_runtime.native_puf_tail_matching.match_invented_native_puf_tail` connects the +tested native selection algebra to a unique household assignment and the real +structural EXPAND declaration. It is a development contract over invented +inputs. It does not authenticate an actual PUF person allocation, housing +domain, recipient observation, or receiving host. + +The API calls `select_native_puf_tail` itself. It accepts AGI-selected and +overlap-selected donors; any selected capital-gains-only donor refuses the +whole request. Source-adjusted AGI is a separately named input for exactly the +selected donor IDs. It is not the selection proxy. Donor and recipient inputs +declare equal annual USD income-year and price-year bases. This equality does +not establish observed equivalence of survey reporting windows. + +Every clone1 person has an explicit known role, age and all seven maintained +recipient proxy components. Every clone1 tax unit has a known filing status. +The complete household-domain roster must agree with live source coordinates. +All six entity provenance tables and five memberships are checked, including +household closure. Missing or contradictory required evidence refuses; no +unknown value is converted to zero. The source/domain/status/role declarations +remain invented even when these checks pass. + +Candidate occupied-housing parents must have one tax unit, exactly one head, +at most one spouse, eligible head/spouse ages, and positive household +IMPORTANCE mass with exactly representable equal halves. GQ, zero mass, +incompatible roles and unrepresentable halves have named exclusions. Extra +stored non-household weights refuse. + +The head/spouse age floor of 15 deliberately follows the reference matching +method and reuses the ACS earnings-universe constant. This is a support +compatibility choice, not a tax-unit definition or evidence of observed roles. + +Matching follows the reference implementation at commit +`f7df78b2a00421f9b90305a9b7db192444075ae0`: stable source-coordinate order before +seeded priorities, donors requiring a spouse first, donor-ID order, the same +filing status, then the nearest AGI band with lower-band tie breaking. Here +"requiring a spouse" means the tested declared projection has a nonzero spouse +payload; it does not imply observed household composition. Head-only payloads +can match a recipient with or without a spouse. No dependent-person amounts +are transferred by this API. + +Capacity counts unique compatible parents, including spouse-specific demand. +An insufficient filing-status group is left wholly unassigned and recorded. +Another independently supported status can proceed. No parent is reused and +no cross-status fallback exists. An empty result has no EXPAND declaration. + +The reference implementation's absolute donor-weight/household-weight +constraint is deliberately absent. The approved support-expansion policy +gives clone1 and clone2 one half each of the selected parent's household +IMPORTANCE weight. Donor support weights remain separately typed selection +metadata. They do not become household mass, and no claim of legacy #964 +weighting equivalence is made. + +The immutable receipt binds the selection, declared source AGI, comparison +basis, recipient inputs, provenance/membership/weight snapshot, seed, +candidate/exclusion lists, capacity counts, role coordinates, assignments and +structural declaration. It is descriptive: `source_admission_issued`, +`matching_qualified` and `release_eligible` are false. The matching receipt is +not yet an issued graph artifact or replay capability. The EXPAND independently +checks its live structural inputs but does not validate this matching receipt. +The role coordinates name clone1 parent records; newly minted clone2 IDs must +be obtained from the actual EXPAND lineage before any later monetary placement. + +Invented tests cover selection, matching, refusals, capacity, role handling, +determinism and cold/required execution of the real six-entity EXPAND. +Test source alone is not a passing result; execution evidence is recorded +separately. Actual +person-projection qualification, actual housing-domain authority, native52 +amount placement, variable-clone receiving joins and a pre-geography host +remain separate work. Atomic geography belongs after the completed spine. +Administrative AGI support informs this method; poverty is comparison-only and +does not choose assignments, weights, exclusions or release decisions. diff --git a/docs/native-puf-tail-structural-expansion.md b/docs/native-puf-tail-structural-expansion.md new file mode 100644 index 000000000..f1b9bfd53 --- /dev/null +++ b/docs/native-puf-tail-structural-expansion.md @@ -0,0 +1,102 @@ +# Invented native tail structural expansion + +`microcosm.build.us_runtime.graph_native_puf_tail_expand` implements a real graph +EXPAND for a bounded invented fixture. It copies selected whole clone1 +households to clone2, splitting each selected parent's household IMPORTANCE +weight equally between the parent and copy. It does not place donor money. + +The fixed policy is `native_tail_matched_parent_half_importance_v1`, with alpha +`(1, 2)`. Donor support weights remain descriptive donor-selection metadata; +they never become household weights. This is a support-expansion convention, +not a PUF return-to-household conversion or a reproduction of PR #964's +weighting. Alpha is fixed prospectively and is not a tuning parameter. + +## Declaration and graph API + +`InventedHouseholdDomain` describes each household's domain, statistical unit +and source-origin coordinates. Supply exactly one row for every incoming +household, including unselected households and group quarters. Domain units +come from `survey_population_domains.declaration`; the input cannot infer +housing status from a household ID or group size. Repeated origin coordinates +must have consistent domains across clones. + +`InventedTailAssignment` supplies a unique donor ID, a unique selected parent +household ID and a finite positive donor support weight. The pairs are already +chosen: no filing-status, AGI-band, head/spouse or modeled person-allocation +qualification occurs in this API. `declare_invented_tail_expansion` validates +and snapshots these records in canonical order. The immutable result and its +digest establish descriptive consistency, not source authority. It always +reports `source_admission_issued=False` and `release_eligible=False`. + +`native_puf_tail_expand_nodes(columns, base=..., declaration=...)` returns an +EXPAND and a same-version claimant for the six clone-index columns. Register +them with `register_native_puf_tail_expand_kernels`. The incoming population +has the US schema, IMPORTANCE household weights, and clone indices 0/1. This +contract does not require or invoke a legacy exact-two-clone validator. + +The kernel reads the four maintained provenance columns for all six entities +through declared slices. The executor supplies their structural IDs and all +five person memberships. The complete domain roster must match the actual +household IDs and origin coordinates in that context. These are **fixture +declarations**; matching the context does not authenticate an actual survey. + +## Structural and weight invariants + +Only declared occupied-housing clone1 parents can be selected. Each must have +one tax unit and all groups must be household-closed. Every group row must have +members, and their source channel and clone index must agree. Unknown domains, +duplicate donors or parents, conflicting origins, group quarters and clone0 +selection refuse. Existing clone2 rows also refuse; this is one expansion. + +For every selected positive parent weight `W`, both descendants receive +`W / 2`. Halves must be finite, positive and sum to exactly `W` in float64. +Unrepresentable submitted splits refuse; there is no epsilon floor, clipping +or redistribution. Selection-level exclusion is a later contract. Clone0, +unselected clone1 and unselected zero-weight/GQ records keep their weights. + +The kernel returns six new-ID-to-parent-ID lineage Series, six clone-index +overlays, and the full typed household weight vector. IDs are deterministic, +collision-free int64 values above each entity's current maximum; exhaustion +refuses. The real executor copies columns, remaps memberships, carries design +anchors and per-person strata, and checks `mass="conserve"`. It receives no +opaque final Frame from this kernel. Arm provenance does not create a new mass +stratum, and this slice declares no partition-ledger guarantee. + +The opt-in executor parameter `expand_require_sole_weight_entity=True` requires +the declared EXPAND weight entity to be the sole explicitly stored weight +entity. It is an exact boolean, not a truthy control. The executor checks cold +input/output and cached input/output topology. This detects an extra person or +group vector even when it numerically equals inherited household weights. +Nodes omitting the flag retain the previous behavior; explicit `False` also +keeps that behavior. + +The receipt binds the fixed policy, declaration digest, donor/parent/new-household +pairs, separate donor support and host IMPORTANCE weights, and all six lineages. +The graph key includes the parameters, declared provenance artifacts, kernel +implementation and complete base population identity. Required replay must +restore validated stored artifacts and cannot silently refit or rebuild. + +## Verification scope and remaining work + +The invented tests construct a small two-survey, six-entity, two-arm fixture +and use the actual graph executor. They check lineage, five memberships, +nullable/signed-zero storage, fixed-assignment donor-weight independence, +per-parent halves, effective entity and stratum mass, design anchors, cold and +required replay, and refusal paths. The graph-core tests cover the strict +topology flag on cold and cached paths. Test source alone is not a passing +result; execution evidence must be recorded separately. + +This structural prototype carries existing columns verbatim to copied rows. +That does not qualify those values as final clone2 monetary inputs. The actual +source/domain issuer, donor selection and modeled role qualifier, native52 +placement, SS/SCF and three late owners, and variable-clone receiving joins are +separate work. Public fixture dataclasses cannot substitute for their owners. + +The full-spine host must place this expansion before clone-specific census +block assignment and derive larger geographies from the block. This primitive +does not validate or perform that host ordering: it must not be used to claim +that copied preexisting geography is a qualified clone2 draw. Actual-source +qualification, joint administrative-target feasibility, calibration, sparse +concentration/survival checks, engine compatibility and release acceptance +remain incomplete. Poverty is comparison-only, never a tuning, selection or +release criterion. Prior wages are excluded. diff --git a/docs/native-survey-household-domains.md b/docs/native-survey-household-domains.md new file mode 100644 index 000000000..f96826404 --- /dev/null +++ b/docs/native-survey-household-domains.md @@ -0,0 +1,48 @@ +# Source-owned household domain projection + +`qualify_current_survey_household_domains(preparation, allocated_population, +clone_population)` retains a genuine survey preparation and verifies its exact +allocation and ordinary clone0/1 prefix using the maintained no-geography +reconstruction. Its private `QualifiedSurveyHouseholdDomains` capsule provides +`checked_rows()`. Each operation independently rechecks source custody, +implementation, Frame contents, typed weights, original design anchors, +ownership and mass ledger, including an unconditional source epoch close. + +The projection joins selected households to authenticated origins by source, +source year, survey year and raw native ID, then to actual assembled support +and spine coordinates. It requires complete one-to-one coverage and exact +rational anchor agreement. Both clone0 and clone1 appear in its immutable +roster. The roster retains institutional and noninstitutional group quarters +as ineligible for the occupied-housing tail stage. Zero mass remains a separate +condition; it does not change source domain classification. ACS and ASEC year +and observation-window distinctions remain explicit. + +The small receipt binds the roster digest and counts without repeating private +raw IDs. `roster_rows` counts both clones; `original_households` counts originals. +GQ design anchors are publisher person weights, distinguished by the retained +statistical unit. Neither decoded receipts nor returned rows issue authority. Copies or +manually constructed capsules cannot borrow the original issuer. Changes to +retained source objects, source bytes, implementation, population contents or +metadata revoke the capsule. Final pure source and population checks follow +source-close and code-read callbacks, so a callback cannot replace the baseline +or return an earlier successful result after mutation. + +This is a development raw-prefix contract. It has no donor-person allocation, +matching, monetary-placement, geography or release authority. Binding it to a +later pre-geography financial/PUF parent requires a separate receiving-owner +contract with unchanged household lineage. It never promotes an +`InventedHouseholdDomain` declaration to source evidence. Complete-source limits +come from existing admission and roster contracts, not the tiny invented tail +fixtures. This work introduces no calibration targets; poverty stays a +comparison-only diagnostic. + +The authored regression suite uses actual source issuers and the real graph +executor over freshly created invented ACS/ASEC bytes. It covers full roster +projection, same-process/store cold/required parity, source and object mutation, +copy refusal, origin join failures, non-prefix refusal and closing callbacks. +This source description is not a test result or actual-source qualification. + +The source tests preserve real disk probes, including the maintained full-source +ACS reserve of approximately 17 GiB. The proposed local guardian requires at +least 20 GiB of actual free disk before launch. Running these tests in CI requires +adequate real disk space; a low-disk CI configuration has not been qualified. diff --git a/docs/native-usual-hours-proposals.md b/docs/native-usual-hours-proposals.md new file mode 100644 index 000000000..0b5f2dd57 --- /dev/null +++ b/docs/native-usual-hours-proposals.md @@ -0,0 +1,71 @@ +# Native usual-hours proposals + +`microcosm.build.us_runtime.current_survey_hours` provides a pure first slice +for a future native `weekly_hours_worked_before_lsr` graph leaf. It reads no +files, changes no Frame, authenticates no source and grants no release authority. +The implementation continues reviewed native source +`fee4aac9d7be2643fe02ec6b855d231abe5980a4`; that continuation contains APIs not +yet available on main. No parser, native owner, graph host, benchmark, dependency +or accepted-source pin changes in this slice. + +`recode_acs_usual_hours(row)` accepts literal source fields for one original +2024 ACS person. It preserves positive WKHP, including40 and99; retains FWKHP +allocation status; and distinguishes source-supported nonwork from unavailable +survey-universe values. FTP blank is NIU; numeric0 is not a valid WKHP answer. +Under16 alone does not establish zero work. Current ESR and last-week hours +are not used. Missing required headers and contradictory codes refuse. + +`recode_asec_age15_hours(row)` accepts one supplied original March2025/income2024 +age15 donor. It checks HRSWK/WKSWORK and final WRK_CK/WTEMP history, allowing +temporary work after initial WORKYN2. Source allocation flags stay separate. +The [Census 2025 FL_665 codebook](https://api.census.gov/data/2025/cps/asec/mar/variables/FL_665.json) +defines 0 as complete supplement nonresponse, alongside codes 1, 2 and 3. +All four codes are retained as valid source response-status evidence. This +recoding does not exclude nonrespondents or relabel their values as observed; +donor-cohort qualification remains the source owner's separate responsibility. +WTEMP0 is NIU, not an explicit no: positive hours/weeks and final WRK_CK1 remain +consistent with WORKYN2 unless WTEMP2 explicitly contradicts work. This retains +the reviewed donor rule, without claiming any supplied record actually occurred. +The recoder requires positive raw MARSUPWT integer units; it does not qualify +that weight or donor identity against actual files. This is deliberately not +the all-age ASEC hours producer. + +`propose_acs_usual_hours(rows, donors=..., age15_policy=..., +under15_policy=...)` returns frozen proposals for original source people. +Rows carry explicit raw literals, source/construction labels and optional donor +identity/allocation evidence. Source values are preserved; missing adult hours +refuse. Age15 completion requires explicitly selecting +`acs-age15-empirical-full-asec2025-v1`. Under15 modeled zero separately requires +`us_hours_under15_zero_completion_v1`, refuses positive wages or nonzero +self-employment, and retains unknown earnings as unknown. Neither policy is +enabled by default. No missing-hours40 fallback exists. + +The empirical CDF arithmetic and seed20260914 follow the previously reviewed +pure hours helper. Draws now bind original native source identity, not historical +population row positions. ACS keys contain survey/income year, literal SERIALNO +and numeric SPORDER. ASEC donor keys contain survey/income year, PH_SEQ, PERIDNUM +and A_LINENO. Duplicate keys and inconsistent duplicate ASEC coordinates refuse. +Ties in donor hours use canonical native identity; recipient and donor ordering +or batching do not change draws. These new draws need not match dense historical +draws. A later clone adapter must transport each original person's proposal +through checked ancestry instead of independently drawing for clone identities. + +The helper labels every supplied donor cohort **unauthenticated**, even when it +has2174 rows. The future source owner must bind the complete reviewed March2025 +cohort, actual source bytes and roles, native joins, allocation/history projection, +policies and implementation; counts and caller-provided hashes cannot establish +that authority. The chosen full-source empirical model is not a generic ASEC15+ +fallback, and these proposals do not attest model quality for native recipients. + +This slice preserves the separation between the ACS native value, its allocation +status, source nonwork completion, modeled under15 zero and empirical age15 draw. +WKL allocation status is outside its supplied fields, so no unallocated-answer +claim follows from source nonwork. Prior wages are not consumed. Last-week hours, +entity structure, weights, geography and other inputs are outside this helper. + +Invented tests cover code/universe refusals, source40/99 preservation, explicit +policies, temporary-work history, allocation provenance, empirical boundaries, +duplicate native identity refusal and permutation/batch invariance. Native +source qualification, parser retention, graph materialization/replay, per-origin +recipient signal gates, actual-data validation and release acceptance remain +separate work. diff --git a/docs/puf2015-canonical59-and-growth.md b/docs/puf2015-canonical59-and-growth.md new file mode 100644 index 000000000..36733c8fe --- /dev/null +++ b/docs/puf2015-canonical59-and-growth.md @@ -0,0 +1,74 @@ +# PUF 2015 canonical donor and 2024 transport + +These source owners turn the statistical 2015 PUF into an explicit return-level donor with 59 canonical outputs. They preserve raw fields, source status and documentary decisions separately from modeled values. The six detailed mortgage balance, interest and origination-year outputs belong to the downstream SCF stage. + +```mermaid +flowchart LR + A[Authenticated PUF typed source] --> B[Observed and derived return fields] + B --> C[Eleven explicit baseline models] + C --> D[QBI model in 2015 money] + D --> E[Source-specific 2024 monetary transport] + E --> F[Canonical donor with 59 outputs typed donor] + F --> G[Eight-feature PUF59 donor interface] +``` + +The source codec retains all ordinary returns, including those missing the demographic supplement. Disclosure aggregates retain their source lexemes but are excluded from individual donors. Neither a demographic gap nor zero reported exemptions means that a source return is dropped. + +## Count measurements + +PUF 2015 MARS 2 combines joint and qualifying widow(er) returns. The matching predictor maps the survey's separate widow category into that combined class. It is named `puf_2015_filing_status_code`; generic model filing status is unchanged. + +The corresponding `puf_2015_capped_return_size` equals one plus an extra unit for MARS 2 plus the sum of the four reported dependent-category counts. The source owner validates their disclosure caps by filing class. On the survey side, the matching measurement caps the reported dependent total before adding the filing-class units. The result is a statistical predictor, not a count of physical people or co-residents. Actual survey membership remains unchanged. + +The separate `puf_person_incidence_capacity` is one for return-level modeled zero/one QBI incidence. It is used only to validate the canonical outcome representation and is excluded from the eight fitted predictors. + +## Observed fields and explicit models + +The decoder emits 33 directly mapped or algebraically derived baseline fields. Eleven additional baseline outputs explicitly model source gaps: a normalized interest split, a component-neutral Social Security carrier, desired contributions proxied by realized deductions, incomplete tuition and employee-expense proxies, and active-partnership earnings. These transformations retain their assumptions in the receipt. + +The interest model uses the existing [IRS 2015 Table 2.1 workbook](https://www.irs.gov/pub/irs-soi/15in21id.xls). It normalizes the four published component amounts within the source AGI band. Mortgage interest, points and mortgage insurance share the available home-interest leaf; the investment component alone enters investment expense. Complementary rounding preserves the source total exactly. This does not create a mortgage balance, origination year or observed loan structure. + +The QBI owner reproduces the archived data model's assumptions with RECID-keyed random draws, retaining a reusable full-cohort employee calibration. Its 15 added leaves and Schedule C replacement are modeled inputs. The statutory engine owns deductions and limits. Ordinary and SSTB Schedule C branches are summed for the donor's total self-employment predictor, matching the survey measurement. + +## Source-specific monetary transport + +The publisher's statistical 2015 convention is the input basis. Raw filing-year/month fields remain intact; the transform does not apply another row-year CPI adjustment. + +Wages and modeled W2 amounts use the [SSA average wage index](https://www.ssa.gov/oact/cola/AWI.html). Social Security carriers use [SSA COLAs](https://www.ssa.gov/oact/cola/colaseries.html) received from January 2016 through January 2024. This transports fixed entitlements and does not predict changes in recipient composition. + +Other income families use the observed 2015 and 2023 national amount/count cells of [IRS Table 1.4 and 1.4A](https://www.irs.gov/statistics/soi-tax-stats-individual-statistical-tables-by-size-of-adjusted-gross-income), with positive and loss magnitudes kept separate. The 2023→2024 step uses the [BLS annual CPI-U series](https://www.bls.gov/cpi/tables/supplemental-files/historical-cpi-u-202412.pdf), explicitly assuming stable real conditional amounts for that year. Expenses, UBIA and selected mixed model outputs use an explicitly named CPI proxy. Every monetary field has a declared rule; incidence fields are unchanged. + +Partnership and S-corporation reporting counts overlap, so their dollar components are summed with the common all-return denominator. Dividend residuals use the ordinary-dividend reporting universe and do not subtract overlapping recipient counts. + +The PUF short- and long-term capital fields remove the effect of carried losses. Published net-loss means that include carryovers cannot be substituted directly. The model uses current-year gross gain/loss components per all return as a declared proxy for signed net flows. The evidence retains both the carryover-adjusted reported net and component-difference net, including their unresolved residual. It does not attribute that difference to rounding without evidence. + +The observed series are source facts. Their application to individual donor amounts, the related-family substitutions, and the last-year bridge are modeling decisions. They do not establish forecast accuracy or current reporting incidence. A same-cohort CPI-only sensitivity is available. Source design weights are divided from S006 hundredths once and are never grown. + +## Validation status + +The ordinary verification executes real source decoding, the QBI model, growth, the deterministic typed envelope and the PUF59 donor interface at 64 and 2,048 invented records. It checks all 59 outputs, sign/zero preservation, physical type and knownness refusal, exact interest conservation, SSTB/base identities, source immutability, stable full-fit subset replay, artifact byte/value replay and malformed-input refusal. + +The 59 invented controls passed. The separately reviewed genuine construction then produced all 59 canonical outcomes for 207,692 ordinary returns in 23.07 seconds. The 64-return subset reproduced the full-cohort outputs when it reused the full QBI employee fit, and the typed donor passed byte/value replay through the canonical donor interface. + +Independent review subsequently found that the version 1 envelope could attach +an old descriptive receipt to altered, domain-valid values. The version 2 codec +now checks the row-ordered identity/weight/source-feature digest and all 59 output +values against that receipt on encode and decode. It also pins the actual +interest-band facts, verifies the embedded growth recipe, requires canonical +JSON and physical integer row metadata, and makes CPI-sensitivity acceptance +explicit. The corrected source passed 80 invented controls, including the 21 +new integrity cases. Formatting followed that run; the composed integration +checkout still needs its own execution checks. + +The original genuine donor and evidence remain on version 1. A version 2 genuine +construction or repacking requires separate verification, and the new decoder +refuses old magic. Serialized receipt consistency does not authenticate an +issuer, establish execution or confer release status. + +This result verifies source donor construction. Social Security component reconciliation, downstream SCF mortgage semantics, conditional-fit quality, recipient placement and calibration remain open release requirements. It does not certify a calibrated population or policy-analysis file. + +## Evidence bundled with the source + +The [national extract](evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json) records the exact public workbook cells, units, hashes, family calculations and unresolved capital residuals. The [index transcript](evidence/puf2015-target2024/INDEX-VALUES.json) identifies the SSA and BLS values and retrieval sources; it is a transcription of retrieved values, not a hash of the original web pages. The [growth recipe](evidence/puf2015-target2024/GROWTH-RECIPE.json) assigns each monetary field a factor and interpretation. + +[Public workbook locators](evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json) provide the four official URLs and exact sizes and hashes without local cache paths. The recipe also preserves hashes of the original local acquisition receipt and extraction script. Those two local files are outside this publication bundle; their recorded hashes do not imply that a reader received them. diff --git a/docs/puf55-qualified-fixed-inputs.md b/docs/puf55-qualified-fixed-inputs.md new file mode 100644 index 000000000..15fc9fd3f --- /dev/null +++ b/docs/puf55-qualified-fixed-inputs.md @@ -0,0 +1,13 @@ +# Qualified fixed inputs for PUF55 survey arms + +The existing PUF55 recipient interface continues to select clone one by default. `qualify_puf55_survey_recipients(run, arm=0)` adds a separate original-arm projection and distinct graph node IDs. Both projections validate the complete two-arm identity and role mapping. This changes no default application or finalization behavior. + +`qualify_puf55_survey_fixed_inputs(run, arm=0)` requalifies the retained financial run and produces the eight current financial targets on the exact person and tax-unit axes. These are **qualified survey-preserving values**: some ACS amounts and component splits are modeled. They are not all raw observations. Tax-unit amounts include every modeled member and are known only when every member is qualified. + +The optional `development_rules=DEVELOPMENT_RULES` tuple enables four explicitly provisional ASEC mappings. It preserves the maintained pension fraction, regular-IRA-as-taxable routing, the net-property total as a rental proxy, and the direct farm amount. These require original reporting/amount knownness and exact agreement with the carried person value. NIU, ambiguous zeros, incomplete routing, off-route contradictions and unsupported ACS values remain unknown. A finite inherited column cannot establish knownness. Source totals and unresolved taxable/component-split flags remain in the source-basis artifact. Graph parameters name the assumptions and explicitly say release science is not accepted. + +`puf55_survey_fixed_input_nodes(qualified)` declares one artifact-only node for the selected arm. It depends on the genuine source/financial edges, full receiving slices, recipient projection and each selected recipient matrix. Register `Puf55SurveyFixedInputKernel(run, development_rules=...)` alongside the existing recipient kernels. Its outputs include `qualification`, `source_basis`, and the typed target artifacts named by `observed_artifact_names(qualified, profile)`. + +Those target artifacts bind exact float64 values, boolean masks, row identities, target names, matrix bytes and the actual matrix producer key. The core `legacy_qrf_apply_observed_matrix_nodes` interface can consume them when the later host integration is enabled. It executes every target model, retains each raw draw, and merges qualified values before subsequent targets. This module does not fabricate an application prefix, fit or apply a model, attach columns, prune records, change weights, or issue a population. + +On required replay the host must validate actual producer identities against its complete compiled graph and call `verify_materialized_puf55_fixed_inputs` with the materialized artifacts and matrix edges. The verifier requalifies the real financial and source owners; a receipt, detached dataclass, checkpoint or caller-supplied hash grants no source authority. Whole-host execution, arm placement/finalization, seed policy and scientific release qualification remain separate integration work. diff --git a/docs/puf55-survey-ss-measurement-decision.md b/docs/puf55-survey-ss-measurement-decision.md new file mode 100644 index 000000000..fc12e6b85 --- /dev/null +++ b/docs/puf55-survey-ss-measurement-decision.md @@ -0,0 +1,107 @@ +# PUF55 survey Social Security conditioning decision + +On 2026-09-09 the build owner adopted +`social_security_filer_joint_spouse_report_sum_proxy` for PUF matching. This +supersedes the pending measurement judgment in +[survey-social-security-source.md](survey-social-security-source.md). The +decision authorizes this conditioning measurement and its explicit fallback; +it does not assert donor/recipient equivalence, allocate beneficiaries, complete +Social Security components, approve a fit, or admit a release. + +For each modeled return, include exactly one `HEAD` and, only when the actual +`filing_status_input` is `JOINT`, exactly one `SPOUSE`. Sum their available +nonnegative source report totals. Preserve combined family and child reports +without splitting, deduplicating, or subtracting amounts. Exclude dependents +from this return feature without changing any person's observed or unknown +Social Security cells or component masks. + +The public helper requires the actual authenticated survey preparation and +qualifies current source reports. It uses the exact modeled return roles +retained by that preparation. It does **not** rerun tax-unit construction using +current money. That limitation is accepted for this conditioning stage; later +production integration must qualify the chosen role/current-money relationship +explicitly. A caller-made role table or decoded receipt grants no authority. +The PUF disclosure-capped return-size predictor is not a person count, and the +coarsened PUF filing class 2 cannot identify an actual spouse: surviving-spouse +returns also map to that class. + +## Availability and refusal + +An available report-sum means every included reporter has an in-universe, +finite source amount. It does not mean those reporters legally own all the +reported benefits. Known zero reports are valid. Unknown components do not +invalidate a known total. Family/child ambiguity remains an explicit limitation +and does not itself trigger missingness. + +An unavailable included report, including an under-15 filer outside the source +question universe, leaves the proxy unknown and selects the separately named +`PUF55_SURVEY_SS_NO_TOTAL` profile. This profile has the same 55 outputs and +eight predictors. The existing `PUF55_SURVEY_SS` retains nine predictors. +Neither produces the four Social Security components. There is no partial sum, +dependent-zero convention, component-sum fallback, silent row deletion, or +implicit change to the 59/65-output profiles. + +Missing/invalid roles, duplicate or orphan membership, conflicting source +values, nonnumeric amounts, negative/nonfinite observations, and known amounts +outside the reporting universe refuse. They cannot route to eight predictors: +the first two PUF predictors still require valid roles. A source qualifier's own +refusals remain refusals; the fallback does not override them. Graph integration +must bind both disjoint recipient routes, account for each recipient exactly +once, authenticate inherited clone membership, and execute the corresponding +complete fit/apply/attachment chain. This numerical helper does none of that +graph admission or fitting. + +## Measurement limits + +[IRS Publication 915 (2015), pages 2–6](https://www.irs.gov/pub/irs-prior/p915--2015.pdf#page=2) +assigns benefits to the legal beneficiary, including a child whose parent +receives the check; joint filers combine spouses' benefits. The return amount +uses benefits net of repayments and includes the Social Security equivalent +part of tier-1 Railroad Retirement, excluding other pension portions. Form +1040EZ omits benefits, and some recipients need not file. These rules motivate +the selected return members but do not establish survey report ownership or +PUF editing conventions. [SSA's representative-payee guidance](https://www.ssa.gov/payee/faqrep.htm) +likewise distinguishes the person managing a payment from its beneficiary. + +The [2025 ASEC dictionary, PDF pages 48–49](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=48) +permits combined family payments and identifies payments on behalf of children. +Its source total covers calendar 2024 for a 2025 interview. The +[ACS 2024 questionnaire, question 43](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +uses the prior twelve months, permits jointly received income entirely on one +person's record, and combines Social Security with Railroad Retirement. +[ACS SSP metadata, PDF page 45](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf#page=45) +defines the age-15 universe, rounded/topcoded amounts and ADJINC price +adjustment. The helper preserves those distinct observation windows; it does +not calendarize ACS or estimate repayments/Railroad adjustments. + +The ninth donor predictor remains the existing transported 2015 `E02400` +carrier under its declared 2024 money-growth convention. Carrier zeros do not +prove absent benefits. Exact 2015 publisher editing/omission rules remain +unverified; the [IRS PUF page](https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files) +reports the 2012–2015 files unavailable. Matching is therefore conditional on +this documented proxy assumption, not a newly authenticated observed 2024 +return total. Using eight predictors does not remove donor coverage concerns +for nonfilers or resolve assumptions in the other donor outcomes. + +## Diagnostics required before relying on a fit + +Family payments on selected reporters may overstate their own benefits; +payments reported elsewhere may understate them. An all-member sum includes +dependent reports and cannot solve this attribution problem. Topcoding, +allocation, return editing and transported donor money can distort zeros and +tails. Growth of fixed 2015 amounts does not predict 2024 cohort composition. + +Report route counts, known zero/positive distributions and tails by survey, +actual filing status, age, child-report reasons, allocation provenance, and +multiple-return household status. Preserve original-household holdouts across +clones. Compare design-weighted held-out performance and recipient overlap for +eight versus nine predictors; donor-only validation cannot establish cross-source +measurement validity. An adult in-universe all-member report sum may be a +labelled sensitivity diagnostic, with under-15 unknownness retained. It is not +the default feature. Benefit-component modeling and release checks remain +separate work. + +This change provides source-bound numerical values, explicit route metadata, +and an isolated graph profile declaration. Actual source guards, route-aware +graph integration, donor projection for the fallback, fits and diagnostics are +subsequent verification steps, not results claimed by this decision. diff --git a/docs/shared-constants.md b/docs/shared-constants.md index 8d7670652..0edd1bfcc 100644 --- a/docs/shared-constants.md +++ b/docs/shared-constants.md @@ -61,3 +61,61 @@ its declared aliases are code, in derive an engine key through `local_authority_engine_key` rather than re-implementing the rule, and resolve codes through `resolve_local_authority_engine_keys`, which fails closed (microcosm#953). + +## US target comparison views + +[`microcosm.build.us_runtime.target_geography_view`](../packages/microcosm-build/src/microcosm/build/us_runtime/target_geography_view.py) +owns the national/state/congressional-district vocabulary that US +candidate-versus-incumbent target comparisons roll up on +(`US_TARGET_GEOGRAPHY_VIEW_LEVELS`, `US_TARGET_GEOGRAPHY_VIEW_ORDER`, +`UNRESOLVED_GEOGRAPHY_VIEW_LEVEL`, `GEOGRAPHY_VIEW_LEVEL_ALIASES`) and the +`us_target_geography_view` / `us_target_spec_geography_view` resolvers that +read it out of a compiled target's declared evidence. + +The same three scopes are advertised by the frozen native measurement kernel +`graph_fiscal_measurement` as its household `FiscalGeographyScope` levels. +That kernel is not edited to import the shared tuple, because doing so would +move a live graph node identity; +`test_us_release_head_to_head_scorer.py::test_view_levels_match_the_native_measurement_kernel_scopes` +asserts the two vocabularies equal instead, and is the consistency test +required by item 5 above. Congressional-district membership itself is not +restated here: it comes from the shared +`microcosm.data.us_critical_targets.is_congressional_district_target`. + +The same module also owns which metadata key identifies an *area* at each +level (`_LEVEL_OWNED_ID_METADATA_KEYS`: `state` → `state_fips`, +`congressional_district` → `congressional_district_geoid`) and which two +sources carry the ledger fact's geography id verbatim +(`_CANONICAL_GEOGRAPHY_ID_SOURCES`). Both matter because the US target +compiler derives a district's parent `state_fips` from its geoid +(`fiscal_targets.py:2825-2830`), so an identifier read without regard to level +would name the wrong area. Read an identifier only through +`us_target_geography_view`; do not re-derive a level-to-key mapping in a +consumer. + +The source groups do not establish an identifier's encoding: hierarchy and +ledger fields accept bare or malformed text too. The view validates canonical +values at their resolved level with the existing fiscal target compiler's +state/CD parsers and `national_age_activation.NATIONAL_GEOGRAPHY_ID`. Bare +values are recognized separately, with the shared state FIPS mapping; other +nonempty values remain unrecognized. Neither validation rewrites values nor +assigns a missing congressional-district vintage. + +The repo has no shared +constant for the state prefix (`"0400000US"` is an unnamed literal in +`fiscal_targets.py`, `congressional_district_vintage.py`, +`congressional_district_geography.py` and `medicaid_take_up.py`). Until one +exists, `target_geography_view` declares no equivalence between them: +`TargetGeographyView.geography_id_is_canonical` is how a consumer counting +distinct areas keeps the encodings apart. `geography_id_is_bare` distinguishes +recognized bare values from unrecognized ones. Scorecard schema 6 reports both +remainders and counts resolved rows without IDs separately from unresolved +scopes; no consumer should convert between encodings on its own. + +If every explicit scope declaration is unsupported, the row stays unresolved +even when its name or other metadata provides CD evidence. The fallback is +reserved for rows without any explicit scope declaration. + +Do not add another geographic-level enumeration to a US comparison, scorer, or +diagnostics module. Extend this definition and both its consistency tests when +a new view is supported. diff --git a/docs/snap-final-export-checks.md b/docs/snap-final-export-checks.md new file mode 100644 index 000000000..d65917856 --- /dev/null +++ b/docs/snap-final-export-checks.md @@ -0,0 +1,90 @@ +# SNAP final-export checks + +`snap_export_acceptance` in `microcosm.build.us_runtime.snap_release_acceptance` +checks an explicitly identified final H5. It returns individual check statuses, +not an overall release certificate. It is not enabled in the builder by default +and has no CLI. + +The interface requires the H5 path and expected SHA-256, an explicit nonempty +`ExportContract`, integer model year, US `TargetRegistry`, and typed +`TargetDiagnostic` rows. Optional `TargetFitRequirement` objects carry existing +reviewed fit decisions. A file hash identifies bytes; it does not authenticate +source custody, calibrated-weight issuance, the registry, or permission to +publish. The release owner must bind those authorities independently. + +The wrapper requires exactly 51 state rows (50 states plus DC) for each of +`snap_households` and `snap_total`, following the current fiscal-target +materializer metadata. It checks diagnostic identities, target values and error +arithmetic. The final estimates come from a simulation reopened from the H5 by +the maintained `default_simulate_factory`, with the release SPM selection. The +interface accepts no precomputed simulation totals or caller-written passing +summary. Input-stage take-up fit and saturation never waive final-export misses. + +## Count and period definitions + +The current `fiscal_targets.py` mapping uses FNS fiscal-year average monthly +participating households. Its model counterpart is the weighted count of SPM +units with positive annual `snap`: the boolean is evaluated at the SPM grain +before summing. It is not a count of physical households with any recipient or +an annual-ever participation measure. Two positive SPM units within a household +count twice at that household's calibrated weight, regardless of how many +persons belong to either unit. + +The benefits counterpart is the weighted sum of annual modeled `snap` at the +same SPM grain. The result records each target's source period, source citation +and model period separately. It does not relabel an older source observation as +a current-year observation or resolve the fiscal-year/calendar-year difference. +Both calculations retain `MicroSeries` weights and use `map_to="spm_unit"`. +Simulation SPM IDs and order must equal those read from the export. + +The [FNS FY2022 eligible-person participation comparison](https://www.fns.usda.gov/research/snap/state-participation-rates/2022) +is explicitly advisory and pending. This slice does not import the old PR's +rate table or infer eligible people from positive-SNAP SPM-unit counts. A future +comparison needs a verified source table and an aligned eligible-person and SNAP +assistance-unit universe; SPM membership alone does not supply that universe. + +## Tolerances and artifact checks + +The wrapper reuses each `TargetSpec.tolerance` as an absolute tolerance and any +matching `TargetFitRequirement.max_abs_relative_error`. If both apply, both must +hold. A row with neither stays pending, even at zero measured error. Existing +fit requirements retain their minimum-match checks. The wrapper supplies no new +statistical tolerance and does not reuse the input-stage take-up tolerance as a +final-export tolerance. + +The H5 passes the maintained denied-pool ingress boundary. Checks cover the +export contract, absence of persisted `snap`, entity IDs and memberships, +nonempty groups, nonnegative finite household weights, the model year, +nonmissing county/state FIPS, nonzero county suffixes, matching county/state +prefixes, and positive-weight support in all 51 areas. County codes are not authenticated against +a particular vintage's county roster. The file hash is checked again after all +engine calculation and teardown I/O, including when teardown raises. A changed file or failed teardown invalidates computed fit. +Missing engine dependencies or an absent final artifact stay pending; malformed +available evidence produces a failed check. Consumers must inspect every check, +including failures alongside pending checks. + +## Validation and remaining proof + +The flat invented suite writes genuine small H5 files with the maintained writer. +Its explicit test-only weighted-series double delegates sums to typed `Frame` +accounting. It exercises wrapper I/O and aggregation semantics, not actual +PolicyEngine formulas or a genuine release. It includes two SPM units in one +household, a multi-person SPM unit, missing/duplicate/mismatched targets, +malformed geography, stale diagnostics, missing and conflicting tolerances, +unweighted engine output, changed SPM order, and mutation during calculation or +teardown. + +Before release qualification, run this wrapper on the exact final exported H5 +with its authenticated release owner, exact hash, real consumer export contract, +model year, corresponding registry and diagnostics, and reviewed tolerance +decisions. Use the pinned PolicyEngine runtime to recompute all 102 state +estimates. Resolve every failed/pending required check and preserve the report +with the artifact and runtime identities. Separately retain the full release's +source, graph, calibration and publication checks. No actual population, model +execution or release approval was performed by this implementation slice. + +This adapts the final-artifact intent of +[Microcosm PR #413](https://github.com/PolicyEngine/microcosm/pull/413), reviewed +at `c538c81d46661fe2d20876391ea1a4371d14cd8d`. Its old source/feed pins, +10% defaults, caller-supplied totals, manifest-summary approval and old-build +failure lists are not part of this interface. diff --git a/docs/survey-social-security-source.md b/docs/survey-social-security-source.md new file mode 100644 index 000000000..276f6e6f0 --- /dev/null +++ b/docs/survey-social-security-source.md @@ -0,0 +1,169 @@ +# Survey Social Security source semantics + +The source-report Social Security total precedes PUF enrichment. The current +qualifier verifies the original observations and preserves their limits; it +does not fit a completion model, assign individual beneficiaries, or certify a +population. The explicit `PUF55_SURVEY_SS` profile has 55 outputs and uses an +upstream-qualified tax-unit total as its ninth conditioning input. Historical +full65 and PUF59 behavior is unchanged. The new profile's native source and +whole-population attachment integration remain pending. + +## Source measurements + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +describes `SS_YN` and `SS_VAL` on PDF page 49. The question covers people aged +15 or older and permits combined family payments on a person's report. A +positive report is therefore not necessarily one individual beneficiary's +payment. The qualifier preserves the source report's grain. + +The retained ASEC original member supplies `PERIDNUM`, household/line/age +coordinates, `SS_VAL`, `SS_YN`, `RESNSS1`, `RESNSS2`, `RESNSSA`, `I_SSVAL` +and `I_SSYN`. Source keys, the complete current cohort, raw member size/hash, +live source issuance and exact current money values must all agree. Source +year and income year are 2024; interview year is 2025. Carried reason fields +are not a substitute for these original literals. + +The [2024 ACS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf) +defines combined `SSP`, its age-15 question universe, and allocation flag +`FSSP`. The qualifier binds the retained 2024 ACS issuance and verifies +`SSP * (ADJINC / 1000000)` against the prepared amount. This remains a rolling +income window expressed in the survey's price basis. `FSSP` is not in the +current native projection, so publisher allocation provenance stays explicitly +unresolved; no observed-value claim is inferred from a missing flag. + +## Mapping judgments and unknownness + +The reason mapping is an explicit judgment at source-report grain. Retirement +and disability map to their corresponding component. Widowed and surviving +child reasons map to survivors; spouse and dependent child reasons map to +dependents. A single resolved component receives the report total. Repeated +reasons in the same component do not duplicate the amount. + +Distinct component reasons, missing reasons, code 7's combined child categories +and code 8's residual category leave the component amounts unknown. The +published reasons restrict possible components where they can; there is no +reason priority, age fallback or equal-share default. Separately observed +component amounts and individual beneficiary assignments are not claimed. + +An ASEC under-15 `SS_VAL=0` is a not-in-universe sentinel. Both ASEC and ACS +under-15 beneficiary totals remain unknown in the projection, with the raw +ASEC zero retained in the separate literal table. An in-universe zero remains +a known report total of zero. Contradictory universe/recipiency combinations +and invalid nonempty codes refuse; blank literals remain distinguishable +from invalid codes. + +An eventual component model may normalize nonnegative scores within allowed +components, preserving the source total and already resolved source basis. +It must supply positive allowed mass. The numerical helper does not authenticate +such a model; the graph must bind real fit/apply artifacts. A reporting-unit +convention, allocation to individual beneficiaries where necessary, explicit +income periods, design-weighted training, native-to-clone inheritance and +held-out checks remain separate required work. + +## Full original ASEC report basis + +`qualify_full_current_social_security(preparation)` adds a full original ASEC +basis without changing `qualify_current_social_security(preparation)`. It +requires the same live authenticated preparation and recodes the same captured +member once. Its `selected` result preserves the existing selected ASEC/ACS +description, including its evidence. + +The returned `asec_frame` retains every original current ASEC person and the +original household DESIGN weights before receiving-sample selection. The +`asec_basis` table uses those native person IDs and the same row order. It +contains report totals, resolved components, reason-allowed masks, reporting +universe, basis origin, allocation codes and labels, every captured source +literal, and current-money status, validity and zero-origin bytes. Every +selected ASEC row must agree exactly with its full-basis row. + +Receiving selection can therefore omit a resolved positive ASEC report while +the full basis retains it for a later donor-selection decision. This API does +not select eligible donors, fit component probabilities, assign beneficiaries, +or attach canonical component values to a receiving population. Under-15 NIU +zeros and ambiguous report components retain their existing unknownness. + +These returned objects contain detached data. `full_social_security_seal` +describes their exact contents; it cannot authenticate a copied object or +replace the live preparation. A future graph host must retain that preparation, +requalify its source after relevant I/O, and bind the fitted artifacts and +native-to-clone attachment separately. The qualifier checks source, callable, +configuration and detached-result identity after its final owner I/O. + +## Conditioning measurement judgment, 2026-09-09 + +The recipient measurement for predictor nine remains undecided. Passing the +numerical `PUF55_SURVEY_SS` mechanism does not establish donor/recipient +comparability or authenticate a beneficiary total. + +[IRS Publication 915 (2015), pages 3 and 5–6](https://www.irs.gov/pub/irs-prior/p915--2015.pdf#page=3) +combines spouses' benefits on a joint return and assigns benefits to the person +legally entitled to receive them. A child's payment remains the child's even +when the check names the parent. Form 1040 line 20a reports net benefits after +repayments, including equivalent tier-1 railroad benefits; Form 1040EZ does not +report benefits. Thus a sum over all modeled tax-unit members is a different +measurement from the filer/joint-spouse return amount. + +Restricting survey records to the filer and joint-return spouse would still +produce a **report-sum proxy**, not an observed beneficiary total. The +[ASEC dictionary, PDF pages 48–49](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=48) +permits combined family payments and includes reasons for payments on behalf +of children. The [ACS 2024 questionnaire, question 43, page 18](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +also permits jointly received income to appear entirely on one person's +record, with no receipt recorded for the other. Its income category includes +Social Security or Railroad Retirement. A parent may therefore report child +payments, or another household member may report an included person's income. +Multiple tax units within one household make reporter membership insufficient +to establish benefit ownership; household totals must not be duplicated across +returns or redistributed without a separate supported model. + +If adopted, a measurement such as +`social_security_filer_joint_spouse_report_sum_proxy` would need an explicit +upstream contract before supplying `puf_conditioning_social_security_total`. +That contract would bind actual return roles, source reporting records, +knownness and income periods; the coarsened PUF filing-class predictor alone +cannot establish joint-return membership. ASEC calendar-year income, ACS's +rolling window, transported donor money, repayment treatment and railroad +coverage remain comparability questions. Component completion is separate. + +Excluding a dependent from a parent's return feature does not assign that +dependent zero benefits. Both surveys' under-15 amounts remain unknown. If an +included filer has no in-universe report, an empty or missing report set cannot +silently become a known zero. Family-payment ambiguity also survives an +age-universe restriction. + +The exact 2015 `E02400` definition and publisher editing/omission conventions +remain unauthenticated by this lookup. The [live IRS PUF page](https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files), +checked on 2026-09-09, says the 2012–2015 files are unavailable. The verified +Form 1040 rules do not independently establish how every PUF record encodes +them. This record adopts no proxy, allocates no beneficiaries and grants no +dataset or release acceptance. + +## Verification scope + +The 52-case control run executes 50 numerical/literal tests and two complete +maintained source issuances from invented original CSV/checkpoint fixtures. +It verifies source-identity joins despite reordered rows, repeat qualification, +defensive returned data, exact source totals, unknown components and unchanged +input frames. Source/control files, installed model source and the 12 admitted +code resources were unchanged before, after and in the external postcheck. + +The run completed in 67.6 seconds with 428.9 MB peak RSS. Receipt SHA256: +`2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. +These controls are not native-source execution, a fitted component model, +native PUF55 integration, policy-engine validation or release acceptance. + +The additive full-original API has nine further passing invented-source tests. +They retain a resolved positive ASEC report omitted by receiving selection and +its original DESIGN weight, compare the selected projection exactly, preserve +under-15 and ambiguous unknowns, and reject configuration, source-literal and +late detached-result mutations. The existing PUF Social Security measurement +module also passes all 31 tests, including retained-source qualification and +mutation callbacks through the unchanged selected API. + +These two separate runs used 63.2 and 63.9 total supervisor-and-child CPU seconds, +with peak combined resident memory of 455.1 and 538.8 MB. Neither run loaded a +country model, fitted a model, executed a graph, accessed actual microdata, or +changed its pinned source/runtime files. Both restored the no-fit profiler, +reported zero guard denials and reaped their child. They establish this source +interface and its existing measurement consumer; a fitted component estimator +and full graph attachment remain separate work. diff --git a/docs/uk-atomic-household-lineage.md b/docs/uk-atomic-household-lineage.md new file mode 100644 index 000000000..067353f48 --- /dev/null +++ b/docs/uk-atomic-household-lineage.md @@ -0,0 +1,28 @@ +# UK household identity after cloning + +The shared atomic-area operators need a stable household key after initial cloning. `uk_runtime.atomic_household_lineage` projects supplied original-source identities through explicit selection and expansion records, using the existing `household_draw_key` encoding. It adds no assignment algorithm, data reader, issuance registry or full-build host. + +A key includes the declared source, source vintage, original household ID and ordered clone path. It excludes final offset IDs, row order, sample size, calibration weights and financial values. Exact int64 IDs above `2**53` remain distinct. Original households keep their previous path; a new child appends its explicitly supplied branch ordinal. An original branch does not gain an inferred zero ordinal. + +## Contract + +`project_atomic_household_keys(roots, *, steps, final_ids)` returns a detached two-column table: int64 `household_id` and nullable-string `geography_household_key`, in the requested final order. Roots have exactly `household_id`, `source`, `source_vintage` and `source_household_id`. An empty selection is a valid description, not evidence that an empty dataset is useful or releasable. + +Steps are immutable descriptions: + +- `HouseholdExpansion(branch, before_ids, after_ids, parent_pairs, child_ordinals)` carries exact household axes, the household part of the shared executor's `receipt["expand"]`, and explicit child ordinals. The existing ordered branches are SPI support, CGT incidence, CGT band donors and geographic support. Each arriving household needs one known preceding parent and one ordinal. Two children of the same parent cannot reuse an ordinal in the same branch. +- `HouseholdSelection(before_ids, after_ids)` explicitly removes households while preserving their keys. This supports the source-family sample between the completed source spine and geographic expansion. It does not check that the sample respected the host's family-sampling rule. + +Each before-axis must have exactly the previous step's household IDs. Expansion cannot remove incumbents; selection cannot add households. Unknown or null parents, repeated or reordered branches, incomplete parent/ordinal coverage, duplicate keys and a mismatched final axis refuse. Axis ordering may change, because ordering is not identity. The function never guesses ancestry from ID arithmetic, support flags or model values. + +## Host responsibilities + +These checks establish internal consistency of supplied descriptions. A dataclass, DataFrame or serialized receipt does not prove its source. The receiving host must retain and check the actual original source owner, complete before/after populations and source/EXPAND/FILTER evidence. It must authenticate the complete structural stage roster, the explicit branch-ordinal convention, and the exact final receiving household axis before and after relevant I/O. A coherent invented history can pass this pure helper; it is not an admission gate. + +The host can extract the household pairs from its verified shared EXPAND receipt without decoding household ID offsets. The helper returns the key column for the existing `uk_atomic_assignment_definition` and `atomic_geography_nodes` APIs. Support-byte provenance, observed-region constraints and versioned mapping relations remain separate responsibilities. Larger geographies continue to derive from the assigned atomic area. + +No production UK host is changed here. María's full-build graph currently uses its existing ladder draw after geographic expansion. Adopting this key and shared assignment must be coordinated in that host. The existing source stage order also interleaves enrichment before later SPI/CGT cloning; moving atomic assignment before every enrichment stage requires a separate staged migration. This helper does not claim that migration, native FRS qualification or a new UK release. + +## Verification scope + +Invented tests cover the full four-branch path, explicit selection, exact large IDs, source and ordinal changes, stable keys under row reordering and pool growth, unknown/colliding ancestry refusal and detached output. A real shared CREATE/EXPAND/FILTER graph supplies the household lineage receipt and replays from cache. Its projected keys then feed the existing seven-node atomic graph, including three system imports, assignment, derivation and its gate. That graph also completes required replay and preserves the supplied entity tables, weights, strata and metadata. No native source, private archive, country engine or external resource is used. diff --git a/docs/uk-national-calibration-runbook-623.md b/docs/uk-national-calibration-runbook-623.md index ca059a55c..7ab1d2921 100644 --- a/docs/uk-national-calibration-runbook-623.md +++ b/docs/uk-national-calibration-runbook-623.md @@ -17,6 +17,15 @@ is the release-cut producer's job, not calibration's. ## Command shape +The current operational path requires this local workspace and its Git identity. +The calibration seam derives that identity from its module's checkout location; +standalone installed-wheel execution does not yet have a supported provenance +contract. Wheel orchestration tests use an explicit invented code pin and do not +certify installed-wheel production provenance. Supporting that mode requires a +separate identity design that binds the actual loaded implementation, rather +than substituting an unrelated checkout's Git HEAD. An unresolved code pin +continues to refuse before input or output processing. + Calibration runs through the rowwise driver's national release role, `tools/build_uk_rowwise_candidate.py --release-role national` (microcosm#823; it replaced the retired `tools/calibrate_uk_national_dataset.py`). The role diff --git a/docs/us-asec-census-person-columns.md b/docs/us-asec-census-person-columns.md index 73592bac7..cafd84767 100644 --- a/docs/us-asec-census-person-columns.md +++ b/docs/us-asec-census-person-columns.md @@ -102,13 +102,39 @@ the column. | `PTOTVAL` | yes | `frame.units._is_microunit_optional_column` → microunit `_precompute_tax_unit_inputs` | | `A_ENRLW` | yes | `frame.units._is_microunit_optional_column` → microunit `_is_full_time_student` | | `A_FTPT` | yes | same; `eligibility_inputs.derive_us_eligibility_inputs_from_manifest` (`is_full_time_college_student`) | -| `NOW_CAID`, `NOW_COV`, `NOW_DIR`, `NOW_MCARE`, `NOW_MRKS`, `NOW_MRKUN`, `NOW_PCHIP`, `NOW_PRIV`, `NOW_PUB` | no | No build reader. Restoring them would only widen the exported person table. A test fails if code starts reading one. | +| `NOW_COV`, `NOW_DIR`, `NOW_MCARE`, `NOW_MRKS`, `NOW_MRKUN`, `NOW_PCHIP`, `NOW_PRIV`, `NOW_PUB` | no | No build reader. Restoring them would only widen the exported person table. A test fails if code starts reading one. | +| `NOW_CAID` | no | Read only by the native current-survey health projection (`current_survey_health_coverage`, through `current_survey_health_source`). That projection captures its own pinned ASEC person member and never reads the pooled H5 frame. A test fails if any other module names the column. | | `A_FAMTYP`, `A_FAMREL`, `PECOHAB` | no | Read only as optional cross-checks by the SPM role derivation (`spm_role_source._OPTIONAL_RAW_CHECKS`). That derivation reads this same member itself and derives the role from it, so restoring these columns would make the check compare the member with itself. #38's partner derivation should restore them together with its reader. | | `LKWEEKS` | no | Already restored for 2022, the only vintage without it, by `weeks_unemployed.fill_asec_2022_weeks_unemployed_source` from the same pinned `pppub23.csv`. The 2023 and 2024 inputs carry it. On the real run it equals the 8/23 file on all 146,133 rows. | `CENSUS_TAX_ID` was never appended: the extractor derives it, and the Census files do not carry it. +### Restorations after #720 + +`ASEC_CENSUS_PERSON_COLUMNS_BEYOND_720` lists restored columns that were not +in the offline fix. The review test keeps the #720 columns exactly the +offline fix's list, and requires every later column to be declared there. + +| column | restored | reader | +|---|---|---| +| `A_LFSR` | yes | `immigration._assign_ssn_card_codes`: the `immigration_status` stage's worker EAD spill uses labor-force codes 1-4 at ages 16+. `immigration.US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS` lists it, and the stage refuses a person table without it. | + +None of the three H5 inputs carries `A_LFSR`, so it is appended for every +vintage. Its reviewed codes, observed on 2026-09-26 in all three pinned +members after their archive and member SHA-256 checks passed, are 0, 1, 2, 3, +4 and 7. No value is missing, and code 0 is the only code for persons under 15. + +| member | 0 | 1 | 2 | 3 | 4 | 7 | persons | +|---|---|---|---|---|---|---|---| +| `pppub23.csv` | 29,948 | 66,156 | 2,539 | 2,064 | 408 | 45,018 | 146,133 | +| `pppub24.csv` | 28,915 | 66,185 | 2,326 | 2,225 | 410 | 44,204 | 144,265 | +| `pppub25.csv` | 28,155 | 65,222 | 2,214 | 2,422 | 408 | 43,704 | 142,125 | + +A base-stage pool built without a Census person source keeps the #720 hole +and has no `A_LFSR`, so its `immigration_status` stage refuses. It does not +fall back to another worker definition. + ## Real-data evidence (2026-09-23, commit `39b8e7b63`) The real-data run used route A's exact base inputs and flags diff --git a/docs/us-asec-source-domain-contract.md b/docs/us-asec-source-domain-contract.md new file mode 100644 index 000000000..597ec5a7b --- /dev/null +++ b/docs/us-asec-source-domain-contract.md @@ -0,0 +1,43 @@ +# ASEC amount-domain contracts + +The current income-routing, interest and child-support qualifiers read monetary +definitions from the pinned `asec_current_money_domains_v1.json` artifact. +Their public amount-entry functions return detached mappings. Routing and +interest cache immutable tuples privately; child support caches immutable JSON +bytes and reconstructs its nested evidence on each call. Replacing an entry, +clearing a returned mapping or editing nested child-support metadata cannot +change a later caller's definitions. The public mapping and evidence shapes +remain unchanged. + +Each relevant field must occur exactly once and contain exactly one vintage for +the current income year. Its dictionary spelling, URL and digest must match the +qualifier's source pin. Missing or duplicate entries refuse qualification. + +The interest total and child-support parsers accept unsigned integer literals. +They explicitly require the corresponding artifact domains to describe +nonnegative integer dollars, with the supported zero and NIU meanings, no +additional missing-value codes or excluded dollar values, and consistent +encoded, dollar and printed ranges. Bounds remain artifact-derived. Re-pinning +an artifact with an unsupported domain requires a parser change and its own +review; it cannot silently change the interpretation of those literals. + +Before capturing the original source member, each qualifier compares its +relevant amount definitions with the actual retained current-money owner's +`MoneyDomain` entries. Names must be unique; entity, grain, native column, +bounds and zero semantics must agree. Interest checks the retained `INT_VAL` +total. Its additional ordinary and retirement-account components retain their +separate published definitions. Child support checks both `CSP_VAL` and +`CHSP_VAL`; routing checks its existing nine fields. + +These checks preserve the existing source meanings. In particular, a +`CHSP_YN` answer describes an obligation, and neither a no-obligation answer nor +an NIU paid amount establishes zero voluntary payments. Receipt, reporting, +allocation and top-code classifications are unchanged. The changes add no +model, canonical engine input, source issuer or native-data admission. + +Validation used invented mutations of the public domain artifact and actual +invented source-owner fixtures: 73 new domain/cache controls, 88 routing tests, +40 interest tests, 10 allocation tests and 40 child-support tests passed under +the existing bounded source guard. The new controls first reproduced 65 +failures against the prior implementation. These results certify the tested +source contracts, not a native dataset or release. diff --git a/docs/us-atomic-property-financial.md b/docs/us-atomic-property-financial.md new file mode 100644 index 000000000..4380c0f91 --- /dev/null +++ b/docs/us-atomic-property-financial.md @@ -0,0 +1,117 @@ +# Current survey property-income host + +`run_atomic_survey_financial` can append the current property-income graph to +the existing financial composition. Pass an explicit `PropertyIncomeOptions` +through `property_income`; omitting it retains the nineteen-node path. + +```python +from microcosm.build.us_runtime.graph_atomic_survey_financial import ( + run_atomic_survey_financial, +) +from microcosm.build.us_runtime.graph_current_survey_property import ( + PropertyIncomeOptions, +) + +options = PropertyIncomeOptions( + scales=(1.0, 1.0, 1.0, 1.0), + atol=1e-10, + rtol=1e-12, + n_estimators=2, +) +run = run_atomic_survey_financial( + **source_arguments, + geography_config=atomic_geography, + property_income=options, + n_estimators=2, + return_values=True, +) +checked = run.checked_view() +``` + +The small tree counts above are development settings. The caller supplies the +existing authenticated source arguments and atomic geography configuration. +They are not replaced by a downloaded graph, a detached Frame or a receipt. + +The thirty-five-node composition consists of the nine-node survey, clone and +atomic-geography prefix; ten existing financial nodes; and sixteen property +nodes. The new branches select eligible original ASEC donors and original ACS +recipients before allocation. Four ordered, DESIGN-weighted conditional models +produce raw component draws, followed by reconciliation to a known adult ACS +property-income anchor. An explicit attachment carries each original result to +both initial clones. Original ASEC components remain separately qualified even +when that person is excluded from fitting. See the +[fragment contract](us-current-survey-property-graph.md). + +The output adds 24 component, raw-draw, diagnostic and knownness columns. It +preserves the complete preceding financial population: other columns, entity +membership, geography, schema, metadata, weights, strata, ownership and mass +history. Missing source anchors and unresolved components retain missing +values and explicit knownness. This mode does not infer property-income zeros +for children outside the survey question's universe. + +The runner retains both the preceding and extended populations. Before issuance, +execution and required cache replay verify property model training against the +original donor and replay four applications against actual recipient features. +Later issued-run checks requalify the existing source owners, validate the legacy +financial output, reconstruct the complete extended population and check the +verified model artifacts' byte identities. They do not refit or reapply the +models. Copied public result dataclasses have no run authority. See the +[model receipt contract](us-property-model-receipts.md). + +The thirty-five-node option leaves the eight existing financial leaves unchanged. +Its run document therefore records `tax_split_rebased: false` and the legacy +capital-gain model's conditioning on its earlier interest and dividend draws. +It does not itself replace tax inputs. Optional tax rebasing is a separate +three-node extension, described below. + +The PUF host derives its upstream node count and checks the actual final +financial writer. Its recipient nodes read all columns of that population, +including the new property fields, so the dependency is present in the graph. +This interface does not establish that a native PUF run, tax rebase, calibration +or release has passed. Candidate execution evidence belongs in `experiments/`. + +## Optional tax rebase + +Pass `rebase_property_taxes=True` together with `property_income=options` to +append the deterministic receiving version, tax split and numerical gate. The +default is `False`; requesting a rebase without property options refuses before +source I/O. This path has 38 nodes. It retains the complete legacy financial +population and the complete 35-node property population separately from the +final `survey_property.tax_receiving` version. + +The rebase uses ordinary interest and dividends with the maintained fractions +and subtraction complements. Separate missing O/D inputs produce missing tax +leaves, replacing earlier predictions; retirement-account interest remains +auxiliary. A new FILTER conservation record is intentional, while existing +Frame mass history and all non-rebased cells remain unchanged. See the +[numerical fragment contract](us-property-tax-leaves.md). + +The host constructs the declarations from the checked clone Frame and the +existing financial/property output descriptors. It never supplies fake values +for future columns. After the real 35-node output is available, it independently +re-declares and executes the three deterministic operations, checks stored +artifact bytes and complete node receipts, and compares every expected full +population with actual cold or cached observations. The issued run retains +the source, declaration, model, artifact, intermediate population and final +population seals. Later checks reconstruct the same tax operations before the +final source requalification and pure lifetime fence. + +An incomplete numerical gate is valid development evidence. The run document +reports `tax_split_rebased: true`, three extra nodes and the actual +`tax_leaf_complete` value; it remains `release_eligible: false`. PUF recipient +qualification explicitly requires a complete independently checked numerical +gate before it reads its additional source inputs. Both PUF recipient graph +nodes carry a typed edge to that gate on the actual receiving version. The +19- and 35-node paths do not add that edge or claim a tax rebase. + +No child zero completion or missing-adult model is introduced. Consequently the +invented source fixture with under-15 and missing-anchor cases must remain a +development run and refuse full PUF qualification. Capital gains retain their +earlier conditioning; rebasing interest and dividends does not update that model. + +The bounded invented-source acceptance covers 11 host controls, including actual +cold and required-cache 38-node runs, issued-view parity, intermediate/final +population preservation, mutation refusals and incomplete PUF qualification. +The numerical fragment has a separate 40-control acceptance, including a +complete-input positive gate and descriptor-only declaration checks. These +checks do not use native survey data, run a country engine or execute full PUF. diff --git a/docs/us-child-verification-operations.md b/docs/us-child-verification-operations.md new file mode 100644 index 000000000..301a7ddc2 --- /dev/null +++ b/docs/us-child-verification-operations.md @@ -0,0 +1,34 @@ +# Child completion verification operations + +The child property-income verifier independently reconstructs its keyed +empirical result and compares actual materialized populations and artifacts. +Required graph cache hits do not remove that owner check. + +A private synchronous verification operation can now retain that independently +verified reconstruction for repeated checks of the exact same child boundary. +The first verification completes before retention. Each subsequent borrow still +performs both genuine source/parent requalifications, implementation checks, +physical input seals and typed artifact/output comparisons. Changed inputs or +retained expected results refuse; they are not normalized into a new baseline. + +Closing disables reuse before running the full verifier once per participating +child against actual retained evidence. It then checks every retained result +and evidence container again, after all closing callbacks. Failure revokes the +participants, nested scopes cannot restart a closing or failed operation, and +no memo survives scope exit. Direct child and financial checks remain complete +outside these scopes. + +The PUF and enrichment checked operations, public SPM bridge and development +checkpoint writer open or join the scope. They retain their own detached output +and owner checks after close. The writer also checks delivered checkpoint bytes +after closing callbacks and removes its handoff document if final verification +fails. This does not cache successful checked views, source-file reads, graph +compilation, registry validation, financial reconstruction, or source epochs. + +The change targets repeated child semantic reconstruction only. Full population +seals, ancestor checks and source hashing may dominate runtime; no speedup or +completed combined proof is claimed by this source change. Invented tests cover +actual first/closing call counts, required replay, nested/separate lifetimes, +changed inputs/expected results, and mutation during warm or closing checks. +Those tests require separate execution evidence. This is not actual-source, +consumer, calibration or release acceptance. diff --git a/docs/us-current-acs-immigration-source-projection.md b/docs/us-current-acs-immigration-source-projection.md new file mode 100644 index 000000000..c1f6e149b --- /dev/null +++ b/docs/us-current-acs-immigration-source-projection.md @@ -0,0 +1,73 @@ +# Selected ACS immigration source projection + +`borrow_current_acs_immigration_projection(preparation)` in +`microcosm.build.us_runtime.current_acs_immigration_source_projection` returns +selected original ACS source inputs from a live authenticated survey preparation. +It internally invokes the existing immigration literal reader and verifies the +retained ACS native owner. Call `validate()` after the consumer's final relevant +I/O; exclusive access to mutable source and output tables remains required. + +The result contains three narrow tables and a descriptive receipt: + +- `raw` preserves the six original string fields SERIALNO, SPORDER, CIT, POBP, + YOEP and AGEP, indexed by the selected receiving person identifier. +- `person` binds typed CIT/POBP/YOEP/AGEP, observed SEX and `is_female`, original + household state, receiving and native identifiers, source namespace, source + year 2024 and observation year 2024. Nullable YOEP, NIU, public-code precision + and lower/upper bounds retain missingness and censoring without a midpoint. +- `households` binds receiving/native household identifiers, original SERIALNO, + original state token ST and typed state FIPS. + +There is no Frame, weight vector, status assignment, fit, stock calibration, +graph attachment, model input admission or release approval. In particular, +selected ACS people are not a complete-country stock denominator. The existing +shared literal reader also verifies ASEC source files, but this result projects +only ACS fields and consumes no prior-income values. It is opt-in and changes no +build default. + +## Source meanings + +The [2024 one-year PUMS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.txt) +defines the domains. The [2024 PUMS code list](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/code_lists/ACSPUMS2024CodeLists.xls) +defines detailed countries grouped under public POBP codes. The person projection +retains the ACS code namespace; it does not claim that the 222-code domain equals +ASEC PENATVTY. + +| Input | Admitted evidence | +|---|---| +| CIT | Codes 1–5: US-born, specified US territories, US parents abroad, naturalized, noncitizen. No unknown code is filled. | +| POBP | Published 2024 public categories, including state births and grouped residuals. 451 means Sudan only. 464 groups Tunisia, Western Sahara and South Sudan; exact South Sudan origin is unavailable. | +| YOEP | CIT 1 requires blank NIU; a nonblank value contradicts the questionnaire universe and is refused. CIT 2–5 require a published code. 1938 means 1938 or earlier; 1939 means 1939–1944; 1945–2024 are individual calendar years. | +| AGEP | Published 0–99 code, directly equal to the native observed age. Source top-coding is retained, without an exact-age claim. | +| SEX | Native source code 1 or 2 and exact agreement with the native boolean. No unallocated-response claim. | +| State | Original household ST through exact original membership; 50 states and DC. This view does not admit Puerto Rico Community Survey semantics. | + +The [2024 subject definitions, Year of Entry, printed page 146](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf) +include CIT 2 and 3 in the entry-question universe although Census calls them +native-born. Entry refers to the latest occasion of coming to live in the US, +with documented reporting ambiguity for repeat entrants. It is not proof of +first entry or continuous residence. Birthplace is not proof of nationality or +legal immigration status. Other public POBP categories can also be grouped or +unspecified: `published_category` deliberately makes no exact-country claim. + +## Identity and verification + +Each row joins the exact selected source namespace/native person key to original +SERIALNO/SPORDER, original household membership, observed age and sex, and the +same household's state. IDs never pass through a float conversion. The pure +projection supports arbitrary integer coordinates; authentic issuance retains +the existing upstream preparation's coordinate contract. + +The live preparation and native issuer remain the source authorities. The view +retains independent immutable seals for the qualified literals and output tables; +altering a literal owner's callback closure does not move those seals. Copies, +detached replacement tables, a copied receipt and a different preparation cannot +stand in for the retained view. Final pure identity and table checks run after +source and implementation-file I/O. These checks detect changes at validation +boundaries; they do not make concurrent pandas mutation atomic. + +Invented archive tests exercise the actual source catalogue, native issuance and +literal capture path, including household and group-quarters observations. Pure +domain/coordinate controls and mutation, copied-owner, late-I/O and editable- +closure controls complement them. This evidence does not qualify real sources, +immigration rules or a release artifact. diff --git a/docs/us-current-asec-child-support-source.md b/docs/us-current-asec-child-support-source.md new file mode 100644 index 000000000..c55593637 --- /dev/null +++ b/docs/us-current-asec-child-support-source.md @@ -0,0 +1,79 @@ +# Current ASEC child-support observations + +`qualify_current_asec_child_support(preparation)` in +`microcosm.build.us_runtime.current_asec_child_support_source` qualifies the +original received- and paid-support observations from the retained 2025 ASEC +person source. The dollar amounts refer to the current 2024 income cohort. + +The source is the [2025 ASEC public-use dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA-256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The qualifier reuses the exact packaged monetary domains and temporal +authority for `CSP_VAL` and `CHSP_VAL`. It adds source questions and flags from +the same dictionary, with their positions, widths and printed universes. + +| Fields | Source meaning | One-based PDF page | +| --- | --- | --- | +| `CSP_VAL`, `CSP_YN` | Amount received and actual receipt answer; receipt universe age 15+ | 52 | +| `CHSP_VAL`, `CHSP_YN` | Annual amount paid and whether payment is required; the latter is an obligation question | 52 | +| `CHELSEW_YN` | Whether a child lives outside the household; age 15+ | 52 | +| `I_CHELSEWYN`, `I_CHSPVAL`, `I_CHSPYN`, `I_CSPVAL`, `I_CSPYN` | Published allocation flags, using the shared I_ANNVAL code meanings | 54 | +| `TCHSP_VAL`, `TCSP_VAL` | Published top-code indicators for positive amounts | 59 | + +## Received amounts + +The printed zero for `CSP_VAL` means none or NIU. An in-universe receipt-no +answer plus zero establishes known nonreceipt. A receipt-yes answer plus zero +remains ambiguous; the qualifier does not complete it with a zero-dollar +observation. Positive receipt amounts are retained, including valid allocated +observations. Under-15 NIU, missing answers, malformed literals and +contradictory receipt/amount pairs remain distinguishable and unknown. + +## Payments and obligations + +`CHSP_YN` asks whether the person is required to pay child support. A no +answer does not rule out voluntary payment. `CHSP_VAL` describes actual annual +payments and prints zero as NIU, so every paid zero remains a published code +with an unknown canonical payment amount. No obligation answer turns it into +an observed zero. + +The paid amount's universe is `CHSP_YN = 1`. The obligation question itself +prints only the bare field `CHELSEW_YN`, while the allocation flag `I_CHSPYN` +prints `CHELSEW_YN = 1`. That discrepancy remains in the evidence. The +qualifier conservatively accepts positive payments on the observed +intersection age 15+, child-elsewhere yes and required-to-pay yes. Other +routes keep the published positive amount and an explicit unresolved status. +This intersection is a qualification choice, not a correction of the printed +universe or proof that voluntary payments do not occur. + +The obligation and child-elsewhere entries do not independently state a +reference year. Their source context is retained; the qualifier does not +issue a current-at-interview or eligibility input from those answers. + +## Provenance and consumption + +The result has two complete tables: selected ASEC people on the common +preparation's person identities, and the original current-cohort literals on +native identities. Source amount validity, statuses and zero origin are +retained alongside the descriptive projection. Allocation/top-code literal +values and parse statuses remain separate from amount knownness. All-zero +allocation flags describe the published codes; they are not a blanket claim +of unallocated or respondent-reported data. + +The qualifier takes the actual authenticated preparation, captures the +pinned original member once through the shared bounded literal reader, +checks its bytes and complete native-key roster, verifies household/line/raw +age coordinates, and compares both source amounts to their retained +current-money values with exact validity and float64 bits. Source row order +does not define person identity. + +`CurrentAsecChildSupportValues` is descriptive transport, not an issuer. +`child_support_values_seal` covers both complete tables, axes, nullable masks +and backing storage, float bits and detached evidence. The actual preparation +is requalified after source I/O, followed by final owner and physical-value +checks. A consumer must retain the actual owner, requalify immediately before +consumption and after its own last relevant I/O, and compare complete value +seals before returning or exporting a successor. Copied data or a matching +digest cannot grant source authority. + +This module assigns no engine leaf, changes no clone, fits no ACS model and +issues no new source admission or release eligibility. diff --git a/docs/us-current-asec-dividend-source.md b/docs/us-current-asec-dividend-source.md new file mode 100644 index 000000000..de977cd18 --- /dev/null +++ b/docs/us-current-asec-dividend-source.md @@ -0,0 +1,114 @@ +# Current ASEC dividend and survivor routes + +`current_asec_dividend_source.qualify_current_asec_dividend(preparation)` +qualifies current ASEC dividend observations and the recorded survivor-income +source routes that could overlap with a broad property-income concept. It +reads the original 2025 person member, whose annual money amounts refer to +2024. It returns source observations and provenance, without modeling ACS +values, assigning tax treatment or authorizing a dataset release. + +The [2025 ASEC public-use dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +is pinned to SHA-256 +`5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The dividend domain also comes from the existing pinned +`asec_current_money_domains_v1.json`. Page numbers below are one-based PDF +pages; positions and widths refer to the printed ASCII layout. + +| Field | Position / width | Page | Printed universe | +| --- | --- | --- | --- | +| `DIV_VAL` | 478 / 6 | 45 | `DIV_YN = 1` | +| `DIV_YN` | 484 / 1 | 45 | Age 15+ | +| `SUR_SC1`, `SUR_SC2` | 648 / 2, 650 / 2 | 50 | `SUR_YN = 1` | +| `SUR_YN` | 664 / 1 | 50 | Age 15+ | +| `I_DIVVAL` | 819 / 1 | 54 | `DIV_YN = 1` | +| `I_DIVYN` | 820 / 1 | 54 | Age 15+ | +| `TDIV_VAL` | 905 / 1 | 59 | `DIV_VAL > 0` | +| `TRNT_VAL` | 917 / 1 | 60 | `RNT_VAL > 0` | + +## Dividend values and source flags + +The projection preserves the exact `DIV_VAL_literal`, parsed +`DIV_VAL_published_amount`, literal and reporting statuses, and a separate +`DIV_VAL_amount_known` mask. `DIV_VAL_amount` contains only the values whose +reporting status is resolved. An in-universe receipt-yes and positive amount +is known; receipt-no and zero establish known nonreceipt. Receipt-yes and +zero remain ambiguous because the dictionary combines none and NIU in its +zero code. Missing literals, NIU and contradictory combinations retain +their source evidence and an unknown canonical amount. No analytical zero +is inferred solely from age. + +Source flags never change this knownness or select donors. A known published +cell can still have publisher allocation or topcoding metadata. It is not a +claim about an uncensored latent amount or a respondent-only report. +`I_DIVVAL` refers to the allocation meanings under `I_ANNVAL` on page 53. +`TDIV_VAL` and `TRNT_VAL` retain codes 0 (not topcoded) and 1 (topcoded), +including missing or malformed flags. `TRNT_VAL` is retained only for future +diagnostics: this qualifier does not read `RNT_VAL` or evaluate that flag's +positive-rent universe. + +The dictionary is internally inconsistent for `I_DIVYN`: its header prints +0–1, while its Values section refers to the `I_ANNVAL` codes 0–9. The result +preserves the literal and code, physical syntax status, separate +`I_DIVYN_header_range_status` and `I_DIVYN_referenced_values_status`, and an +explicit `I_DIVYN_codebook_status`. Codes 2–9 therefore retain the conflict; +they are not silently dropped or admitted under a preferred interpretation. + +## Survivor source routes + +`SUR_YN`, `SUR_SC1` and `SUR_SC2` each retain raw literals, parsed codes, +literal status and source labels. Code 8 identifies regular payments from +estates or trusts. It signals a possible property-income overlap, without +establishing an amount or tax classification. Code 10 means other or don't +know: its literal is valid, but its property route remains unspecified. + +`survivor_property_route_clear` is nullable. It is true only for an +in-universe known receipt-no with both source slots NIU, or receipt-yes with +every slot readable, at least one active slot, and neither code 8 nor code +10. Receipt-yes with an explicit code 8 sets it false. Missing or unreadable +receipts, unresolved slots, contradictory nonreceipt routes, NIU and code 10 +leave it unknown. Under-15 rows remain unknown; incomplete under-15 literals +are labeled unresolved rather than contradictory. Independent booleans +retain the presence of codes 8 and 10 even when the receipt is unresolved. + +This clearance describes only the two recorded source slots. The dictionary's +`SRVS_VAL` entry (PDF page 49, printed page 6C-28) also includes unedited third +and fourth sources. Their types are not qualified here, so two clear visible +slots cannot establish complete absence of survivor estate/trust income. +The property donor bridge conservatively excludes positive survivor receipts +until that additional scope is qualified. This clearance does not establish +that the person has no property income. No survivor amount is captured, no +donor exclusion is applied, and a missing slot is never an observed zero +amount. A later measurement bridge must explicitly choose and validate how +these routes affect donor eligibility. + +## Retained source boundaries + +The qualifier borrows the original authenticated survey preparation and its +retained native ASEC/current-money owner. Before capture it checks the live +money domain against the pinned dividend contract. The domain reader +requires a unique field and current vintage, refuses unsupported negative, +missing-code or zero semantics, and caches only immutable values. Public +entry mappings are detached on every call. + +The exact member's archive and member digests, row count and byte count +must agree with the retained owner. A single bounded capture is checked +again after reading. All current-year native keys, original household and +person-line coordinates, and source ages are joined before selection. +`DIV_VAL` validity and every valid float64 bit must match the actual money +owner, including people omitted from the selected survey. Original +`DIV_VAL_parent_statuses`, `DIV_VAL_parent_validity` and +`DIV_VAL_parent_zero_origin` remain in the projection. + +`CurrentAsecDividendValues(person, asec_literals, evidence)` is descriptive +transport. `person` is indexed by `person_id` and includes `native_person_id` +and `source_age`; `asec_literals` retains the complete current-year literal +table indexed by native person ID. `dividend_values_seal(values)` covers +both tables, their axes, exact float bits, nullable masks and backing +storage, and detached evidence. The qualifier rechecks the original +preparation after source I/O and verifies the complete seal before return. + +Consumers must retain and requalify the actual preparation before +consumption and after their last relevant I/O, and compare the complete +values seal. Matching JSON, a copied dataclass or a digest does not grant +source authority. No host, graph attachment, tax leaf, source issuer, +original/PUF clone change or release gate is added here. diff --git a/docs/us-current-asec-income-routing-source.md b/docs/us-current-asec-income-routing-source.md new file mode 100644 index 000000000..22968e633 --- /dev/null +++ b/docs/us-current-asec-income-routing-source.md @@ -0,0 +1,272 @@ +# Current ASEC income reporting and routing source + +`microcosm.build.us_runtime.current_asec_income_routing_source` qualifies the +exact retained 2025 ASEC person member against the authenticated current-money +owner and projects five original-channel income families the current survey +graph does not yet supply: + +| Family | Printed amounts | Printed receipt / routing | +| --- | --- | --- | +| `pension_annuity` | `PNSN_VAL`, `ANN_VAL` | `PEN_YN`, `ANN_YN` | +| `retirement_distribution` | `DST_VAL1`, `DST_VAL1_YNG`, `DST_VAL2`, `DST_VAL2_YNG` | `DST_YN`, `DST_YN_YNG`, `DST_SC1`, `DST_SC1_YNG`, `DST_SC2`, `DST_SC2_YNG` | +| `net_property` | `RNT_VAL` | `RNT_YN` | +| `farm` | `FRSE_VAL` | `FRSE_YN`, `ERN_YN`, `FRMOTR` | +| `other_income` | `OI_VAL` | `OI_YN`, `OI_OFF` | + +Thirteen published allocation flags are read alongside them. + +Both returned frames carry a digest: `evidence["projection_sha256"]` over +`person` and `evidence["literals_sha256"]` over `asec_literals`. Both are +re-checked at the end of the qualifier, so a host comparing the documented +digests detects a corrupted transport on either frame. + +`qualify_current_asec_income_routing(preparation)` takes a live +`AuthenticatedSurveyPopulationPreparation`, requalifies it, captures the pinned +2024 member once, joins it to the money parent by exact native keys, and returns +`CurrentAsecIncomeRoutingValues(person, asec_literals, evidence)`. The returned +values are descriptive transport. Constructing or mutating them issues no source +authority: a consuming host re-runs this qualifier and compares the result +around its own I/O, exactly as the accepted unemployment qualifier requires. + +## What the qualifier establishes + +- **The exact source member.** The 2024 pin is matched against the retained + native coverage receipt (member name, archive digest, member digest, row count + and byte size) before the capture, and the captured file's stat identity and + SHA-256 are re-checked after it. +- **Row-for-row parent correspondence.** `PH_SEQ`, `A_LINENO` and `A_AGE` from + the member must equal the money parent's `source_household_id`, `A_LINENO` and + `A_AGE` at the 2024 scope positions. The join is by native key, so member row + order cannot change the projection. +- **Literal/amount identity.** Every projected amount is the money owner's own + value, checked against the member literal, with the parent's `statuses`, + `validity` and `zero_origin` carried through as `amount_status_*`, + `amount_validity_*` and `zero_origin_*` columns. +- **A single bounded pass.** One capture, one read, one projection. There is no + per-family re-capture, no nested run issuer, and no PUF borrow. + +## What it deliberately does not establish + +- `PNSN_VAL` is printed as the *total combined amount of pension income received + from all pension sources* — `PEN_SC1`/`PEN_SC2` enumerate company, union, + federal, state, local, military and railroad pensions. It is not an observed + private or taxable amount. The archived `TAXABLE_PENSION_FRACTION = 0.590` + split is a modelled assumption and is not applied or referenced here; + `pension_annuity_private_share_applied` and + `pension_annuity_taxable_amount_known` are constant `False`. +- Account code 4 is `Regular IRA`. That identity does not observe any taxable + fraction, so `retirement_distribution_taxable_amount_known` is constant + `False` and no taxable component is derived. +- `RNT_YN` asks about land, property rented to others, royalties, roomers or + boarders, and estates or trusts, while `RNT_VAL` asks only about *income from + rent after expenses*. Both scopes are carried on every row and + `net_property_component_split_known` is constant `False`, so the total is not + independently labelled rental and no subdivision is separately modelled. +- `FRSE_VAL` is farm self-employment, not nonfarm `SEMP_VAL`; + `farm_is_nonfarm_self_employment` is constant `False`. Its printed label + includes the composite clause naming `ERN_VAL` (when `ERN_SRCE=3`) and + `FRM_VAL`, so the total already contains those components. +- `retirement_distribution_regular_ira_amount` is published only when every + applicable slot is fully resolved on both axes. An unreadable account code + could itself be a regular IRA, and a declared account whose amount is a + "none or niu" zero does not observe a zero dollar distribution, so both leave + the regular-IRA share unknown rather than at zero. +- `OI_OFF` code 20 is the reported alimony category. Nothing maps any other + code — including 19, `anything else` — onto alimony or onto a miscellaneous + residual; `other_income_residual_rule_applied` is constant `False`. + `other_income_routing_status` separates `reported_category` from + `receipt_without_category`, `category_without_receipt`, + `unresolved_receipt_routing`, `missing_category_literal`, + `unrecognized_category_literal` and `niu_category`, and + `other_income_is_reported_alimony` is set only on a `reported_category` row, + so an unreadable receipt literal never yields a reported alimony receipt. +- ACS-channel people are not projected at all. `acs_components_modeled` is + `False` in the evidence and the projection covers only ASEC-channel rows. + +## Reporting status vocabulary + +`receipt_status` is the single classifier. Only `known_receipt`, +`known_recipient_zero` and `known_nonreceipt` establish a dollar reading +(`KNOWN_AMOUNT_STATUSES`); every other status leaves the canonical amount +unknown rather than completing it with zero. + +| Status | Meaning | +| --- | --- | +| `known_receipt` | in universe, receipt yes, amount non-zero (a signed loss included) | +| `known_recipient_zero` | in universe, receipt yes, amount zero on an entry whose printed zero is valid dollars — `ANN_VAL` alone, whose NIU is the separate `-1` code | +| `receipt_with_net_zero` | in universe, receipt yes, amount zero on a signed net measure (`RNT_VAL`, `FRSE_VAL`). Distinct from a gross entry's recipient zero, but **not** a known amount: both print `0 = none or niu` | +| `ambiguous_recipient_zero` | in universe, receipt yes, amount zero on a gross entry whose printed zero reads "none or niu" | +| `known_nonreceipt` | in universe, receipt no, amount zero | +| `niu` | in universe, receipt 0, amount zero or a declared non-money code | +| `missing_amount` / `missing_receipt_literal` | one side of the pair is absent | +| `unrecognized_receipt_literal` | the receipt literal is malformed or outside the printed range | +| `contradictory_no_nonzero`, `contradictory_niu_nonzero`, `contradictory_declared_niu_amount`, `contradictory_outside_reporting_universe` | retained source contradictions, excluded from every canonical amount | +| `contradictory_offroute_evidence` | distributions only: the route that does not apply carries dollars or an answered recipiency | +| `unresolved_slot_composition` | distributions only: an applicable slot declares an account whose amount is a "none or niu" zero | +| `outside_reporting_universe` | the printed universe excludes the row and the literals agree | +| `unresolved_reporting_universe` | the printed universe cannot be resolved from the retained literals | + +Which recipient zeros resolve is read from the pinned domains artifact, not +decided here: `_zero_is_dollars` compares each entry's `zero_semantics` against +`valid_zero_dollars`. Eight of the nine entries record +`none_or_niu_not_distinguishable_from_amount_alone`, including both signed net +measures, so a signed zero is separated by label but never completed. + +The receipt code labels are per entry, not shared: `receipt_codes(field)` reads +the printed zero label from `RECEIPT_ENTRIES`, so `OI_YN` reports `none or niu` +where `PEN_YN` reports `niu` and `FRSE_YN` reports `Niu` exactly as printed. + +Other-income routing (`other_income_routing_status`) follows the same universe: +`OI_OFF` is printed for `OI_YN = 1` and `OI_YN` for persons aged 15+, so a row +outside or unresolved on that universe reports +`outside_reporting_universe_routing` or `unresolved_reporting_universe_routing` +and never a reported category. + +Universes are per family and never reduced to age alone. `PEN_YN`, `ANN_YN`, +`RNT_YN` and `OI_YN` print `All Persons aged 15+`. `FRSE_YN` prints +`ERN_YN=1 or FRMOTR=1`, so the farm universe is `True` when either literal is +`1`, `False` when both are known and neither is, and unresolved otherwise — +a nonfiler is never treated as a non-recipient. + +## Source-level questions preserved as evidence + +These are printed-dictionary facts, kept distinguishable from modelling +judgments. The first four are recorded under +`evidence["dictionary"]["printed_universe_questions"]`; the rest under +`evidence["dictionary"]["printed_scope_and_code_questions"]`: + +- `DST_VAL1`'s printed universe is `DST_SC1 = 1` although its label names the + source-1 distribution amount; taken literally that would restrict it to 401k + accounts. Retained verbatim, not silently corrected. +- `DST_SC2_YNG`'s printed universe names `DST_VAL_YNG`, a field with no + dictionary entry. +- `I_DSTVAL1COMP`'s printed `Universe:` line is empty. +- `DST_YN` and `DST_YN_YNG` print only the age-58 split. They print no 15+ floor + as the four age-universe families (`PEN_YN`, `ANN_YN`, `RNT_YN`, `OI_YN`) do, + and they are not gated on other literals as the farm family is — `FRSE_YN` + prints `ERN_YN=1 or FRMOTR=1` and so carries no age floor either. Coverage + below age 15 is therefore recorded as unresolved rather than resolved either + way. +- `OI_YN`'s printed zero label is `none or niu` where `PEN_YN`, `ANN_YN`, + `DST_YN` and `RNT_YN` print `niu`, so a zero receipt literal is not + interchangeable across families. +- `DST_SC1` is gated on `DST_VAL1 > 0 and a_age ≥ 58` while `DST_SC1_YNG` is + gated on `DST_YN_YNG = 1 and a_age < 58` — the two routes are not symmetric. + +Both printed recipiency literals are retained per row +(`retirement_distribution_receipt_58_*` and `..._receipt_young_*`) alongside all +four slots. The age route selects which pair applies; +`retirement_distribution_offroute_receipt` and +`retirement_distribution_offroute_nonzero` flag an answered recipiency or a +non-zero amount on the route that does not apply, so the complete slot/receipt +pattern stays inspectable rather than being reduced to the applicable half. + +## Amount semantics from the money owner + +The authenticated money owner normalizes `ANN_VAL`'s printed `-1` to a stored +zero and records `CodebookStatus.DECLARED_NIU`. `amount_state` therefore reads +the dollar meaning of a cell from the parent's status axis, never from the +stored number: re-deriving NIU from the number would read that cell as a zero +dollar annuity. `evidence["declared_niu_normalized_to_zero"]` (a top-level key) lists the +affected entries. + +The money owner restates non-2024 cohorts to the pinned price basis, so the +literal-identity join is taken only on `person_years == 2024`; widening it to +the pooled 2022/2023 cohorts would require handling that restatement first. +`evidence["restatement_note"]` records this. + +## Allocation provenance + +`*_allocation_origin` is derived from published flags only: + +- `publisher_allocated` — some applicable flag is non-zero. +- `published_flags_all_zero` — every applicable flag is populated and reads + zero, and every field in the family carries a published flag. This is **not** + an assertion of non-allocation: each flag prints a conditional universe (for + example `I_RNTVAL` is printed for `RNT_VAL > 0`) and those universes are not + evaluated here, so a non-recipient row sits outside its own flag's universe. +- `published_flags_all_zero_with_unflagged_fields` — the same, but the family + also holds a field the dictionary does not flag. +- `allocation_flag_not_populated` — a flag literal is absent. Every flag here + prints a conditional universe, so this is not a defect and not "no + allocation". +- `unresolved_allocation_provenance` — a flag literal is malformed or outside + its printed range. + +`PUBLISHED_ALLOCATION_FLAG_BY_FIELD` names which flag covers which field. +`UNFLAGGED_FIELDS` records the ten fields for which the 2025 dictionary +publishes no flag: `PNSN_VAL`, `FRSE_VAL`, `FRSE_YN`, `OI_OFF`, `OI_YN`, +`DST_VAL1_YNG`, `DST_VAL2_YNG`, `DST_YN_YNG`, `DST_SC1_YNG`, `DST_SC2_YNG`. +`AMBIGUOUS_FLAG_COVERAGE` records three printed ambiguities: `I_DSTSCCOMP`'s +label names `DST_SC(2)` while its universe names both routes, so its coverage of +the under-58 source codes is unresolved; `I_DSTSC`'s label uses the same +`DST_SC(2)` notation, so mapping it to both `DST_SC1` and `DST_SC2` reads the +parenthesis as a slot count that the dictionary does not state; and `I_FRMYN` +prints an empty `Values:` block, so only its `(0:9)` range header is published +and no meaning is claimed for its codes. A nonzero `I_FRMYN` alone yields +`allocation_code_meaning_unpublished`; a documented nonzero allocation flag +such as `I_ERNYN` can still establish `publisher_allocated`. All-zero readings +remain descriptions of the published codes, not assertions of nonallocation. +The original flag values and parse statuses remain unchanged. No row is ever labelled a raw +respondent value. + +## Constants + +The nine printed money entries are **not** retyped here: they are read on demand +from the packaged `asec_current_money_domains_v1.json` under its pinned digest +(`money.RESOURCE_PINS[0]`), which already attests printed position, length, +page, range, universe and values per vintage. The routing code systems +(`RECEIPT_CODES`, `ACCOUNT_CODES`, `OTHER_INCOME_CATEGORIES`, +`ALLOCATION_ENTRIES`) are not in that artifact and are defined here from the +pinned dictionary. + +The printed entry tables (`RECEIPT_ENTRIES`, `ACCOUNT_ENTRIES`, +`ALLOCATION_ENTRIES`, `OTHER_INCOME_CATEGORY_ENTRY`) are `NamedTuple`s, so every +use site and the emitted receipt reach their fields by name rather than by +position. + +Two adjacent definitions exist in tree and are deliberately **not** imported, so +this qualifier keeps the accepted source-only import neighbourhood +(`asec_coverage_authentication`, `asec_current_money`, `source_csv_builtin`, +`survey_population_preparation`, `support_provenance`) and stays free of the +modelled stage machinery: + +- `retirement_distributions.US_RETIREMENT_DISTRIBUTION_REQUIRED_SOURCE_COLUMNS` + — the same eight `DST_SC*`/`DST_VAL*` column names. +- `alimony._ASEC_ALIMONY_OTHER_INCOME_CODE` and + `alimony._ASEC_STRIKE_BENEFITS_OTHER_INCOME_CODE` — the same category codes 20 + and 12. +- `retirement_distributions._VALID_ACCOUNT_CODES` — the same 0-7 account code + domain that `ACCOUNT_CODES` gives printed labels for. + +`test_routing_code_systems_agree_with_the_existing_domain_constants` asserts +both agreements, per `docs/shared-constants.md` rule 5. If a later change makes +the modelled modules importable from a source qualifier, import those +definitions directly and drop the local copies. + +`RINT_SC1`/`RINT_SC2` reuse the identical 0:7 retirement-account enumeration for +retirement *interest*. They are out of this slice; a future consumer mapping +account codes to outputs must decide whether they share this code system. + +## Remaining work + +This lane delivers source qualification only. Still open, and explicitly not +decided here: + +1. **Canonical attachment.** Which projected columns become graph leaves, on + which clone, and under what name. These families overlap PUF-owned values, so + the unemployment/health both-clone policy cannot simply be reused. Root's + active amount owner owns `graph_us_survey_enrichment`. +2. **ACS clone0 components.** ACS publishes aggregate anchors + (`acs_retirement_income`, `acs_interest_dividend_rental_income`) rather than + these components. A conditional model plus a documented reconciliation rule + is needed, and the existing interest/dividend draws and ACS `INTP` overlap + mean rent cannot be added independently without checking that aggregate. +3. **Tax composition.** The pension private/taxable split and the distribution + taxable fraction remain unobserved. Any split must be approved explicitly + against the source, not inherited from the archived assumptions. +4. **Net property decomposition.** The receipt/amount scope mismatch must be + resolved before rent, royalties, roomers/boarders and estate/trust income are + modelled as separate components. +5. **Other-income residual.** Categories other than 20 are preserved but + unrouted; a residual-to-miscellaneous rule is a separate, reviewed decision. diff --git a/docs/us-current-asec-interest-source.md b/docs/us-current-asec-interest-source.md new file mode 100644 index 000000000..d4cbfbfd8 --- /dev/null +++ b/docs/us-current-asec-interest-source.md @@ -0,0 +1,91 @@ +# Current ASEC interest observations + +`current_asec_interest_source.qualify_current_asec_interest(preparation)` +qualifies ordinary and retirement-account interest from the original 2025 ASEC +person member, referring to income in 2024. It preserves the already qualified +`INT_VAL` combined total and exposes the published components and their +reporting, account, allocation and disclosure metadata. + +The source is the [2025 ASEC public-use data dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA-256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The exact total-income domain is reused from the pinned +`asec_current_money_domains_v1.json`; the additional fields use this same +dictionary. Pages below are one-based PDF pages. + +| Fields | Meaning and printed universe | Page | +| --- | --- | --- | +| `INT_VAL`, `INT_YN` | Combined interest; amount universe `INT_YN = 1`, receipt universe age 15+ | 46 | +| `TRDINT_VAL` | Interest excluding retirement-account interest; `INT_YN = 1` | 50 | +| `RINT_YN` | Retirement-account interest receipt; age 15+ | 49 | +| `RINT_SC1`, `RINT_SC2` | Account identities, including 401k, 403b, Roth IRA, Regular IRA, KEOGH, SEP and other; `RINT_YN = 1` | 49 | +| `RINT_VAL1`, `RINT_VAL2` | Interest in the two reported account slots; corresponding account code greater than zero | 49 | +| `I_INTVAL`, `I_INTYN` | Combined-interest allocation flags, with distinct composite code systems | 56 | +| `I_RINTSC`, `I_RINTVAL1`, `I_RINTVAL2`, `I_RINTYN` | Retirement-account allocation flags; `I_RINTSC` names slot 1 only | 57 | +| `TRINT_VAL1`, `TRINT_VAL2`, `TTRDINT_VAL` | Published top-code indicators for the respective positive component amounts | 60 | + +The account, receipt and applicable allocation code systems reuse the income +routing qualifier. `I_INTVAL` accepts the printed codes 0 and 11–15; its header +interval does not make codes 1–10 valid. There is no individually published +allocation flag for `TRDINT_VAL` or `RINT_SC2` in this dictionary. Allocation +labels describe publisher processing, not observation validity or tax status. + +## Amounts and knownness + +The result keeps raw source literals, parsed published amounts, parse status, +reporting status and canonical amount knownness separately. Missing and +malformed components remain unknown. Interest amounts must be nonnegative +integers within each field's published encoding; these disclosure ranges are +not latent-income bounds. + +`TRDINT_VAL` describes its range as a dollar value, so a published zero with +combined-interest receipt `yes` is an observed zero ordinary component. +`INT_VAL` and `RINT_VAL1/2` describe zero as none or NIU, so zero with receipt +`yes` remains ambiguous. A retirement-interest recipient with account code 0 +and slot amount 0 has an `unreported_account_slot`, not an imputed account +amount. Receipt `no` plus account code 0 and amount 0 establishes known +nonreceipt. Under-15 NIU values never become analytical zeros. + +The diagnostic `combined_minus_published_components` subtracts the three +parsed published component cells from `INT_VAL`. Its knownness means that all +four published numbers are readable; it does not establish that all three +components are reported observations. `all_component_amounts_observed` +separately records that stronger condition. The diagnostic may be negative; +the qualifier never balances, clips or replaces any published number. + +## Source lifetime and composition + +Qualification borrows the actual `AuthenticatedSurveyPopulationPreparation` +and its retained native ASEC/current-money owners. It matches the original +member's archive digest, member digest, byte count and row count, captures that +member once, and verifies the captured file after reading. Current-cohort +`PERIDNUM` keys must form an exact roster; household coordinates, person line +numbers and raw ages must match the retained parent. Source row order is +irrelevant. The original `INT_VAL` validity and float64 bits must agree with +the current-money owner before selected ASEC people are projected. + +`CurrentAsecInterestValues(person, asec_literals, evidence)` is descriptive +transport. Its seal covers both complete tables, axes, nullable masks and +backing storage, exact float bits and evidence. The qualifier rechecks the +actual preparation after its last source I/O and checks the complete value +seal before returning. A consumer must retain the actual preparation, +requalify it immediately before consumption and after its own last relevant +I/O, and compare the full value seal before returning or exporting a +successor. A copied dataclass, evidence dictionary or matching digest cannot +grant source authority. + +The only existing helper change is an optional explicit column/amount-grammar +roster on the private income-routing literal reader. Its original defaults +remain in effect for the income-routing qualifier. This module adds no host, +run issuer, graph execution or model fitting. + +## Bridge work still required + +[ACS response guidance](https://www.census.gov/programs-surveys/acs/respond/get-help.html) +includes interest credited to retirement accounts in its broad +interest/dividend/property item. The live guide currently refers to the 2025 +questionnaire; it is not an archived 2024 instruction artifact. This +ASEC qualification therefore keeps ordinary interest and retirement-account +interest separate for a later, reviewed measurement bridge. It does not +declare either component taxable, tax-exempt, regularly withdrawn, or equal +to an ACS component. It does not alter tax leaves, original or PUF clones, +financial imputation, calibration or release eligibility. diff --git a/docs/us-current-asec-property-basis.md b/docs/us-current-asec-property-basis.md new file mode 100644 index 000000000..40a4be5ea --- /dev/null +++ b/docs/us-current-asec-property-basis.md @@ -0,0 +1,132 @@ +# ASEC property donor basis + +`build_asec_property_basis` is a pure operation over the selected, qualified ASEC +person descriptions. It does not qualify a source, issue a handle, read files, +fit a model or assign tax leaves. The owning country host retains the actual +preparation and source owners, checks them before consumption and requalifies +and checks their complete seals after its last relevant I/O before return or +export. Reconstructing the descriptive input or output dataclasses grants no +source authority. + +## Inputs and outputs + +The keyword-only inputs are the `person` tables from the current interest, +income-routing and dividend/survivor qualifiers, named `interest`, +`income_routing` and `dividend`. Every table must have the same exact ordered +`person_id` index, matching unique `native_person_id` values and matching source +ages. Both ID axes are int64. Source ages may use different numeric storage +but must be finite, integer-valued ages 0–99 and equal without tolerance. No +sorting or inner join repairs a mismatch. A common row permutation is valid; +permuting one source alone refuses. + +`original_household_membership` is an int64 Series on that exact person index. +`original_household_design_weights` is a finite, nonnegative Series indexed by +unique int64 `household_id`, with exactly the membership's household set. +Extra, missing or duplicated household keys refuse. These inputs must be the +original household design mapping, before importance allocation or cloning; +this pure operation cannot authenticate that origin. Zero-weight persons are +retained. No engine or microsimulation aggregation occurs. + +The descriptive `AsecPropertyBasis` contains: + +- `person`: the component values, reported total, discrepancies, named retirement + derivations, original household mapping and the two eligibility masks. +- `provenance`: detached copies of all three supplied qualified tables, with + `interest.`, `income_routing.` and `dividend.` prefixes. Published amounts, + unknownness, allocation and disclosure indicators are preserved independently. +- `exclusions`: independent reasons for exclusion on the complete person axis. + A person can have multiple reasons; reason totals must not be summed as a + disjoint partition. +- `summary`: counts and original design-weight mass for the whole cohort, each + eligibility mask, the joint-fit complement and each exclusion reason. + +The shared names live in `property_income_constants.py`. `PROPERTY_COMPONENTS` +is, in order, `property_ordinary_interest`, `property_retirement_interest`, +`property_dividends`, `property_broad_receipts`. `PROPERTY_REPORTED_TOTAL` names +`property_reported_total`. They are donor observations or named routing +contributions, never already-reconciled model draws. + +## Measurement bridge + +Ordinary interest uses qualified `TRDINT_VAL`; dividends use qualified +`DIV_VAL`; broad signed property receipts use qualified `RNT_VAL`. The latter +is not a pure rental-income tax leaf. Dividends need a qualified yes/positive +or no/zero receipt. A receipt-yes zero for dividends or signed property stays +unknown under the published none-or-NIU coding. + +Retirement-account interest uses the two qualified account slots. Known no +with both slots NIU and zero supplies the already-qualified zero contributions. +Known yes requires at least one declared account with a known positive amount. +Every other slot must be another known positive account or an explicitly unused +slot: readable account code 0, published amount 0, in-range amount literal and +`unreported_account_slot` source status. The latter contributes a **derived** +zero and gets its own boolean, slot count and derivation label. Its original +canonical amount stays unknown in provenance. An amount zero by itself, +a declared account with unknown/zero amount, yes without any active account, +a missing code, or contradictory receipt/account evidence does not resolve the +retirement total. The derivation applies only when the complete route resolves. + +The reported donor total is separately retained as `INT_VAL + DIV_VAL + RNT_VAL`. +The component sum uses ordinary plus retirement-account interest, dividends and +broad property receipts. Both `interest_component_discrepancy` and +`reported_minus_component_total` remain visible, with signed magnitudes. No +balancing rewrites an ASEC observation. Allocation never filters the donor fit. +Disclosure flags do not prove the cause of a discrepancy. + +Other-income clearance requires known nonreceipt with a NIU category, or a +readable reported category outside property codes 5–8 and unspecified code 19. +A reported outside category resolves its route even if its own dollar amount +is ambiguous; no other-income amount is included in this basis. Missing, +unreadable or contradictory routing stays unresolved. The dividend qualifier's +nullable survivor clearance is checked against its readable codes and retained +as `survivor_visible_routes_clear`. It covers only `SUR_SC1` and `SUR_SC2`. +The 2025 dictionary describes `SRVS_VAL` as including edited sources 1/2 plus +unedited sources 3/4 (PDF page 49, printed page 6C-28); those additional source +types are not qualified here. A positive survivor receipt therefore cannot +establish complete absence of estate/trust income even when both visible +slots are outside codes 8 and 10. + +The first bridge accepts only known survivor nonreceipt with readable NIU +slots, exposed as `survivor_full_scope_clear`. Positive receipts have a separate +`survivor_additional_sources_unresolved` exclusion and weighted coverage row. +Possible visible property overlaps and unknown visible routes retain their own +exclusion reasons. Overlapping exclusion rows must not be added together. +Neither other income nor survivor amounts enter the component or reported sum. + +`reported_total_eligible` requires age 15+, a known finite reported total, and +clear other-income routes and complete survivor scope. It is a diagnostic sample, not a second +aggregate model. `joint_component_fit_eligible` additionally requires all four +components and an exactly zero interest discrepancy. Missing or under-15 values +are never turned into analytic zeros. Negative and zero net totals remain valid: +`(100, 20, 0, -120)` has total zero and keeps both positive interest components. + +The summary reports `design_weighted_person_mass` (one mapped original household +design weight per selected person) separately from `union_household_design_mass` +(each household with at least one selected person counted once). Overlapping +selection groups can share household mass. Stable finite summation avoids +row-order-dependent loss of small weights. These are coverage diagnostics, not +calibrated population claims. Conditional donor support, bias from exclusions +and held-out model quality still need the model owner's diagnostics. + +## Source and acceptance boundaries + +The source contracts use the pinned +[2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +(SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`). +The bridge targets the broad property concept in the +[2024 ACS questionnaire](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18). +The live ACS guide follows the 2025 form; its retirement-interest wording and +the historical 2016 Census methods discussion corroborate the concept without +establishing vintage or respondent identity. ASEC calendar-year and ACS +rolling-year measurement remain different. Broad property routing and +retirement-account earnings remain distinct from taxability. + +The bounded tests use invented tables and a composition test through the real +pure source projectors using invented literals. They cover both identity axes, +permutations, missing/contradictory routes, unknown account slots, signed and +zero-net examples, separate masks, allocation preservation, weight mapping, +empty/zero-weight records and refusal of malformed/nonfinite descriptions. +They do not run source qualification, native data, the country model, fitting, +cloning or replay. The later graph owns joint draws and signed reconciliation; +the later host must preserve original ASEC observations and share survey draws +across original/PUF clone pairs before the PUF alternative takes ownership. diff --git a/docs/us-current-asec-retirement-basis.md b/docs/us-current-asec-retirement-basis.md new file mode 100644 index 000000000..890b15df8 --- /dev/null +++ b/docs/us-current-asec-retirement-basis.md @@ -0,0 +1,164 @@ +# ASEC retirement candidate ledger + +`current_asec_retirement_basis.py` describes potential retirement-bridge support +from the accepted retirement-detail and income-routing projections. It preserves +source observations, route uncertainty and published accounting differences. +It produces no fiscal inputs, fitted model, graph host or source authority. + +## API + +`build_asec_retirement_basis` accepts keyword arguments `retirement_detail`, +`income_routing`, `original_household_membership`, +`original_household_design_weights`, and `assumptions`. The two DataFrames must +have identical ordered int64 `person_id` indices, exact native person IDs, and +original interview ages. Shared amount observations and pension receipt codes +must agree. A joint permutation is accepted; a one-sided permutation refuses. +The current contract requires at least one selected person. + +Membership must cover exactly those selected person IDs in the same order. +The weight Series must have a unique int64 `household_id` index containing +exactly the households represented by that membership, with finite nonnegative +values. Extra unselected household keys refuse, including keys from a larger +original survey. The caller must select the exact original household weights +without replacing them with clone, importance, or calibrated weights. Original +weight custody remains external; a Series does not authenticate DESIGN status. + +It returns `AsecRetirementBasis` with six descriptive fields: + +- `person`: source identifiers, observed routed subtotals, candidate family + intervals, accounting differences and candidate-support flags. +- `slots`: ten records per person, retaining six pension/disability/survivor + slots and four applicable/off-route distribution slots, source codes, + published amounts, knownness, source statuses and bridge routes. +- `provenance`: detached, prefixed copies of both complete supplied projections, + including allocation and disclosure flags. +- `exclusions`: uncertainty and route diagnostic masks, including cases that + remain numerically usable. These masks are not all applied as donor filters. +- `summary`: counts and supplied original DESIGN-weight support for each + candidate tier and diagnostic mask. +- `assumptions_payload`: canonical immutable bytes identifying the choices. + +The source projection dataclasses and these outputs contain mutable descriptive +tables. Neither their type nor a digest grants source admission. A future host +must retain and requalify the actual original source owners, complete person +scope, table seals and original DESIGN weights around its relevant I/O. This +pure function does not write to either input, calibrate weights or attach clones. + +## Required choices + +Every `RetirementCandidateAssumptions` field must be supplied. No country caller +or development scenario is enabled by this change. + +| Field | Supported values | +| --- | --- | +| `pension_annuity_regularity` | `unresolved`, `assume_regular` | +| `disability_pension_eligibility` | `unresolved`, `assume_qualifying` | +| `survivor_annuity_overlap` | `unresolved` | +| `withdrawal_regularity_netting` | `unresolved` | +| `aggregate_accounting` | `exact_visible_balance_only` | + +`assume_regular` is a measurement scenario about pension/annuity regularity. +`assume_qualifying` explicitly assumes both the disability-pension scope and +regularity needed for the candidate bridge. Neither is inferred from an annual +amount, age, `DIS_HP`, or `DIS_CS`. The latter source answers remain provenance. +There are no private-pension, regular-withdrawal or taxable fractions. Changing +either supported scenario changes `to_bytes()`; unsupported choices refuse. + +## Routes and candidate amounts + +The accepted [retirement-detail source](us-current-asec-retirement-detail-source.md) +and [income-routing source](us-current-asec-income-routing-source.md) retain the +published codebooks. This ledger imports those definitions and applies a separate +bridge route classification: + +| Family | Candidate routes | Separate or unresolved routes | +| --- | --- | --- | +| Pension | Codes 1–6 | 7 Railroad; 8 unresolved | +| Disability | Codes 2–5, subject to the explicit scenario | 6 Railroad; 1/7/8/9 other compensation; 10 unresolved | +| Survivor | Codes 1–4 are visible candidate subtotals | 5 Railroad; 8 property; 9 annuity overlap; 6/7/10 unresolved | +| Distribution | Applicable account codes 1–7 retained individually | Regularity/netting remain unknown; account 4 is a type of IRA | +| Other income | 2/13 indicate additional retirement candidates | 1 Social Security overlap; 8 property overlap; 19 unspecified | + +The retirement ledger never consumes account-earnings amounts or adds Railroad +amounts to Social Security. Other-income amounts are retained separately and +never added to the candidate sum. Categories 2/13/19 leave combined candidate +scope unresolved; known other categories can remain outside that sum. These +routes do not establish any person's tax treatment. + +A source-known amount and a published literal remain separate. A declared +positive slot contributes to its observed routed subtotal. A readable unused +NIU slot can contribute an explicitly labeled structural zero to a numerical +comparison, but its source-known amount remains missing. The subtotal is only +a sum of the readable known slots; it never asserts that unknown slots are zero. +Yes-receipt dollar zeros stay ambiguous where the source domain says none/NIU. +Annuity preserves its distinct valid dollar zero and published `-1` NIU encoding. + +The four published total-minus-slot comparisons are recomputed and checked +against R1. `DBTN_VAL` compares the main distribution slots only; the applicable +known distribution total is supplied independently by the routing owner. No +comparison is allocated to another source or attributed causally to disclosure. +For age 58 and over, the main slots apply and this exact comparison must be +finite and zero before a distribution candidate interval is available. A +positive residual leaves additional scope unresolved; a negative residual is +contradictory accounting. Both retain the published DBTN total, applicable slot +total and signed difference without a cap, clipping or redistribution. The main +slot comparison does not constrain the separate applicable young slots below 58. + +For pension/disability candidate bounds, all relevant slots must be readable and +receipt-consistent, and their exact integer-dollar sum must equal the aggregate. +Positive discrepancy leaves additional scope unresolved; negative discrepancy +is contradictory accounting. Neither is clipped, used as a cap, or silently +allocated. Readable outside routes are subtracted only under this exact balance. +With unresolved regularity/scope the lower bound is zero; under the named +scenario it includes only known candidate-route amounts. Unresolved source +routes can widen the candidate interval without becoming identified labels. + +Positive survivor receipt keeps full survivor bounds unknown, even when its +published comparison balances. Visible slots do not prove the scope of sources +3/4. Only qualified no receipt with two NIU slots and zero total supplies a +zero survivor family amount. Source 9 alongside positive annuity is explicitly +flagged; no overlap priority or max/sum assumption is applied. + +A fully resolved applicable distribution composition produces a zero-to-total +candidate interval only when the applicable aggregate check also passes. +Missing/unreadable account types, ambiguous active zeros or off-route +answers/dollars prevent that interval. Off-route receipt and account literals +must be readable NIU codes, and off-route amounts must be readable zeros; +unreadable literals are uncertainty, not evidence of absence. The routing owner's +known total is still preserved when the stricter candidate test fails. + +Routed subtotals and distribution account subtotals are evidence only, not +admitted candidate amounts. In particular, account subtotals can be finite when +all account types/amounts are readable but the applicable receipt is unreadable +or the DBTN comparison fails. `distribution_account_composition_known` describes +that account evidence; it does not establish a valid receipt, aggregate balance, +regularity, netting, or an available interval. Consumers must use the candidate +interval availability and family statuses rather than infer admission from a +finite subtotal. Raw observations remain in the detached provenance regardless. + +Under-15 people retain their source evidence with outside, unresolved-outside or +contradictory-outside status and no candidate zeros. Combined bounds are available +only when every family and other-income scope passes. `candidate_basis_eligible` +means this conservative numerical support exists. `candidate_point_under_assumptions` +means the candidate bounds coincide; `point_identified` and +`fiscal_outputs_produced` remain false. No training label is created merely by +calling a quantity an interval endpoint. + +## Source and validation boundaries + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +provides the source categories and domains. The [2024 ACS subject definitions, +page 95](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf#page=95) +provide the target's regularity and disability scope. The ledger's conservative +accounting and scenario choices are separate decisions. It does not equate the +surveys' reference periods, allocate joint income, or derive source regularity +or tax treatment from those documents. + +Tests use the actual pure R1/routing projectors on invented records. They check +source routes, exact accounting, unknowns, assumption identities, interview-age +routes, native IDs above `2**53`, one-sided mutations, permutations, detached +provenance and descriptive design-weight support. Person mass uses the supplied +original household weight for each included person; household mass counts each +included household once. The Series has no independent weight-kind authority. +Allocation/disclosure flags do not filter candidates. No native data, source +capture, engine, fitting, calibration, PUF or release acceptance is involved. diff --git a/docs/us-current-asec-retirement-detail-source.md b/docs/us-current-asec-retirement-detail-source.md new file mode 100644 index 000000000..e125201e4 --- /dev/null +++ b/docs/us-current-asec-retirement-detail-source.md @@ -0,0 +1,108 @@ +# Current ASEC retirement details + +`current_asec_retirement_detail_source` qualifies pension, disability and +survivor source details from the current ASEC member owned by an authenticated +survey preparation. This is source evidence for a later ACS retirement bridge. +It produces no model, tax leaf, payment-frequency classification or new issuer. + +## Source and field ownership + +The source is the [2025 ASEC public dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`, +for calendar-year 2024 income. Page references below are physical PDF pages; +the corresponding printed pages are 6C-23 through 6C-39. + +| Amount fields | Qualification | +| --- | --- | +| `PNSN_VAL`, `ANN_VAL`, `DST_VAL1/2`, `DST_VAL1_YNG/2_YNG` | Reuse immutable printed domains from the existing income-routing owner. Compare exact retained current-money values and validity. They remain reference observations here; their canonical receipt/account interpretation stays with that owner. | +| `DIS_VAL1/2` | Explicit dictionary domains on page 44, also compared against actual retained current-money values and validity. | +| `PEN_VAL1/2`, `SUR_VAL1/2`, `DSAB_VAL`, `DBTN_VAL`, `SRVS_VAL` | Independently qualify literals from the exact authenticated source member using published width, range, universe and zero semantics. These seven fields are absent from the current retained-money roster; the qualifier never fabricates a `ready.field()` for them. | + +The live money domains must agree with the eight existing fields. A future +money roster containing one of the seven independent fields fails until the +corresponding retained comparison is reviewed. The amount-entry cache contains +immutable entries; each public call returns a detached mapping. + +The complete source join verifies native person keys, household IDs, person +line numbers and source ages. Selected original ASEC persons retain their +actual composed person IDs. No clone or PUF population is modified. The source +literal table covers the complete current source roster; the person table +covers selected ASEC persons. + +## Knownness and comparisons + +`PEN_YN`, `DIS_YN` and `SUR_YN` are separate from their two source codes and +amounts (pages 44–45, 47 and 49–50). A positive readable amount is canonical +only with a valid positive source code and yes receipt. A no receipt, NIU source +slot and zero amount yields explicitly derived known nonreceipt. A yes receipt +with a declared source and zero amount stays ambiguous. A yes receipt with a +zero source slot and amount stays unreported, not an observed analytical zero. +Missing, malformed, out-of-range and contradictory values stay distinct. + +People under 15 never acquire analytical zeros. Unreadable outside-universe +answers stay unresolved; readable nonzero outside-universe evidence is flagged +as contradictory. `DIS_CS` and `DIS_HP` are preserved source answers. The latter +includes limited work as well as prevented work and does not establish the +ACS disability-pension criterion. + +Four raw numerical comparisons retain total-minus-slot differences: + +- `PNSN_VAL - PEN_VAL1 - PEN_VAL2`; +- `DSAB_VAL - DIS_VAL1 - DIS_VAL2`; +- `SRVS_VAL - SUR_VAL1 - SUR_VAL2`; +- `DBTN_VAL - DST_VAL1 - DST_VAL2`. + +These comparisons include readable published none/NIU zeros; they are not +proof of analytical completeness. Missing or malformed components produce an +unknown comparison. No difference is allocated to a pension, taxable income +or missing source. `DBTN_VAL` is retirement distributions, not disability, and +its printed formula uses the main slots rather than young-person slots. + +`SRVS_VAL` includes edited sources 1/2 and unedited sources 3/4 (page 49). +Its visible pair is therefore never declared exhaustive. Source labels retain +Railroad Retirement, estate/trust, annuity and unspecified categories without +assigning them to an ACS aggregate. In particular, positive survivor receipt +with two visible non-property routes does not prove the absence of additional +estate/trust income. + +Allocation and top-code flags preserve their own published domains. Source +allocation flags with values 0/1/9 do not acquire all intermediate header-range +values. `I_SURVL2` retains its printed `SURV_VAL2` universe spelling. Allocation +does not invalidate an otherwise observed amount; a disclosure flag does not +explain a particular component discrepancy by itself. + +## Why the ACS bridge remains separate + +The [2024 ACS questionnaire, question 43g](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +asks about regular account withdrawals, while [ASEC Q98Ar](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf#page=205) +asks about any withdrawal/distribution. “Regular IRA” is an account label, not +payment frequency. Account type alone does not establish taxability. The +qualifier assigns neither regularity nor tax treatment and makes no ACS +retirement-component allocation. + +## API and validation boundary + +`project_retirement_detail_literals(DataFrame)` is a descriptive pure +projection. `qualify_current_asec_retirement_detail(preparation)` requires the +actual preparation and original source ownership, captures the pinned member, +checks source identity and retained money, and requalifies the parent after +the last source I/O. Its last checks include actual owner identity and a full +physical result seal. + +The retained-money comparison follows the existing owner's annuity encoding: +published `ANN_VAL=-1` matches normalized positive zero with `DECLARED_NIU`, +while the descriptive literal and published amount remain `-1`. A dollar-zero +literal cannot substitute for that NIU/status pair. Other valid dollar amounts +retain exact float64 comparison. This distinction is tested through actual +invented source preparation, as well as direct mismatched-status controls. + +`retirement_detail_values_seal(values)` covers table axes, dtypes, exact values, +nullable Float64 backing storage and masks, literals and detached evidence. +The returned frozen dataclass contains mutable descriptive tables and grants no +source admission. A consuming host retains/revalidates the actual parent and +the result at its own boundaries; a copied dataclass is not an issuer. + +Tests use only invented records and the real source-preparation path. They +cover source joins, retained versus independent fields, member-byte drift, +late parent mutation and result mutation beneath null masks. No native source +or country engine is needed, and passing tests do not certify a release. diff --git a/docs/us-current-property-income-sources.md b/docs/us-current-property-income-sources.md new file mode 100644 index 000000000..b722a1218 --- /dev/null +++ b/docs/us-current-property-income-sources.md @@ -0,0 +1,109 @@ +# Original property-income source composition + +`qualify_current_property_income_sources` composes the source branches for the +property-income model. It borrows the existing shared-predictor preparation, +qualifies the original ACS anchors and ASEC interest, income-routing and +dividend/survivor observations, and applies the reviewed pure ASEC donor basis. +It fits no model, supplies no tax split and performs no clone attachment. + +The arguments match the existing shared predictor boundary: the actual +preparation, allocated population and clone population, with optional shared +demographic conditioning and geography configuration. The allocated/clone +objects satisfy that existing ownership contract. Their weights never supply +the donor weights. All donor membership and DESIGN weights come from the +original source Frame selected by the shared qualifier. + +The result is descriptive `QualifiedPropertyIncomeSources`. Its source values, +source Frame and full origin document are retained for inspection and later +source/clone joins. The object has no issuer registry and cannot replace the +actual retained preparation. The complete descriptive seal is a mutation +check, not a credential. Downstream owners must qualify before consumption +and after their last relevant I/O before returning or exporting a successor. + +## Selected branches + +`donor_frame` is the original ASEC DESIGN Frame restricted to jointly eligible +persons, with referenced groups pruned by the existing Frame selector. +`donor_columns` contains the qualified shared features, reported property total +feature and four original components. It uses the same exact ordered person +IDs as that branch. The four component columns and reported total import their +canonical names from `property_income_constants.py`. + +`recipient_frame` selects original ACS persons with a finite, known adult INTP +anchor. `recipient_columns` contains the same shared features plus the qualified +ADJINC-adjusted anchor under the reported-total feature name. The existing +recipient matrix codec preserves their ordered identities and float64 bits. +`recipient_matrix` is the encoded matrix. Source AGEP and canonical shared +feature age remain distinguishable; no raw age is overwritten by the feature. +Shared earnings features preserve the existing qualifier's declared NIU operator; +this composition adds no new zero completion for earnings or property anchors. + +Adding these model columns to a graph population is a later visible graph +operation. The selected Frames themselves keep all original source columns, +weights, memberships, strata and metadata. The source composition does not +quietly change a Frame's tax leaves or model columns. + +The complete original selected source axis is never reduced in `origins` or +`source_frame`. `origin_document` preserves all original entity origin records; +`donor_basis` contains every original ASEC row's component knownness, provenance, +exclusions and design-weight summaries. `recipient_diagnostics` preserves every +selected ACS row's raw literals and parsed anchor metadata, plus separate +under-15 and unknown-anchor exclusions and the eligible-recipient mask. + +An empty eligible branch has an empty indexed columns table and `None` for +its Frame, and no recipient matrix when no ACS recipient is eligible. Complete +diagnostics remain available. The existing Frame selector and matrix codec +require nonempty inputs; the composition does not manufacture a record or zero +to satisfy them. A later model host must refuse fitting an empty branch or make +an explicitly reviewed completion decision. + +## Exact identity and lifetime + +The complete shared source Frame, its origins and the original preparation's +person origin records must agree in order, source channel and native identity. +Each ASEC qualifier must exactly match the selected ASEC person and native-ID +axes; the pure basis also checks their source ages. ACS anchors must match the +selected original ACS axis. Donor feature and recipient matrix axes are checked +before selection. No sorting, inner join or missing-key fill hides a mismatch. + +The donor household design weights are mapped by exact original membership. +Their person mapping must match the original Frame's resolved DESIGN weights; +importance or clone weights are never substituted. This is source construction, +not engine microsimulation aggregation. + +Each initial qualifier's complete physical description is sealed as soon as it +returns, and checked while subsequent qualifiers do I/O. After pure composition, +all five original source qualifications are repeated and compared to those +seals. Every returned value, including nullable data beneath missing masks, +is checked throughout. The actual preparation is then rechecked after the last +qualifier's I/O. The final checks are pure: retained issuer identity, nested +original source Frames, allocated/clone support, the exact optional geography +configuration payload and all complete derived seals. + +The income-routing qualifier has no standalone physical-seal API, so this +composition uses the existing full table-stamp helper over its complete person +and original-literal tables plus evidence. Existing shared/ACS/interest/dividend +seal APIs are reused directly. No source-reader implementation or new authority +framework is introduced. + +## Verification scope + +The invented fixture creates closed source bytes before issuance, then executes +the actual survey source graph and source composition. It includes a negative +ACS anchor, malformed adult anchor, under-15 blank anchor, a resolved ASEC +ordinary/retirement-interest route, and excluded ASEC overlap/unknown cases. +Controls check ordered axes, weight kind, adjusted bits, all returned seal +surfaces and copied preparation refusal. A separate injected mutation isolates +the final original-source I/O fence using previously checked descriptive values; +it does not claim another complete source replay. + +These checks run without native microdata, QRF, PUF or a country engine. Model +quality, donor conditional support, country financial integration, both-clone +fanout, replay and release acceptance remain later boundaries. + +The initial composition acceptance used donor basis `2c959096eec0441e2acd975e63522aac7f1b7016`. +A subsequent primary-source review found that survivor slots 1/2 do not establish +complete absence of estate/trust overlap: the aggregate can include sources 3/4. +The country host must adopt the separate conservative donor-basis correction +before using this branch for fitting. The initial test's eligible donor reports +no survivor income, so its observed values do not rely on a yes-survivor clearance. diff --git a/docs/us-current-survey-amount-successor.md b/docs/us-current-survey-amount-successor.md new file mode 100644 index 000000000..70d0b28ae --- /dev/null +++ b/docs/us-current-survey-amount-successor.md @@ -0,0 +1,236 @@ +# Current survey amount successor + +## Optional workers compensation, 20 September 2026 + +`run_us_survey_enrichment(..., groups=("unemployment", "health_costs", +"workers_compensation"))` adds the person input `workers_compensation`. The +default remains the first two groups. The new route borrows the genuine +preparation and captures the same authenticated current ASEC member, joining +`WC_VAL`, `WC_YN`, age, and native person/household/line coordinates to the +existing current-money owner. It uses the maintained strict direct amount +mapper on qualified values. A carried column or detached projection cannot +establish source authority. + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +PDF page 51 (printed 6C-30), asks the WC receipt question of people aged 15+. +`WC_YN` is 0 for NIU, 1 for yes, and 2 for no. `WC_VAL` is asked of recipients; +its zero code means none or NIU. The question includes workers compensation +and other job-related injury or illness payments. This route preserves that +published concept, without selecting a subset by `WC_TYPE`. + +In-universe yes/positive and no/zero pairs retain their canonical amounts. +NIU, under-15 observations, yes/zero pairs, contradictions and missing or +unrecognized responses retain their raw values and unresolved canonical +amounts. No child-income zero convention is introduced. Published known +amounts are not claimed to be unallocated responses. + +The existing amount graph extends its source projection and declares WC-specific +original ASEC design-weight donor selection, fit and ACS recipient draw before +the shared clone attachment. +Only eligible ACS originals aged 15+ receive a draw; each original draw is +shared by both support clones. Both ASEC clones retain the same source amount +and reporting knownness. All prior columns, typed weights, identifiers and +memberships remain under the existing parent checks. The current source +predictors and 2025 ASEC / 2024 ACS transport choice are unchanged. + +Bounded invented verification exercises genuine source capture and refusal +after source mutation, then the actual declared QRF fit/apply nodes, required +cache replay and source-keyed clone attachment. The numerical component test +uses explicit invented producers; it does not issue a native enrichment owner. +Combined host acceptance with the other new fragments, actual-source coverage, +held-out fit quality, calibration and release qualification remain separate. + +Implementation plan and source inventory, 12 September 2026. This is a local +successor to the retained survey PUF55 run, not an amendment to the frozen d35 +native pilot. The branch starts at reviewed integration 99543c3. + +## Immediate scope + +| Native ASEC field | Canonical person input | Reporting evidence | +| --- | --- | --- | +| UC_VAL | unemployment_compensation | UC_YN receipt code; amount question universe UC_YN = 1 | +| PHIP_VAL | health_insurance_premiums_without_medicare_part_b | All persons | +| PMED_VAL | other_medical_expenses | All persons | +| POTC_VAL | over_the_counter_health_expenses | All persons | + +These four names are absent from both PUF tax-detail ownership rosters. They +already have qualified current-money domains and direct amount mappings in +`cps_carried_current.py`. The complete legacy carry helper is not reusable: +it also writes existing PUF leaves and a legacy Social Security decomposition. +The modern `survey_social_security` source owner remains the authority for +Social Security; no age-62 zero fallback or prior wages is introduced here. + +The current ASEC source is the authenticated original 2025 person member, +income year and price basis 2024. ACS observations use the already qualified +2024 source predictors, whose earnings refer to the rolling prior 12 months. +That temporal harmonization is a documented model choice, not equivalent +observation windows. No new donor dataset is needed. + +## Amount and reporting status are different + +The [2025 CPS ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +PDF page 50, defines UC_YN codes 0 (NIU), 1 (yes), and 2 (no), with the receipt +question asked of persons aged 15 or older. UC_VAL is asked of recipients; +its zero code conflates none with NIU. The maintained domain metadata pins +this dictionary to SHA256 +`5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The three health amounts have all-persons universes (PDF page 83). + +A small bounded UC_YN projection reads the same exact authenticated original +member. Native person and household/line coordinates must join the original +current-money owner exactly; carried columns and detached JSON do not confer +source authority. The projection retains the original amount, amount validity, +source classification, zero_origin, literal receipt code and reporting status. +Today's upstream preparation requires complete qualified current money before +issuing its handle. Literal missing-amount controls test the join's unknown +semantics; they do not claim that a missing amount passes this unchanged parent +prerequisite. Reporting unknownness can still occur with numerically valid amounts. + +UC attachment and donor policy: + +- A valid positive amount with an in-universe yes code is known receipt. +- A zero amount with an in-universe no code is known nonreceipt. +- Yes with zero is an ambiguous amount, not known nonreceipt. It remains in + the source projection but is excluded from fit and canonical attachment. +- NIU, blank/unrecognized codes, missing amounts, and contradictory amount/ + receipt pairs remain unresolved and are excluded from fit and attachment. +- Out-of-universe age does not infer a zero. ACS persons outside the receipt + question universe also retain unknown UC rather than receiving an adult draw. + +The health fields retain qualified valid all-persons amounts, including genuine +zero-dollar codes. No noanswer cell is converted to zero. Allocation and source +status remain evidence; known source values are not claimed to be unallocated. +PHIP_VAL is deliberately retained as the declared premium source, rather than +silently substituting PHIP_VAL2, whose treatment of inconsistent reported zero +premiums differs. The [Census health insurance user note](https://www.census.gov/programs-surveys/cps/technical-documentation/user-notes/health-insurance-user-notes/User-Notes-Selected-CPS-ASEC.html) +describes that distinction. Person-level conditional fitting preserves modeled +cross-component dependence; it does not certify household or insurance-unit +correlation, which still needs empirical validation. + +## One family, explicit graph operations + +The family has one parameterized source projection and attachment mechanism. +Each selected field declares its source transformation, donor eligibility, +recipient matrix, QRF fit and draw as graph nodes/artifacts. UC has its own +reporting mask. The health group uses an ordered PHIP → PMED → POTC conditional +chain, retaining observed health-component dependence through prior-target +conditioning instead of fitting the three amounts independently. This avoids +filling unknown UC to make a joint matrix finite. Fits use the existing regime-gated QRF implementation and +current source predictor qualification; donor weights remain original design +weights before allocation or cloning. No PUF value becomes an ASEC donor. + +Draws are keyed to the original ACS source person, then deliberately shared by +its original and PUF support clones. This preserves source observations and +paired support interpretation; clone labels are checked during attachment. +Source amount values and reporting status are copied to both ASEC clones. +Every existing parent column, owner, axis, membership, atomic geography, +weight, design anchor and mass ledger must remain identical. + +The parent is the exact `SurveyPuf55Run` checked by +`check_survey_puf55_run`. The original handle is retained and checked before +consumption and after source/store/export I/O. A checked JSON payload or copied +dataclass cannot issue authority. The extension copies kernel/source-codec +registries privately, preserves all prefix declarations/keys/artifacts, and +verifies actual observed populations on cold and required replay. Failed checks +revoke the original handle. The checked-output branch supplies this additive +API; the frozen native packet is left untouched. + +## Verification + +First run cheap invented controls: all UC code/amount/universe combinations, +health zero and unknown behavior, literal bounds and source-coordinate joins, +clone pairing, missing/duplicate source IDs, invalid source/clone channels, +ownership collisions and raw/status/zero-origin mutation. + +Then an actual invented authenticated-source fixture runs the family through +real graph cold and required replay, QRF train/apply, coverage, complete Frame +store/export/readback, parent retention and unchanged prefix seals. Rejection +controls mutate source projection, parent handle/population, declared graph, +artifact bytes, fit/draw lineage and output attachment independently. Acceptance +requires closed receipts and independent review. No native run, engine outcome, +held-out quality, calibration or release certification is claimed by these tests. + +## Remaining input inventory + +The accepted invented PUF baseline has 161 required names, of which 100 are +absent. Its 61 present names are not necessarily complete: some PUF leaves are +populated only on PUF clones. Counts are inventory for that fixture, not a native +coverage measurement. The next stage reduces absent names while reporting +unresolved cells separately. + +| Input family | Existing source producer or route | Integration requirement | +| --- | --- | --- | +| UC and three health amounts | authenticated current-money owner; direct `cps_carried_current` maps | This stage | +| Social Security components | `current_social_security_source`, `survey_social_security` | Preserve source reporting basis and unknown components; never legacy age split | +| Health coverage | Authenticated `current_survey_health_source` and explicit `graph_current_survey_health` fragment | This stage; seven narrow ACS concepts remain unknown where the survey cannot distinguish them | +| Disability, student status, children, veteran status | `eligibility_inputs` | Qualify literals and reporting universes before reuse | +| Parent relationships | `eligibility_inputs._own_children_in_household` resolves PEPAR1/PEPAR2 then drops links | Issue 884: retain parent_1_id/parent_2_id with household/line and clone joins; unresolved and absent differ; consumer issue 9404 is separate | +| Marital roles | `relationship_inputs` | Current source observations and clone preservation | +| Hours, work history, occupation, wage/tips | `hours_worked`, `org_wages`, `weeks_unemployed`, `sipp_tips` | Source/QRF qualification and explicit ownership; prior wages excluded | +| Child support, disability income, workers compensation | respective maintained runtime modules | Current source amount/universe qualification, then existing fit contracts | +| Housing, energy, care, education | `housing`, `energy_subsidy`, `adultcare`, education producers | Household/SPM source and clone-keyed joins; reconcile PUF tuition ownership | +| Financial wealth, vehicles, loans | SCF/SIPP producer modules | Actual donor authentication, existing fit dependencies and held-out quality | +| Retirement distributions/contributions | maintained distribution/contribution modules | Reconcile already PUF-owned taxable IRA/pension and self-employed pension leaves | +| Pregnancy, SSI disability, voluntary filing | maintained producer modules | Resolve source authority and distinct behavioral assumptions | +| Take-up | `take_up_contract` | Unsourced rates remain unresolved; no convenience constant fills | +| Mortgage details, marketplace benchmark ratio, remaining retirement details | Not yet mapped in this bounded inventory | Identify actual producer/owner or make an explicit source/model decision | + +The inventory used tracked files and `rg --no-ignore` within explicit source +directories because ordinary searches can skip the tracked build package. +Existing function names establish implementation leads; they do not prove that +those producers are integrated into the current graph or current native file. + + +## Fixed country composition + +`run_us_survey_enrichment` retains the original checked `SurveyPuf55Run` and +assembles the amount family and independently source-qualified health fragment. +Both attach to `survey_puf55.receiving`; the health CREATE and final attachment +explicitly consume `survey_amounts.attach/attachment`. Health adds nine coverage +inputs and their literal/status/knownness evidence after all four monetary leaves. +The host preserves the complete parent Frame, owners, design anchors and mass ledger, +compares every unchanged prefix version against the original parent manifest, and +requalifies both original-source projections before issuing its output. There is +one private issued-handle registry and one public checked country output, not a +family-level authority or executor. The health fragment's descriptive values do +not grant access to native sources. + +The combined invented fixture includes the exact source ACS HINS/flags and ASEC +NOW/flags before authenticating source owners. It runs the original financial and +PUF stages, both enrichment fragments, a required replay, input-coverage diagnosis +and complete Frame store/readback. No native build, model engine execution, +calibration, or release approval is implied by these checks. + + +## Invented acceptance, 12 September 2026 + +The final production source passed the 271-node combined graph: 245 validated +prefix hits followed by 26 newly executed enrichment nodes, then a required +replay with all 271 hits. Complete parent columns, owners, design anchors, +geography and the mass ledger were preserved. The full resulting Frame was +stored and read back with exact physical comparison. The fixture has 18 people; +required-name coverage improved from 61 to 74 of the 161-input profile, leaving +87 missing names. Presence does not imply complete values or statistical quality. + +All 18 people have the three health-cost values and an ESI observation. UC has +10 known and eight unresolved values. The narrow Medicaid-at-interview input has +eight known and ten unresolved values, preserving the ACS semantic gap. These +are counts from invented records, not estimates or native-data diagnostics. + +Acceptance combines five passing ordered tests from the third full attempt and +one corrected, focused final parent-revocation test. The third attempt retained +its failed result because that last test named a nonexistent generic income +column; it stopped before mutating anything. The final test uses the actual +`employment_income_before_lsr` input and verifies permanent parent and child +revocation after mutation, including refusal after restoring the original value. +It rebuilds a real parent and one child, without repeating the already-passed +required replay or readback. The focused attempt passed with all 857 Python +source pins stable, 624.872 seconds wall time and 885,669,888 bytes peak RSS. + +The separate 32-case cheap suite covers UC reporting classifications, literal +missingness, source/clone identity, ownership collisions and source-origin +serialization when `person_id` names both an index and a column. The serialization +binds the original int64 axis and its name separately from ordered column values. +Independent review covers the frozen UC source, health fragment, combined host +and this serialization correction. Preserved failed attempts remain part of the +local acceptance record; no native execution or release certificate is implied. diff --git a/docs/us-current-survey-household-roles.md b/docs/us-current-survey-household-roles.md new file mode 100644 index 000000000..a5bb2bf10 --- /dev/null +++ b/docs/us-current-survey-household-roles.md @@ -0,0 +1,47 @@ +# Current survey household reference-person roles + +The canonical `is_household_head` leaf means the observed reference person of a surveyed household. `current_survey_household_roles` qualifies that role from the retained ACS/ASEC preparation and original source owners. `graph_current_survey_household_roles` supplies a private source projection and a binding node on the existing population after cloning. This fragment is not yet wired into the country host or validated on native inputs. + +## Source meanings and unknownness + +| Source evidence | Qualified household role | +| --- | --- | +| ASEC `A_EXPRRP` code 1 or 2, every relationship token named, exactly one reference person in the published household | Reference person is true; other members are false | +| ASEC missing/unnamed relationship, no reference person or multiple reference persons | Whole affected household remains unbound, with the owner's reason | +| ACS housing-unit roster with exactly one `RELSHIPP` code 20 and all relationship tokens admitted | Reference person is true; other members are false | +| ACS missing/invalid/mixed housing-unit roster or incorrect reference-person count | Household role remains unbound | +| ACS group-quarters codes 37 or 38 | Published code remains known; housing-unit reference-person role remains unbound | + +The ASEC interpretation reuses `asec_demographic_source` and its pinned dictionary contract: `A_EXPRRP` has a 1–14 printed range, but code 6 has no named meaning. A value inside the range is therefore insufficient. The recorded field locators identify the 2023/2024/2025 dictionary pages 21/22/23 and the all-persons universe. Its owner proves the equivalence of published and prepared household memberships before classifying a role. `P_SEQ`, roster line number, family relationship, age and tax-unit position are not substitutes. + +The ACS interpretation reuses `asec_demographic_source.acs_household_reference_states`, reading original `RELSHIPP` tokens from the retained catalogue's pinned person archive. It reads whole selected household rosters so a selected subset cannot conceal another reference person or an invalid relationship. A group-quarters person is not an observed non-reference member of a housing unit; the named group-quarters code is retained separately from the unresolved canonical leaf. Synthetic household construction and the harmonized `A_EXPRRP` recode cannot supply reference-person evidence. + +These are the requirements already recorded by `graph_composed_asec_binding` for its unbound `is_household_head` column. Their exact immutable declarations live in the import-free `survey_demographic_contract` module; the historical composed stage re-exports the same names and still refuses its own canonical writes. The later role qualifier imports only the shared declarations, not the composed runtime. Its source identity and first/final live checks bind both complete shared tuples, while the old composed stage includes the helper in its existing implementation inventory. This fragment pins that requirement and refuses changes. It does not relabel the composed stage as having supplied this observation. + +Neither arm supplies relationship-allocation evidence through this fragment. The current ASEC readset has no allocation flag for `A_EXPRRP`; ACS `FRELSHIPP` is outside this fragment's readset. The public receipt records allocation provenance as unresolved and makes no respondent-reported or equivalent-evidence claim. + +## Binding and inspection + +The private projection retains exact original person identifiers, source relationship codes, role states, reasons, household verdicts and knownness. The binding fans each original onto its two initial clones using shared provenance columns with exact int64 identity. It writes only `is_household_head`; an existing leaf is an explicit rewrite on a population version below its CREATE node. The expected-output verifier reconstructs the full population through the maintained patch operation and compares all other cells, entities, relationships, weights, geography, provenance, owners and mass history through shared replay equality. + +A known incumbent must have a qualified binding and agree with it. Contradictions and unsupported known incumbents both refuse without changing the original value. This includes an artificial group-quarters head: it cannot become source-confirmed headship. `household_role_reconciliation` reports aggregate disagreement and unsupported-incumbent counts for separate adjudication. A null incumbent is filled only when the source role is qualified. Source-unbound nulls stay null. + +The qualifier accepts only the actual retained preparation type and registry entry. It checks source bytes, live functions, code objects, defaults, closures and contract constants around the first and final preparation/ACS owner callbacks. After the last callback and source identity read it checks retained buffers and independently reconstructs the projection. The graph kernels similarly bind the host callback, qualified inputs and detached output seals. These are bounded in-process integrity checks, not a general execution sandbox. Public artifacts contain only the declared aggregate receipt; source identities and rowwise evidence remain private. + +## Independent-minor audit + +This lane does not create a generic `independent_minor` or emancipation variable. Existing `spm_role_source.py` defines the distinct `is_spm_independent_minor_role`, reconstructed from `SPM_HEAD`, `A_FAMTYP` and `A_FAMREL`, before an age gate. Its documented consumer reconciliation uses SPM adult/child/person counts; it is not household reference-person status, tax dependency, filing status or statutory emancipation. + +`tools/build_us_spm_role_enrichment.py` calls `derive_spm_role_source` for the separate legacy BuildP source-enrichment contract described in [the SPM runbook](us-native-spm-role-source-enrichment.md). Those retained source/release checks do not establish authority for the new combined graph or its ACS arm. A future SPM lane needs an explicit graph owner, current complete-unit source reconstruction, unit-count reconciliation, clone mapping, and verified consumer input semantics. This source-only audit did not import a country engine or inspect installed model definitions, so it makes no new consumer-compatibility claim. No age-only, head-only or tax-unit approximation is introduced. + +## Validation scope + +The proposed tests use invented observations through the real classifier, clone binder, graph compiler/executor, ContentStore and cold/required replay. They cover GQ unknownness, unsupported/conflicting incumbents, exact large integer IDs, complete population preservation, public receipt boundaries and mutations at first/final callbacks. A separate file uses the maintained tiny source fixture and real preparation, ASEC demographic and ACS catalogue owners, including both group-quarters types. + +At this source checkpoint the tests are written but unrun. Source AST, lint and CI inventory checks establish neither execution success nor data certification. The runtime proposal requires separate review of source/resource maps and guard limits before either invented phase. Native qualification, country-host integration, calibration and release verification remain separate gates. + +The finite roster has 71 pure cases, 58 graph cases and 23 actual-owner cases. The import-boundary successor adds exact historical assignment-AST parity, compatibility re-export identity, actual module-body import interception, shared-file source sealing and shared-constant mutations across source and graph owner callbacks. The earlier 113-case phase A attempt stopped during collection because the old eager composed-runtime import reached an unadmitted schema probe; no substantive tests ran. That failed packet and held phase B remain preserved. The successor tests are authored but unrun; no schema/resource permission is added. Additional controls verify non-nullable Boolean storage, invalid incumbent dtypes, and detached coordinate/roster mismatches against actual retained ASEC and ACS owners. An unsupported known Boolean incumbent refuses before the final unresolved-Boolean defense; tests do not fabricate nullable cells inside non-nullable storage to force that later branch. + +The **64 MiB serialized private-artifact cap is an operative capacity limit**, independent of the 2,097,152-original-person ceiling. Per-row JSON size varies with identifiers and values, so the person ceiling is not a supported national capacity claim. The maintained ASEC demographic owner also caps its own full input at 600,000 people. The first applicable source, row or byte limit governs, and none has been validated here against a full national universe. Larger builds need a reviewed storage and resource plan rather than silently increasing limits or assuming the row ceiling can be reached. + +Roster lines supply coordinates only. The current 2025 ASEC dictionary prints `A_LINENO` as 01–16, while ACS `SPORDER` uses 1–20. The fragment's common 20-coordinate ceiling therefore does not establish the suggested failure for a legitimate current ASEC line above 20; this review found no such documented value and did not widen production bounds. The exact ASEC owner and coordinate comparison remain required. [ASEC 2025 dictionary, physical page 22](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf). diff --git a/docs/us-current-survey-housing.md b/docs/us-current-survey-housing.md new file mode 100644 index 000000000..cfa623939 --- /dev/null +++ b/docs/us-current-survey-housing.md @@ -0,0 +1,80 @@ +# Housing participation in the common US survey graph + +The common ACS–ASEC graph connects independent survey housing observations to +two country-model inputs: `receives_housing_assistance` and +`takes_up_housing_assistance_if_eligible`. Both inputs attach to the modeled +assisted family's SPM unit. The country model calculates program awards, +allocates those awards across SPM units and applies the SPM resource cap. + +`graph_us_survey_enrichment.run_us_survey_enrichment` includes this connection +after the current survey amount and health fragments. It uses the retained +source preparation and original common frame; it does not construct a separate +ASEC–PUF population or infer participation from `SPM_CAPHOUSESUB`. + +## Observations and modeled inputs + +The qualifier reads the authenticated original ASEC household and person +members and ACS household and person members already owned by source +preparation. The existing ASEC housing classifier distinguishes known receipt, +known nonreceipt, unknown, not-in-universe and conflicting responses. The +attached household evidence preserves literal fields, status, allocation +quality, knownness, source year and survey year. + +`housing_observed_receipt` remains nullable. `housing_receipt` is the completed +model input, and `housing_receipt__origin` records whether it came from an +observation, a modeled exclusion or imputation. A modeled value never changes +the source observation's knownness. + +The projection records these assumptions: + +| ID | Assumption | +| --- | --- | +| A1 | The household reference person's SPM unit represents the assisted family. | +| A2 | Interview-time participation carries through the preceding income year. | +| A3 | Public-housing and reduced-rent reports represent the modeled HUD assistance family. | +| A4 | ACS group quarters are excluded from this modeled household-assistance family. This is a modeled exclusion, not observed nonreceipt. | +| A5 | Independently observed owner households are excluded from this modeled public/lower-rent family. Positive or conflicting source answers cannot be overridden by this assumption. | + +ASEC `A_EXPRRP` and ACS `RELSHIPP` identify the reference person. Row order and +ASEC `P_SEQ` do not establish that role. A household with an ambiguous reference +role refuses attachment. Explicitly classified group quarters may lack a +reference person only when their completed modeled receipt is false. + +An owner household with positive or conflicting public/lower-rent answers +stops qualification with `OWNER_SOURCE_CONFLICT`. The owner exclusion cannot +silently override those source answers. + +## Household imputation and attachment + +Known ASEC households supply one binary label per household to the existing +regime-gated QRF operator. The donor frame retains complete households and +their original household design weights before geographic allocation and +cloning. Predictors are reference-person age, household size and household +employment and self-employment income. When the parent qualifies demographic +conditioning, the corresponding reference-person predictors also participate. + +The model fills only unresolved household participation cells. One draw per +original household attaches to both of its clones through explicit source and +clone identities. If every household already has a completed input, the graph +omits fitting and application. The connection preserves the receiving frame's +existing columns, entity memberships, weights and mass ledger. + +Only the reference person's SPM unit receives the two true participation +flags. Other SPM units sharing that household receive false. No program-dollar +or SPM-dollar variable is produced by this fragment. + +## Verification and release scope + +The host binds source projections, donor values and weights, recipient matrix, +fitted-model history and attachment outputs. It reconstructs each population +change during readback and checks live implementation and configuration around +owner callbacks. Mutating a retained observation, source mapping or output +invalidates the affected result. + +The focused tests use invented source files and Frames. They exercise source +knownness, household donor grain, reference-person routing, clone attachment, +cap independence and graph replay. These code checks do not establish +imputation quality on national data or a revised 2025 poverty rate. The +projection and attachment therefore carry `release_eligible: false` until the +separate native population, country-model, calibration and release checks are +complete. diff --git a/docs/us-current-survey-property-graph.md b/docs/us-current-survey-property-graph.md new file mode 100644 index 000000000..61c0c4a45 --- /dev/null +++ b/docs/us-current-survey-property-graph.md @@ -0,0 +1,55 @@ +# Current survey property-income graph + +The opt-in `graph_current_survey_property.py` fragment adds 16 operations to the existing US financial graph. It puts qualified ASEC components and modeled ACS components on the complete initial clone pair. The country financial host retains the original source owners, verifies every graph observation and artifact, and issues the result. The fragment creates no separate issuance registry. + +`PropertyIncomeOptions(scales=..., atol=..., rtol=..., n_estimators=...)` requires all four settings. The options object is frozen and has a canonical `to_bytes()` representation. No numeric modeling default is supplied. The first three components are nonnegative; broad property receipts can be signed. A negative or zero anchor can coexist with positive components and an offsetting negative broad-property component. + +## Visible operations + +| Operation | Meaning | +| --- | --- | +| `survey_property.source_projection` | Bind the actual qualified property composition, the existing financial predictor projection/matrix, original host artifacts, and geography validation when present. Emit a projection and exact donor/recipient matrices. | +| `survey_property.asec_eligible_donor` | Filter the original CREATE by the exact eligible ASEC person IDs. Preserve original household DESIGN weights, including their person mapping. | +| `survey_property.asec_donor_columns` | Materialize shared predictor columns, reported total, and four qualified components. | +| `survey_property.acs_eligible_recipient` | Filter original CREATE to ACS persons whose source anchor is known and in the adult reporting universe. | +| `survey_property.acs_recipient_columns` | Materialize shared predictors and adjusted INTP as the reported-total feature. | +| `survey_property.fit.000`–`.003` | Fit the existing DESIGN-weighted chained QRF over the four components. | +| `survey_property.apply.000`–`.003` | Draw once per eligible original ACS person, preserving ordered prior-component conditioning. | +| `survey_property.draws` | Retain raw draws and their complete model/application history. | +| `survey_property.reconcile` | Apply the shared signed-total reconciliation rule with the explicit scales and tolerances. | +| `survey_property.attach` | Join original source identities to both initial clones and retain all existing rows, entities, weights, geography, metadata, and legacy financial leaves. | + +Both model branches leave CREATE before importance allocation. Their FILTER operations return person keep masks, preserving actual raw source columns. Shared predictor materialization, including the existing analytical treatment of ACS earnings NIU, happens through declared column operations. The FILTER does not substitute the separately prepared feature frame for CREATE. + +The projection binds complete descriptive physical seals, including unknown masks and excluded rows, not just the finite fitting matrices. It carries the donor exclusion summary and recipient exclusion counts. No excluded person silently becomes a zero-valued donor or recipient. Empty eligible donor or recipient branches are currently refused before fitting; an empty-branch execution policy is not implemented. + +## Attachment and knownness + +The attachment owns the four `property_*` components, their four `draw_property_*` columns, `property_reported_total`, reconciliation adjustments, nullable active-bound diagnostics, residual and objective, two source-basis discrepancy columns, and two knownness flags. `owned_columns()` is the authoritative roster. + +ASEC persons retain each individually qualified component and reported total, including NaN for an unresolved value. Donor exclusion does not erase otherwise known amounts. The source-basis discrepancies remain available, and the complete source qualification/exclusion descriptions stay bound to the projection. ASEC raw-draw and reconciliation diagnostics are absent because no draw or reconciliation is performed on these observations. + +For example, a qualified survivor estate/trust route can exclude an ASEC person from the joint four-component fit while that person's ordinary interest and dividends remain known. The attachment copies those independent amounts to both initial clones. The separate tax-leaf successor splits each known amount independently; the donor exclusion does not turn it into an unknown tax input. A focused regression exercises this path with authenticated invented survey files and the real attachment and split functions, without fitting a QRF or rerunning the complete financial host. + +Eligible ACS persons receive one ordered joint draw and its reconciled components. Their original `person_id` and pandas row index are checked separately, then the original support ID, native source ID, source label, and exact clone roles establish the two-row fan-out. Excluded ACS persons receive NaN for model components and the attached anchor, with false `property_anchor_known` and `property_components_known`. Their original source anchor status remains in the qualified source projection; under-15 values are not silently converted into a new analytical zero. + +`property_anchor_known` describes the attached anchor's finite value. `property_components_known` describes all four attached components being finite; it is distinct from eligibility for donor fitting. Reconciliation bound flags use nullable booleans so an unperformed reconciliation does not become a false observation. + +## Tax and period boundaries + +The eight existing financial tax leaves remain unchanged. Attachment reads all eight and explicitly follows the legacy financial attachment, checks them against its retained draw history, then adds property columns. The legacy capital-gains chain still conditions on legacy INT and DIV draws. The receipt records `tax_split_rebased: false` and the capital-gains limitation. `property_legacy_interest_draw_minus_reconciled_interest` measures the legacy ACS interest draw minus the reconciled ordinary plus retirement interest; it is a diagnostic, not a new tax assignment. + +Broad property receipts are not identified with rental income or a tax-law leaf. Retirement-account earnings are not retirement withdrawals. Tax-leaf rebasing and a CAP model conditioned on reconciled components require a separate declared successor. + +Periods remain those of the qualified sources: ASEC 2025 interview and calendar-2024 income; ACS 2024 rolling prior twelve months, adjusted by ADJINC to the declared price basis. No equivalence of the reporting windows is asserted. The person-level conditional chain is not a certification of household-level dependence or held-out fit quality. + +## Host integration and verification + +- `current_survey_property_nodes(qualified, clone_frame, host_pins=..., options=...)` declares the fragment. +- `register_property_kernels(...)` registers the country and shared numerical kernels over the retained source owners. +- `reconstruct_property_results(..., artifacts=..., legacy_matrix_producer_key=...)` returns eight expected non-fit results: projection, two filters, two model-column operations, draws, reconciliation, and attachment. The host patches these at their declared population versions and checks actual observations. The separate model-receipt verifier binds the eight fit/apply receipts to actual donor/recipient populations and stored model bytes. +- `verify_materialized_property_income(..., legacy_population=..., population=...)` requalifies actual source owners and compares the full final population against attachment applied to the independently checked legacy population. + +Reconstruction validates the complete ordered raw-draw history, training-model references, recipient index, person IDs and summaries. It invokes the existing deterministic reconciliation rule to derive the expected reconciliation columns; it never fits or draws another model. Store artifact type, producer-key and key validation remains a prerequisite owned by the host. The host must retain its existing final source, implementation, cache, full-population and issuance checks after relevant I/O. + +Changing only scales affects the reconciliation and attachment declarations, leaving source selection and QRF fitting declarations unchanged. The absence of property options in a host call preserves the existing strict legacy path. Graph replay validates cached output observations as well as labels; a cache hit is not source admission or a release verdict. diff --git a/docs/us-fact-to-target.md b/docs/us-fact-to-target.md index 5a4f56a55..b8a4b9fed 100644 --- a/docs/us-fact-to-target.md +++ b/docs/us-fact-to-target.md @@ -30,9 +30,11 @@ Rationale: microcosm replaces the survey's tax-benefit measurement with imputed, computed, and admin-calibrated values — that is the product. Fitting a survey-derived tax-benefit quantity launders the measured-with-error version back in and destroys the held-out validation -signal (the scorecard's win column is held-out-only for the same reason). -Release gates may *fail* a certification on a held-out poverty regression; -*fitting* the statistic is categorically different and prohibited. +signal. Poverty comparisons must also stay outside candidate-selection scores +and release gates based on agreement with a survey estimate. Holding a measure +out of the calibration objective does not justify tuning toward it afterward. +Do not choose weights, imputations, take-up assignments, target deferrals or +hyperparameters because they bring poverty closer to the published rate. For raw survey margins, prefer an administrative source when one covers the same cell and concept — e.g. congressional-district income binds from @@ -46,12 +48,12 @@ because all of its facts are state-grain, while the compiler admits ACS only at congressional-district geography (`tools/build_us_target_parity_manifest.py::_FAMILY_EXCLUSIONS`). -**Corollary: deviations from official poverty metrics are never inherently -problematic.** A model that corrects benefit underreporting should, all -else equal, sit below survey-based poverty rates; divergence from Census -numbers is expected by construction. Treat official statistics as -comparators — direction and composition anomalies are investigation flags, -not "misses". +**Poverty remains a comparison diagnostic.** Report differences using comparable +definitions, populations and periods. A gap can prompt investigation, but does +not establish a defect or require a change. Correct measurement and +implementation errors on independent evidence, even if the correction moves +the poverty rate farther from the survey estimate. Correctness checks remain +release requirements; closeness to a survey poverty rate does not. ## 0. Mint the fact (ledger repo) @@ -111,6 +113,21 @@ exclusion register instead. wiring is safe and normal (the keogh ALD facts rode the feed unmapped for weeks). +### Table 1.1 size-of-AGI shares + +Cross-period national Table 1.1 return counts and AGI from the $100,000 lower +edge can bind through `_rebase_stale_soi_agi_size_distributions`. Each class +uses its own vintage's national denominator and the latest eligible national +control of the same measure. Missing denominators or controls drop the class. +The amount ager completes only the interval after the control year; counts do +not receive a CBO dollar factor. Other tables and cross-period lower-income, +filing-status and state slices remain excluded by this rescue. + +The [native integration note](us-native-soi-agi-targets.md) documents exact +source lineage, the distinction between source and alignment years, preserved +same-period behavior, and the separate support and release qualifications. +No tail support or held-out improvement follows from compiling these rows. + ## 4. Keep the exclusion register honest `US_FISCAL_TARGET_SUPPORT_EXCLUSIONS` is keyed by `source_record_id` and its diff --git a/docs/us-full-asec-immigration-donor.md b/docs/us-full-asec-immigration-donor.md new file mode 100644 index 000000000..34223a1b0 --- /dev/null +++ b/docs/us-full-asec-immigration-donor.md @@ -0,0 +1,94 @@ +# Full original ASEC immigration donor + +`borrow_full_asec_immigration_donor(preparation)` returns a checked retained source +view for a future immigration consumer. It assigns no status, admits no national +stock proxy, and does not attach anything to the graph. The preparation must be +an authentic retained native preparation; a selected Frame or copied receipt is +insufficient. + +The view reuses the native owner's complete current-cohort roster with +`selected=None` and its descendant constructor. Original `HSUP_WGT / 100` +fractions supply household DESIGN weights. Selection fractions, source shares, +clones and the legacy parent's pooled weights do not rescale this donor. +The accompanying household projection retains the exact literal weight, +numerator/denominator, reduced fraction and original household key. No person +weight, MARSUPWT authority, stock normalization or prior-wage input is created. + +The Frame contains only structural IDs, original person keys, the reviewed +23-field immigration evidence, `age`, `is_female`, and household `state_fips`. +Raw literals remain separately available without loss. Sex comes from the +maintained authenticated A_SEX/AXSEX source reader; unresolved sex binding +refuses. State comes from original household GESTFIPS through the maintained +state reader, with a valid state/DC code required. Carried parent predictors do +not fill missing source evidence. The existing observed-age normalization keeps +common age equal to original A_AGE, including top codes. ASEC observation year +is 2025 and income year is 2024. Neither state residence nor legal status is +asserted as an income-year observation. + +## Owning-source token admission + +Every one of the 23 required fields must contain a nonblank numeric literal. +Raw surrounding whitespace and leading zeroes remain in the raw view; the +numeric projection admits the corresponding integer code. Blanks, malformed +text, infinities, unexplained codes and unreviewed response values refuse before +any future rules coercion. No blank becomes zero. Legitimate printed NIU codes +remain distinct integers, including PEAFEVER = -1 and the appropriate zero codes. +This validates source code meanings, not a legal-status classification. + +The [2025 ASEC person dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +provides the following admitted values: + +| Fields | Admitted codes | +| --- | --- | +| PRCITSHP | 1–5; unexplained negative header values refuse | +| PEINUSYR | 0–28; the enumerated 27/28 values override the stale 0:26 header | +| A_AGE | 0–80 and 85; 80 and 85 remain the printed top codes | +| A_MARITL / A_SPOUSE / A_HSCOL | 1–7 / 0–16 / 0–2 | +| A_LFSR | 0, 1, 2, 3, 4, 7; no prior-wage inference | +| MCARE, CAID, IHSFLG, CHAMPVA, MIL, SS_YN, SSI_YN | 0–2, preserving each field's own zero/NIU meaning | +| PEN_SC1/2, RESNSS1/2 | 0–8 | +| PEIO1COW / A_MJOCC | Enumerated 0–8 / 0–11; unexplained header-only values refuse | +| PEAFEVER | -1, 1, 2 | +| SPM_CAPHOUSESUB | Exact whole dollars 0–99,999; trailing decimal zeroes accepted, fractional dollars unresolved | + +PENATVTY uses the 162 printed country/area codes in Appendix J of the +[2025 ASEC technical documentation](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf). +The person dictionary's Appendix H cross-reference does not identify the actual +2025 country appendix. Header-only missing/response codes are not assigned a +nationality. The monetary field agrees with the maintained +`asec_current_money_domains_v1.json` contract (2025 dictionary physical page 61, +printed 6C-40). If actual source tokens require additional meanings, keep the +literal owner intact and review that domain explicitly; do not default them. + +## Retention and validation + +The caller must retain the view and call `validate()` after its last relevant +I/O. The view retains the authentic preparation, native source owner, original +literal qualifier and sex source owner. Independent immutable seals live in an +external issued-state tuple, not solely in an editable validation closure. +Validation checks source owners and implementation bytes, then rechecks pure +source/output/issuance seals. Detached copies, mutated projections, changed +source anchors, altered implementation, or changes during final foreign I/O +refuse. A receipt and a Frame alone confer no authority. The projected Frame's +metadata and mass history remain neutral; the retained receipt explicitly keeps +status assignment and national stock alignment false. + +The tests use tiny invented files through the genuine source issuers: a selected +preparation omits a household/person retained by the full donor, original row +orders differ, and exact HSUP_WGT fractions survive. Pure token controls cover +all 23 fields. Malformed owning evidence reaches the donor after genuine source +preparation and refuses with a field-specific code. Independent demographic +coordinates and ordered household/state alignment have pure mismatch controls. +Genuine preparations also reach donor refusal for unresolved sex and blank, +unnamed or out-of-area state codes on a donor omitted by support selection. +The positive source case asserts the complete ordered person-column surface. +Original-anchor changes, detached copies, output mutation and late I/O mutation +are also tested. Omission/duplicate/weight validation belongs to the retained +preparation, rather than a donor-level refusal claim. No actual population, +country model, fit, calibration, full-51 proof or legal-status distribution was run. + +Later work still requires reviewed #779 rule adoption with per-row observation +years, explicit admission of its mixed-date stock proxies, source-stable draws, +ACS transfer/reconciliation and original allocation weights, clone attachment, +graph replay, and final exported-artifact/model qualification. This source slice +settles none of those downstream decisions. diff --git a/docs/us-full-original-amount-donors.md b/docs/us-full-original-amount-donors.md new file mode 100644 index 000000000..ebed211dd --- /dev/null +++ b/docs/us-full-original-amount-donors.md @@ -0,0 +1,53 @@ +# Optional full-original ASEC amount donors + +`run_us_survey_enrichment(..., full_original_amount_donors=True)` fits the +selected amount groups using complete original current Annual Social and Economic +Supplement (ASEC) support from the Current Population Survey, independently of +which households were selected for the receiving survey spine. The default is +`False`, preserving the existing selected-donor path. The supported groups include +unemployment compensation (UC), health costs and the separately opt-in workers' +compensation (WC) group. The separately opt-in [child-support group](us-native-child-support.md) +requires this option and fits received support only; paid support stays observed-only. +Selecting full-original donors does not itself add an amount group. + +The retained authenticated preparation supplies the full original 2024-income, +2025-survey ASEC roster and current-money source. A separate graph `CREATE` node +exposes this donor population with its original household DESIGN weights. +Group-specific `FILTER` nodes retain rows with jointly known targets and finite +required predictors. The existing fitted artifacts and apply nodes then draw for +the selected original American Community Survey (ACS) recipient axis. The graph +copies each original draw to its two existing clones. It introduces no new +receiving rows, allocation, geography assignment or clone draws. + +Predictors are current age, employment income and self-employment income. When +the financial parent uses demographic conditioning, observed source sex and state +are also required. The source projection keeps missing predictors and targets +unknown; their exclusion from donor fitting does not turn them into zero. UC and +WC retain the existing literal receipt/universe qualification. A reported zero, +unknown response and outside-universe source observation remain distinct. +Recipient age applicability is separate from donor source qualification and is +unchanged. Prior-year income fields are not included in the new donor frame. + +The selected receiving originals, observed source amounts, memberships, IDs and +IMPORTANCE weights are unchanged. Full donor support is not a claim that every +original is eligible for fitting: each amount group still applies its own +knownness filter. It also does not qualify a release or establish scientific +equivalence with a published dataset. Donor exclusions, rare positive coverage +and sensitivity to the expanded fitting support still need actual-data review. + +The new projection borrows the live source owners; it does not issue a replacement +source authority. Descriptive receipts and cached frames cannot recreate that +authority. The host binds the option, full source frame and exact donor/model +identities, reconstructs materialized outputs on replay, and requalifies sources +at its final I/O boundary. The additional source module participates in the +enrichment implementation hash. These implementation changes invalidate affected +enrichment cache identities even with the option off; earlier source/runtime +snapshots remain separate immutable studies. + +Invented tests cover a genuine source preparation whose selected receiving spine +omits a positive WC donor, that donor's inclusion in the maintained numerical +graph, cold execution and required replay, source observation preservation and +identical draws across the two existing clones. The receiving frame in that +numerical test is an explicit component fixture. Full combined financial and +enrichment owner acceptance and an actual-data fit remain separate verification +steps. diff --git a/docs/us-household-role-namespace.md b/docs/us-household-role-namespace.md new file mode 100644 index 000000000..2d9568272 --- /dev/null +++ b/docs/us-household-role-namespace.md @@ -0,0 +1,36 @@ +# Separate carried headship from source-qualified roles + +Native preparation preserves a carried `is_household_head` column as +`legacy_prepared_is_household_head` on its detached source copy. Every value, +missing bit and physical dtype is preserved. The original source Frame, +structural IDs, membership, weights, strata and mass history remain unchanged. +The preparation receipt records the versioned namespace rule and which sources +supplied the carried column. A preexisting reserved diagnostic name refuses. + +This separates two meanings that previously occupied the same column. Legacy +ASEC preparation can derive headship from a within-household sequence. ACS +preparation supplies a dense relationship-derived boolean, including false for +group quarters. Neither establishes a source-qualified housing reference-person +role for every row. In particular, the source role is unresolved for ACS group +quarters, rather than an observed false answer. + +The existing role operation now owns a fresh nullable canonical +`is_household_head`, using the original ASEC `A_EXPRRP` and ACS `RELSHIPP` +observations and their existing universe rules. It still refuses conflicting or +unsupported known canonical incumbents. It does not use the renamed diagnostic +to fill an unknown. `household_role_reconciliation` separately reports legacy +agreement, disagreement and known legacy values without qualified counterparts, +including counts by survey. Relocating a column must not hide these differences. + +The rename happens at the existing authenticated preparation boundary, before +spine assembly and support cloning. The graph's same-dtype rewrite constraint +is unchanged. All preparation and descendant keys change with the source code +and receipt; an old checkpoint cannot confer a new live source qualification. + +Tests preserve both dense and nullable legacy storage, reject collisions and +modified copies, retain disagreements in diagnostics, and pass the maintained +genuine source issuers over invented housing and group-quarters records. Existing +canonical-conflict refusals remain tested separately. These controls do not +certify actual-source counts, a full graph run or a published file. The final +model/export boundary must still handle any out-of-universe nullable role under +an explicit, checked applicability convention. diff --git a/docs/us-immigration-observation-years.md b/docs/us-immigration-observation-years.md new file mode 100644 index 000000000..582c183e6 --- /dev/null +++ b/docs/us-immigration-observation-years.md @@ -0,0 +1,91 @@ +# Immigration rules and source observation years + +The immigration rules and controls are adopted from Microcosm #779 at +`7ee36ac1bea125218912996b1030b77a208db963`. The port replaces the earlier +wage proxy with the source `A_LFSR` labor-force predicate and carries the paired +status codec, evidence-constrained humanitarian reconciliation, and existing +composition bands together. Only the immigration stage is adopted from that +revision; other source stages and transfer families keep their current contracts. + +`observation_year_column` is an optional keyword on assignment, evidence, +candidate-mask, reconciliation, composition-gate, and ACS-transfer APIs. It names +a numeric person column containing a finite integral calendar year on every row. +Missing columns, missing rows after a join, strings, booleans, fractional years, +and non-calendar values refuse. It does not authenticate a source or establish +which year belongs to it. A native caller must bind the values to retained source +evidence: 2025 for current ASEC demographics and 2024 for current ACS. ASEC income +year 2024 remains distinct from its demographic observation year. + +Both ASEC arrival helpers and mixed-source profile arithmetic use the explicit +row year. This matters at cohort boundaries: age 33 and entry midpoint 2007 imply +age at entry 15 in 2025 and 16 in 2024. Assignment, immutable-row reconciliation, +candidate selection/replay, and final composition validation must use the same +convention. The composition gate accepts scalar `time_period` (default 2024), +matching assignment. Original ages and the PEINUSYR 27/28 approximation are +unchanged. Omitting the keyword retains the adopted rules' scalar convention; +finding a column with a suggestive name does not enable it. + +For paired ACS transfer, the column must exist on both frames and must survive +the donor projection. The three evidence predictors remain citizenship, +raw origin code, and approximate entry year, alongside the existing age/sex/state requirements. The +observation year is not added as a QRF predictor and does not change pattern +seeds. Rows without arrival evidence use a constant zero in the feature frame, +so the ASEC/ACS observation-year difference cannot leak through NIU entry values. +The rule profile's cohort arithmetic is unchanged. The execution contract version +4 binds the feature version, NIU value and complete ASEC approximation table; +paired checkpoint patterns independently bind the same contract so even a caller +without an outer bank identity cannot resume stale feature semantics. +The year column name and ordered typed person-ID/year pairs on both +frames enter the immigration checkpoint pattern identity. Existing bank +validation refuses stale patterns and rebuilds them. A caller-written outer bank +identity cannot bypass that check. Non-immigration patterns are unchanged. + +Reconciliation receipts also record ordered resolved-weight and mutable-mask +digests. `us_immigration_humanitarian_transfer_selection_masks` optionally accepts +`reconciliation_receipt` and refuses changed weights, mask or seed. This checks +the numerical selection inputs; the receipt does not authenticate source evidence +or controls. Explicit-row-year receipts use `time_period: null` and retain their +separate observation-year declaration rather than claiming scalar 2024. + +The published arrival documentation has a material inconsistency. The standalone +[2025 ASEC dictionary, printed 6C-6](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=27) +lists PEINUSYR 28 as 2022–2024; the +[2025 technical documentation, printed 6C-6](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf) +lists 2022–2025. Both enumerate codes 1–28, with no 29, despite a stale 0:26 +header. The retained value 2023 is an explicit midpoint/floor-midpoint +approximation compatible with either interval; it does not resolve their endpoint +disagreement or prove an exact entry year or statutory cohort. Raw code 28 and +this unresolved interval must remain visible at any later source-admission step. + +The technical documentation's Appendix J identifies 459 as Zaire (DR Congo), +and 461 as Zimbabwe. The +[2024 ACS Place of Birth list](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/code_lists/ACSPUMS2024CodeLists.xls) +identifies 451 as Sudan only; South Sudan is inseparable in public code 464 with +Tunisia and Western Sahara. The rules do not add that entire residual group. +The complete ACS and ASEC code domains are not interchangeable; matching active +named-country selectors does not create a general lossless crosswalk. + +ACS YOEP is also not universally an exact year: the +[2024 PUMS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.txt) +uses 1938 for 1938 or earlier and 1939 for 1939–1944. The reported year concerns +the most recent move to live in the US, not verified first entry or continuous +residence. Complete YOEP/CIT domain and source qualification remain separate from +this legacy rule helper; its permissive numeric input checks do not admit native +source literals. + +These are rule interfaces, not a native status owner or a release certificate. +The separate full-original-ASEC source view must remain available without +assigning statuses. Later native assignment must reject unauthenticated existing +status pairs, qualify owning tokens before legacy coercion helpers, use original +ASEC household design weights once, and preserve original identities. Later ACS +reconciliation requires an authentic complete-source selection and the receiving +population's allocated pre-clone weights. Candidate exhaustion remains an error. + +The imported control values are unchanged mixed-date imputation assumptions, +including broad unauthorized estimates, cumulative admissions, trailing flows, +TPS registrations, and an explicit-zero withholding assumption. They are not +measured legal-status stocks for a common calendar year. This port does not admit +those assumptions for a native release, update targets or tolerances, enable +native assignment or graph nodes, or qualify an actual export. Source-owner +admission, full-source reconciliation, no-worse comparisons, and final engine +and export checks remain separate work. diff --git a/docs/us-input-coverage-diagnostic.md b/docs/us-input-coverage-diagnostic.md new file mode 100644 index 000000000..d52f57726 --- /dev/null +++ b/docs/us-input-coverage-diagnostic.md @@ -0,0 +1,16 @@ +# US population input-coverage diagnostic + +Select `USInputProfile.NATIVE_NATIONAL_CD` explicitly for the approved native +scope. Its 163 names are the existing 165-name national/CD profile minus +`self_employment_income_last_year` and `previous_year_income_available`. +`employment_income_last_year` was already absent. The report's +`scope_excluded_inputs` records all three canonical prior-year income names, +whether present or absent in the supplied population; it does not remove +physical columns or classify missing data as non-applicable. Other required +names, assigned-block diagnostics and unresolved applicability stay unchanged. +The historical 167-name profile and default 165-name diagnostic retain their +existing behavior and report no native scope exclusions. This explicit name +profile does not resolve the separate native scientific/source gates or alter +historical release requirements. + +[`diagnose_us_input_coverage`](../packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py) takes a `Population`, its `CompiledGraph` and attached `RunManifest`, and returns a typed `PopulationInputCoverage` report. Its default `USInputProfile.NATIONAL_CD` describes 165 national/congressional-district input names; [`required_us_inputs()`](../packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py) keeps the historical 167-name default, extracted from the current release coverage manifest. The national/CD profile omits only engine `block_geoid` and `tract_geoid`, while separately reporting assigned `census_block_geoid`; neither profile adds `employment_income_last_year`. The diagnostic checks complete declared current ownership and replayed storage, then reports actual entity grains, source-channel/clone groups, null and invalid values, typed weights, and masked-writer versus carried rows. Missing grain and applicability remain unresolved. It verifies consistency of supplied graph objects; source ancestry and source requalification remain the host's responsibility. [Twenty-five frozen controls](../experiments/us-input-coverage25-20260910.json) passed under the earlier 163/161/159-name profiles, including actual cold/required replay and late-mutation refusals, with 515 physical source/owned checks and zero model/resource inputs. This is scoped evidence, not maintained-CI validation: four eager calibration imports differ from the frozen run and retain their newer maintained versions. The report establishes neither complete input coverage, statistical signal, applicability, native build acceptance nor release eligibility. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md new file mode 100644 index 000000000..8914d6bea --- /dev/null +++ b/docs/us-launch-review.md @@ -0,0 +1,454 @@ +# US graph build: review guide + +This branch consolidates the US graph build work for review. The intended file +supports national and 119th congressional district analysis from one population. +It is still a development build: complete enrichment, fiscal calibration and +release verification have not passed together. + +## Architecture and review order + +The intended construction order is ACS + CPS ASEC → complete survey multispine +including its initial support/PUF clones → household Census block assignment → +derived larger geographies → donor enrichment → rules evaluation and calibration +→ verified full and pruned exports. Each resulting household retains its assigned +location through the later stages. The PUF +operator acts on the combined survey frame. A separate ASEC–PUF base is not the +intended architecture. + +National and local analysis share one fully constructed, enriched Frame. The +analysis branch begins after harmonization, cloning, atomic geography assignment +and imputation. Calibration changes weights; L0 controls sparsity. An export can +select a geographic scope and remove zero-weight households, but must preserve +every retained record's input values, knownness, entity relationships and assigned +geography. There is no separate national or local enrichment operation. A source +or mapping correction produces a new common parent Frame for both views. + +Release verification must bind each full or pruned export to that parent's +identity and its calibration target/weight specification, compare retained +inputs by stable entity ID, and reject changed inputs, dangling entity links or +regenerated geography. Weight and scope changes are explicit exceptions; record +values are not. This is the required contract, not a claim that all of these +checks have already passed on the release candidate. The separate legacy +ACS-local hours omission in [#765](https://github.com/PolicyEngine/microcosm/issues/765) +illustrates the drift this common construction path removes. + +Pre-calibration source quality and post-calibration export integrity are distinct +checks. Joint calibration may change an origin's contribution or set its weights +to zero. A pruned export must not be required to make every survey origin +independently resemble a national population, or to rerun imputation to satisfy +such a requirement. + +The supplied-parent export comparison now passes 36 focused checks, including +actual checkpoint write/readback, complete household selection, unchanged input +values and missingness, signed and unsigned stable IDs, and full/pruned/local +weight and scope handling. A household with zero original weight can receive +positive calibrated weight without mutating its parent or other entity weights. +The comparison binds the full ordered weight vector and specification, including +rows outside the selected scope. It deliberately reports calibration ancestry +and release eligibility as unverified: the owning build must establish those +and seal its inputs across export I/O. See the +[scoped export acceptance](../experiments/us-common-frame-export36-20260910.json). + +The earlier ten-node survey prefix assigns a block before cloning, and its +thirteen-node age-calibration extension passes cold execution and required +replay on invented originals. A twenty-node extension now carries atomic +geography through current financial imputation and required replay. Native +Delaware and complete national block support pass source normalization; +national/CD acceptance of the enriched population remains pending. The earlier joint tract/district +operator is a separate path. See +[Geography assignment](geography-assignment.md) for the country contracts, +implementation and source boundaries. + +The 10 September sequencing correction now passes 70 inventoried controls on +invented inputs, across seven separately guarded runs, plus one separate default +compatibility case. The nine-node prefix completes its initial clones before +block assignment; the nineteen-node financial extension and twelve-node age +extension retain that geography through cold execution and required replay. +Assignment uses the qualified original-source key and stable clone discriminator, +so changing row order or numeric household coordinates does not redraw the same +clone. Observed state/PUMA constraints, input knownness, entity links, complete +population ownership and weights remain checked. Different clones may draw +different blocks; an occasional shared block is valid. + +The financial donor FILTER requires the exact typed geography-validation artifact +and retains original ASEC DESIGN weights. The shared gate emits this artifact +only when requested, preserving its default API for UK consumers. Two test-only +corrections repair the CREATE source declaration and isolate the obsolete-order +budget counterexample from unrelated serialized storage differences; production +matches the independently reviewed candidate. See the +[scoped postclone acceptance](../experiments/us-postclone-geography-70-controls-20260910.json). + +The earlier ten/twenty-node native runs retain their original order. Native +postclone execution, the new-order PUF host, held-out model quality, national/local +fiscal calibration and full release acceptance remain pending. The age control +uses invented targets and does not establish that complete calibration path. + +| Area | Main source entry points | What to review | +| --- | --- | --- | +| Graph execution and storage | `packages/microcosm-graph/src/microcosm/graph/{decl,executor,manifest,store,codecs,schema}.py` | Typed inputs and outputs, ownership, artifact ancestry, cache identity and nullable round trips | +| Survey source preparation | `packages/microcosm-build/src/microcosm/build/us_runtime/{acs_population_catalogue,asec_population_catalogue,survey_population_preparation}.py` | Source identity, entity links, source universes and current income transformations | +| Combined survey and clone | `us_runtime/{graph_composed_population,graph_survey_population,graph_combined_clone}.py` | ACS and ASEC composition before the clone; native versus detail channels | +| Enrichment | `us_runtime/{full_puf_enrichment,graph_full_puf_enrichment,graph_current_survey_puf_transfer}.py` | Target ordering, conditioning, observed-value preservation and complete replay | +| Conditional models | `packages/microcosm-fit/src/microcosm/fit/{qrf_target,graph_legacy_train,graph_legacy_apply_matrix}.py` | Reusable model artifacts, deterministic draws and target regimes | +| Geography | `atomic_geography.py`; `us_runtime/{atomic_block_support,atomic_block_api_sources,survey_atomic_geography,graph_atomic_survey_population}.py` | Atomic block assignment after complete initial cloning, observed-source constraints, versioned mappings, stable clone identities and the optional typed validation prerequisite | +| Survey mass and calibration | `us_runtime/{survey_origin_budget,graph_survey_budget,graph_survey_calibration}.py`; `packages/microcosm-calibrate/src/microcosm/calibrate/{group_bounds,solve}.py` | Original survey mass, grouped bounds, fixed support and existing ungrouped solver behavior | +| Compatibility | `packages/microcosm-build/src/microcosm/build/{frame_checkpoint,us_runtime/__init__}.py` | Current-main APIs, checkpoint metadata and existing country consumers | + +Paths beginning `us_runtime/` are relative to +`packages/microcosm-build/src/microcosm/build/`. +The branch contains a substantial consolidation; review each area and its tests +before deciding how to land it. Shared model work overlaps with +[#873](https://github.com/PolicyEngine/microcosm/pull/873). Anthony considers his +runtime proposal [#885](https://github.com/PolicyEngine/microcosm/pull/885) +superseded by this direction; it is reference material, not a prerequisite to +retain or merge separately. This does not imply that its GitHub PR is closed or +that this draft has received his approval. + +## Parallel review and dependencies + +Completion of this integration draft is not a prerequisite for all UK work. +Depend on identified shared changes where necessary, with independently reviewed +PRs and compatible interfaces. Keep the US native build, UK candidate repair, +source/target reconciliation and shared runtime work moving in parallel. + +| Reviewer | Review focus in this draft | Work that can continue independently | +| --- | --- | --- | +| Anthony | Shared graph/Frame and calibration compatibility; target hierarchy and metadata flow into diagnostics/dashboard, including #855 | Target/dashboard changes follow #900; UK staging #896 proceeds independently of full UK graph conversion and native US PUF integration | +| Maria | Register each existing UK pipeline stage as a graph node using this branch's shared machinery; verify UK behavior and coverage | #900 lands before graph conversion; UK imputation, support/fit diagnosis and candidate validation continue; graph-node registration stacks on #893, with stage reordering in a later separate PR | +| US integration owner | Resolve overlap, maintain exact compatibility evidence, complete the US population and release checks, and extract shared changes agreed in review | US-specific source preparation, imputation, quality diagnostics and progressive local builds | + +Coordination updated 10 September: Maria is deliberately stacking her UK +graph-node registration on #893. That specific dependent work should pin the +parent revision and coordinate shared-interface changes here. Her source/target +repairs and Anthony's staging work do not have to wait for the whole draft. +PR #900 merged at 14:54 UTC and is incorporated here from main at +`6f7571e1ab8c516154289bd8ebf75cb2646f03b5`. The combination preserves both parent +histories and does not rerun or recertify the frozen US controls. +Anthony's hierarchy/diagnostics [#855](https://github.com/PolicyEngine/microcosm/pull/855) +and its dashboard consumer follow the target reconciliation in +[#900](https://github.com/PolicyEngine/microcosm/pull/900). Registering UK graph +nodes and changing their order are separate review steps. The US integration +work must not build a competing full UK graph; its shared atomic-area adapter +is an input to Maria's owned integration. + +Record shared dependencies as named changes with pinned revisions. Extract a +smaller shared prerequisite where useful, while preserving compatibility with +Maria's intentional stack. Proposed sequencing does not certify or merge any PR. + +The shared post-clone geography interface now has a tested additive validation +artifact contract. The default gate API remains unchanged; UK registration can +adopt the optional typed prerequisite independently of the US stage ordering. +Existing target/source repairs can proceed on their reviewed base. Passing this +draft's invented controls does not certify Maria's UK candidate, and the final +US release need not gate her independent source and calibration work. No +whole-PR merge is implied by this review map. + +## Verification and review scope + +The first native twenty-node survey, block-geography and financial cold pilot +now passes at 1/1000: 1,584 source households and 3,464 people become 3,168 +household records and 6,928 person records after cloning. Ten financial nodes ran +cold over the stored ten-node prefix, which retains the block assigned before +cloning. The composition retains location inheritance and attaches seven financial +fields while preserving the other population fields. It took 110.12 minutes and +11.45 GB peak memory. The final receipt and independent source/resource +postcheck pass. This is the original frozen 491-source implementation; newer +cache and budget changes are separate revisions. Required native cached replay +now also passes: all twenty final nodes and ten prefix nodes were cache hits, +the four non-manifest exports matched the cold bytes, and the portable manifest +retained the same content and stored-artifact identities. Replay took 74.17 +minutes and 11.31 GB peak memory. Independent postchecks verified the recorded +result and exact code/resource identities without reopening native inputs or +exported payloads. PUF augmentation, model quality and calibration remain pending. +See the [scoped replay result](../experiments/us-native-atomic-financial-required-replay-20260910.json) and +[scoped native cold result](../experiments/us-native-atomic-financial-cold-20260910.json). + +The previously published head `a85c9cbb79081a980e7d3afe8916c80153de5bf8` +passed 36 integration controls and 57 enrichment/replay controls on invented +inputs. The 36 cover eight solver, eleven snapshot/storage, nine geography and +eight facade cases. The 57 include cold execution, required replay, reconstruction +through a fresh Frame store, inherited tax-unit values and malformed ancestry +refusals. Source/control, model-declaration and resource pins matched before, +after and in external postchecks; the tax-benefit engine was not executed. + +Those receipts are identified by SHA-256 +`ad8ab6d8761db3d21343bc1fc60023c0dfa305e1367622630a5de6c68892d62a` +and `85a192f8c5c851648875d66b90bea6b8a4ee11e86e67d1875c3ca34a076ee014`. +These checks do not certify subsequent source additions or the entire repository. + +CI exposed test-helper collection failures and a pool/registry circular import. +This revision adds the test helper directory to pytest's path and moves the +registry import after the pool functions it validates. Two fresh-process import +regressions are assigned to the US-engine CI lane; their runtime result remains +pending. No import-order result is inferred from an AST check. + +The corrected checkpoint test selection passed all 58 invented controls against +the current integration checkpoint source, including nullable integer widths, +missing-value masks, deterministic round trips and malformed v4 refusals. It +took 5.36 seconds wall time and 348.5 MB peak RSS. All 45 source/control files, +three provider pins and two code resources matched in the external postcheck. +Receipt: `f8009ec6d07d076eff6a90ec97061d538756293547a21758ab2bc38fa5f7aab8`. + +Fable's earlier replay review closed seven findings. A separate review of the +published consolidation identified the checkpoint test gap and additional +grouped-bound, metadata-store and facade coverage work. The portable grouped-bound, +fixed-support and metadata-store selection now passes all 117 tests with no skips. +The fixes reject nonfinite stored JSON as `StoreCorrupt` and normalize the deprecated +`apg` alias before grouped-mode validation. Source/control, resource and provider +bytes match before, after and in an external postcheck. Receipt: +`3674af9d610417897539da6eae766726d0f582ae6dd3bc2c3e550fefa45bab68`. +The historical platform fixture and complete facade import coverage remain separate. +Environment-specific prepatch byte fixtures do not establish portable CI byte +parity. Existing v1 graph stores remain preserved; v2 execution uses a new store +rather than silently promoting old frames. + +Private runtime records and generated population artifacts stay outside this +source PR. Receipt identifiers are audit references, not reproducibility inputs. + +The composed 156-case test run against source commit `8157300` at documentation +head `78d84f9` passed 152 cases and failed four current-survey cases with +`PRODUCER_CHANGED`. Its execution guard also recorded unexpected test-bytecode +and installed-model cache-directory probes. Source/control, model-source and +resource bytes remained unchanged. This is a failed integration run, not a +replacement for the earlier passing component evidence. Its receipt is +`67ff10fef45c58d2d6ea15c72b4311a4daa901dd2aae263fbb863d38c13bc7c8`. +The successor now passes all 314 selected invented integration and identity +checks with no skips or unexpected refusals. This includes the original 156 +cases, five metadata-copy/mutation regressions and 153 encoder controls; seven +synthetic-issuance controls remain explicitly outside this selection. Shallow +copy, deepcopy and pickle previously added a class cache entry captured by +Python 3.14 annotation closures. An explicit immutable-metadata representation +avoids that mutation without weakening producer checks. Two stale ACS pins were +updated only after verifying AST equality with their accepted source revisions. +All 476 source/control, 5,983 model-source and 12 resource entries matched before, +after and in the external postcheck. The run took 253.5 seconds and 738.7 MB peak +RSS. Receipt: +`d940913f918af4455dab2b9e680d55f9efc7bb922f72ff3d8fe26b0530b070f7`. + +A separate 16-case current seven-node age-development run initially passed 13 +cases and failed three on exact diagnostic reconstruction. The checker omitted +two solver-option fields added by the consolidated solver. Reconstructing the +grouped solver's closing-state selection and empty selection receipt fixes that +mismatch while retaining exact whole-document comparison. All 16 tests now pass, +including actual seven-node execution and late target/successor mutation checks, +in 170.3 seconds at 541.9 MB peak RSS. All 469 source/control, 5,983 model-source +and 12 resource entries matched before/after/current checks. Receipt: +`5849dd42e914e4d8304a5d290ff451da45910e7ac9a61e76fff36468949ffe3a`. +Broader CI's missing helpers and stale schema, seed and runtime classification +expectations remain separate from these selected passing suites. + +## Component evidence and remaining work + +Separate component work has verified the national Census joint-geography source +artifact: 2,462 PUMAs, 87,841 joint cells, all 50 states plus DC and 436 districts +including DC. PUF source ingestion retained 207,692 ordinary records and excluded +four disclosure records. A corrected SCF wage-code interpretation passed source +preparation and model-mechanism checks. Those results belong to their respective +component revisions; they do not establish a calibrated result on this branch. + +The corrected canonical-donor codec v2 was rebuilt from the same accepted typed +PUF source. All 207,692 returns and 59 outputs passed selected-cohort replay and +byte/value serialization checks in 24.37 seconds, using 1.95 GB peak RSS. The 481 +source/control files and four resources matched their before/after/current pins. +Receipt: `6b334ca8df6c6ba721b60f84d28c43151a1e1b54e097616a3c0fa208daeb8cba`. +This run constructs the donor; it does not fit or place it onto survey recipients. + +The identity encoder passed two 40-case preflights and four 40-case +timing runs in both execution orders. The 36 tiny invented cases retain exact +identity bytes and mutation checks. Nine of the 72 case/trial comparisons were +slower, including the small Python-string Frame in both orders; larger numeric +fixtures improved substantially. These timings do not establish genuine-build +speed. It is now integrated and passes the current 314-case selection, with the +newer current-wage projection preserved byte for byte. That projection was absent +from the benchmark's accepted baseline. + +The following work remains open: + +The [US and UK release path](us-uk-release-path.md) puts these implementation +items in the complete population-quality, calibration and consumer-release +sequence. Passing component controls is one step in that sequence. + +1. Extend the accepted native twenty-node atomic and financial cold/replay + pilot to the complete survey scale. Its + invented cold/required replay, complete retained + fields and geography, default three-feature versus explicit five-feature + conditioning, and both final mutation refusals pass on invented inputs. + Two test-code mistakes in the initial four-case run were corrected and + rerun separately; the passing mutation cases were preserved. See the + [composition acceptance](../experiments/us-atomic-financial-composition-acceptance-20260909.json). + Native fit quality and admission of the financial result into subsequent + calibration remain unassessed. +2. Integrate the adopted Social Security conditioning measurement and qualify + recipient return roles before native PUF55 fitting. Thirty-one invented + controls pass the authenticated filer/joint-spouse report-sum proxy, source + preservation, missingness and refusal checks. An incomplete selected report + uses a separately declared eight-predictor route; a known total uses nine. + The helper uses the preparation's retained modeled roles, whose relationship + to current-money tax-unit construction still needs explicit qualification. + See the [measurement decision](puf55-survey-ss-measurement-decision.md) and + [scoped control result](../experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json). + Route-aware graph fitting and attachment remain pending. The genuine + 207,692-return canonical donor has already passed its 55-output projection; + that does not establish recipient matching. Complete survey + beneficiary/component modeling, and supply SCF loan inputs. Separate component runs passed + 128 profile/compatibility controls and 26 current-survey predictor controls; + see [the implementation status](current-survey-puf59-progress.md) and + [donor construction and growth](puf2015-canonical59-and-growth.md). + The genuine 207,692-return donor construction does not establish survey fit + quality, complete enrichment or a releasable population. +3. Verify every applicable model input on each source/clone channel. The proposed + national/CD profile retains 161 required model inputs, with state and county + but without requiring the engine to consume block and tract. Independently, + the build must retain one assigned Census block and derive its larger + geographies. Prior wages remain excluded. A source operator's existence is + not complete cell coverage. +4. Retain the earlier seven-node age-development result as historical evidence; + the twenty-node cold/replay pilot in item 1 is the newer native milestone. + The seven-node run passed at 1/1000: 1,584 source households, + 3,168 records after the support clone, final export/readback and owner/target + verification, 71.1 minutes and 8.26 GB peak RSS. Its source is frozen at + `a9895d8a5`, before the new Social Security, PUF55 and atomic-geography work. + The prior two-hour failure and stale-derived-attachment failure remain failed + historical runs. [The scoped acceptance record](../experiments/us-survey-age-development-20260909.json) + does not certify enrichment, full national/CD calibration or a release. +5. Run real model evaluation, national/CD calibration, holdout checks and complete + export/replay verification, followed by a dashboard bound to that exact file. + Check every pruned analysis file separately. + +These are active workstreams. No release, merge or deployment is implied by this +draft, and the source changes do not relax the outstanding acceptance checks. + +The two-route numerical finalization layer now passes 46 invented controls, +including 110 actual target fits, 220 applications and comparisons against the +maintained whole-cohort finalizer for both routes together and either route +alone. The tests also refuse changed donor row order, weights and late raw or +donor mutations. The v2 numerical receipt seals the complete finalized candidate +before return and refuses a change during receipt encoding. This verifies the +numerical layer; typed model ancestry and complete survey attachment remain +separate graph checks. +The first canonical donor CREATE attempt failed during kernel construction +because its live-code checker followed a circular function closure. The +[failed observation](../experiments/us-puf55-canonical-create-failed-20260910.json) +is preserved. The cycle correction passed its ten new marker controls, but +the [39-case follow-up](../experiments/us-puf55-canonical-cycle39-failed-20260910.json) +still refused before CREATE because the code-state snapshot changed during +initialization. The changing member was the serialized Python code object; +the corrected marker retains the actual code object and its immutable fields. +After correcting three obsolete test accesses to the public PopulationView API, +all 47 canonical CREATE/converter controls pass, including cold execution, +required replay and teardown. The corrected numerical and canonical sources are +adopted locally. These checks use invented donors; the native full PUF host +remains pending. See the +[46-control numerical evidence](../experiments/us-puf55-numerical-output-seal-46-controls-20260910.json) +and [47-control canonical evidence](../experiments/us-puf55-canonical-create-47-controls-20260910.json). + +All 149 selected calibration controls now pass on the combined US/UK solver, +including grouped bounds, fixed zero support, informed gates, budget search, +ordinary best-iterate behavior and strict diagnostic payloads. The run took +4.57 wall seconds at 394 MB peak memory; independent checks confirm the exact +34 source files, copied fixtures and all 149 unique test results. The preceding +run stopped before tests because two still-denied import probes exceeded their +reporting ceilings; the fresh invocation changed only those reporting ceilings +and run identities. No data access was added. This is invented-data solver +verification, separate from native national/CD calibration; see the +[calibration evidence](../experiments/us-calibration-consolidation-149-20260910.json). + +Recipient qualification now covers all 28 distinct original public controls: +27 passed in the original run and the five-case corrected detached-output run +passed, with four overlapping controls. The original run remains failed because +one NaN mutation test did not change its input bits; its correction flips a bit +in the actual float64 storage. This is a compound result across two source +snapshots, not a single passing 28-case run. Exact original-source issuers and +the 22-node cold/required graph are included. See the +[recipient evidence](../experiments/us-puf55-public-recipient-controls-20260910.json). + +The population-only Census API adapter passes thirty-five invented controls. +The native Delaware source control preserves all 15,317 populated blocks and +reconciles them to the independent Census state total of 989,948. It counts +4,881 zero-population blocks, checks CD119/PUMA joins and exactly reads back its +79,769-byte support artifact. The run took 10.94 seconds and 1.735 GB peak RSS. +The first attempt correctly refused an incomplete ZIP-member declaration; +a separate metadata probe established the six-member archive roster, and the +corrected check still decompresses only `NationalCD119.txt`. The +[accepted source control](../experiments/us-atomic-native-de-corrected-1-control-20260909.json) +preserves the original failed evidence. This is not national population or +release acceptance. Delaware-only support must not enter the national survey +graph. + +The subsequent population-only national acquisition and normalization now pass +for all fifty states and DC. The 104 exact sources produce 5,769,942 populated +blocks, independently reconciling every state to a total of 331,449,281 people. +Every block retains its identity and population, with complete CD119/PUMA joins. +The full support artifact is 28,862,508 bytes and passes exact serialization and +readback. Normalization took 82.97 seconds and 6.814 GB peak RSS. Its SHA-256 is +`5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960`; +see the [national source control](../experiments/us-atomic-native-national-1-control-20260909.json). +Postchecks reauthenticated code and compared the 104 native pins from bounded +receipts without reopening source bodies. National support normalization and +household assignment in the small native financial pilot now pass. Evaluating +geographic fit and calibrating the enriched survey remain separate acceptance +steps. + +The financial successor's unchanged positive control now passes with both +original fixture teardowns and final code/resource checks: 647.19 CPU seconds, +651.21 wall seconds and 570.9 MB peak RSS. This separately reviewed invocation +used a 1,200 CPU-second budget; the earlier 600-second attempt remains recorded +as incomplete. No memo or profiler was used. The [positive control evidence](../experiments/us-financial-successor-positive-20260910.json) +checks financial replay and a subsequent weight-only change. The [seven remaining refusal and callback controls](../experiments/us-financial-successor-remaining-controls-20260910.json) +now also pass with their original shared fixture and teardowns: 798.61 CPU seconds, +802.89 wall seconds and 563.6 MB peak RSS. All eight original admission controls +are accepted on the same source snapshot. A subsequent source review found an +additional detached-view callback gap in the budget and weight-only views. +The correction is adopted after four targeted controls passed in 543 wall +seconds, including nine callback branches and independent source/resource checks. +The original eight and corrected four are separate source-version checks; see +the [correction evidence](../experiments/us-budget-detached-view-correction-20260910.json). +The native pilots exclude this overlay, and native calibration remains separate. + +A separate, bounded profile of the unchanged invented cold/required financial +fixture passes its test, original teardown and all final code/resource checks: +303.90 CPU seconds, 308.26 wall seconds and 594.3 MB peak RSS. This measures the +fixture rather than the additional financial and weight-only admission work. +See the [profile evidence](../experiments/us-financial-fixture-profile-20260910.json). +Loaded ACS code verification accounts for 180.20 cumulative seconds inside the +298.15-second profiled fixture. Its nested function checker runs 2,009,108 times +and uses 91.05 self seconds; complete Frame identity uses 7.31 cumulative seconds. +Cumulative timings overlap and profiling adds overhead. The invocation-local +code comparison memo now passes all 18 focused invented controls, including +changed source, aliases, globals, closures, mutable constants and the historical +catalogue caller. Source and resource checks pass before, after and in the +external postcheck. See the [control evidence](../experiments/us-acs-loaded-code-controls-20260910.json). +The paired full-fixture run also passed correctness and final checks, but took +319.79 wall seconds versus 308.26 for the baseline. The memo added eligibility +work and demonstrated no speed improvement, so its implementation was not +adopted. The exact experimental patch and [paired comparison](../experiments/us-acs-code-memo-comparison-20260910.json) +remain recorded. A separate [one-call diagnostic](../experiments/us-acs-producer-profile-20260910.json) +now passes without constructing a fixture: the original producer takes 0.656 +profiled wall seconds. Its 157 compiler calls take 0.238 seconds, including +0.122 seconds inside 103 AST parses; those overlapping times must not be added. +A bounded bytecode compilation cache now passes 25 invented controls and final +code/resource checks, with fresh source reads and every loaded-function check +retained. No native-data speedup is established. The compilation cache has separately passed +the complete invented fixture and is adopted; the native financial cold and +required-replay pilots passed with their original frozen code. The earlier +startup refusal remains preserved and confers no data acceptance. + +Current CI separately reports stale source-attested spec/seed fingerprints. +The worker resource identity test now follows the actual lazy import closure +and includes an invented opened-JSON regression; its new CI result is pending. +Neither the component passes nor this profile establishes a green consolidation. + +## Related reviews + +- [Microcosm → Orrery exporter, #888](https://github.com/PolicyEngine/microcosm/pull/888): + already published separately. It preserves the supplied schema metadata and + exact large integers; graph visibility does not infer domain verdicts. +- [Orrery review index, #6](https://github.com/TheAxiomFoundation/orrery/pull/6) + and [search improvements, #4](https://github.com/TheAxiomFoundation/orrery/pull/4). +- [Dynamics graph integration, #420](https://github.com/PolicyEngine/microcosm-dynamics/pull/420): + owned by the separate retirement-model task and tested on synthetic inputs. +- [Candidate methods and paper updates, site #72](https://github.com/PolicyEngine/microcosm.institute/pull/72): + public component evidence and remaining release checks. + +The accepted local shared viewer remains a separate consumer. There is no package +upgrade or competing generic graph shell in this consolidation. diff --git a/docs/us-native-child-support.md b/docs/us-native-child-support.md new file mode 100644 index 000000000..659c64aad --- /dev/null +++ b/docs/us-native-child-support.md @@ -0,0 +1,49 @@ +# Optional native child-support amounts + +Select `groups=("child_support",)` together with +`full_original_amount_donors=True` in `run_us_survey_enrichment`. The default +amount groups and selected-donor behavior are unchanged. This is a development +transport assumption from ASEC 2025 income-year 2024 observations to ACS 2024; +scientific qualification and release validation remain outstanding. + +The sole fitted target is `child_support_received`, from qualified `CSP_VAL`. +The maintained zero-aware QRF uses all original current ASEC persons with known +receipt or known nonreceipt and finite current predictors, retaining their +original household DESIGN weights. It includes known nonreceipt zeros; ambiguous +recipient zeros and missing or contradictory source routes are not donor labels. +A selected receiving frame can omit a donor without removing it from this fit. +Eligible recipients are original ACS persons aged at least 15. One draw is shared +by both existing clones. Observed ASEC amounts remain source values, including +unknown values; under-15 ACS recipients remain unresolved. + +`child_support_expense` is an observed-only additional output. It retains only +qualified positive `CHSP_VAL` payments under the documented source route. Paid +zero/NIU is not treated as known nonpayment. There is no expense model, no inferred +absence of voluntary payments, and no expense or native-input-profile completion +claim. `CHSP_YN` describes an obligation, not observed payment incidence. + +Both outputs carry separate source-knownness, literal, route, allocation and +topcode diagnostics under `survey_child_*`. The `survey_current_*_origin` labels +distinguish ACS modeled receipts from ASEC source answers and unresolved values. +A modeled value does not alter its source observation mask. The source qualifier +returns the complete literal table separately from its selected observation view. + +If either canonical child-support column already exists, this graph node owns an +explicit rewrite from its newly qualified source/model values. The inherited +values are not accepted as evidence. Supported carried storage is NumPy +`float32` or `float64`; the new values must round-trip without loss, including +unknowns. Other storage refuses. Only these two declared columns can be rewritten; +all unrelated columns, structural IDs, weights, source records and clone mappings +remain unchanged. Earlier source projection, fitting and reporting nodes retain +the prior population version and their complete input checks. A final keep-all +`FILTER` opens a derived version after every prior host branch; only the two +canonical child amounts are written there. The version adds an explicit +conserved-mass ledger record without changing represented mass. Reconstruction +refuses any changed entity axis. The attachment evidence lists the replaced +columns and storage. + +The implementation reuses existing source qualification, full-donor CREATE and +FILTER nodes, fit/apply artifacts, model verification and final attachment. It +adds no new source issuer or release authority. Changing the maintained amount +host's implementation closure changes its cache identity, including for existing +routes; old runtime/source snapshots remain independently pinned. diff --git a/docs/us-native-geography-tenure-bindings.md b/docs/us-native-geography-tenure-bindings.md new file mode 100644 index 000000000..d0b7e748a --- /dev/null +++ b/docs/us-native-geography-tenure-bindings.md @@ -0,0 +1,31 @@ +# Native geography and tenure bindings + +The canonical state fragment converts the existing `assigned_state_fips` into household `state_fips`. It requires exact equality with `survey_observed_state`, which the native source projection qualifies from ACS `ST` and current ASEC `GESTFIPS`. Both inputs must be canonical two-digit strings in the shared state FIPS constants. There is no default-state fallback or float conversion. + +`graph_current_survey_state.state_binding_node` depends on the typed `geography.gate` artifact and the two household inputs. The existing graph assigns blocks after complete initial cloning and derives geographic layers through the shared atomic support. This fragment neither assigns blocks nor parses block identifiers to construct a second mapping. Each clone keeps its existing assignment. The gate orders the graph; its serialized bytes alone grant no source authority. + +The output is `int64` when new. If the receiving population carries `state_fips`, the declaration uses the graph's explicit full-cell rewrite and replaces it from the qualified derived input. Matching, incorrect, or missing incumbent values are not used as evidence. An existing `int64` or nullable `Int64` dtype is retained, as required by the graph rewrite contract; unsupported incumbent storage refuses. Every other column, axis, membership, weight and mass record is unchanged. The kernel does not import the country engine. + +This is an independent fragment. The enrichment host must explicitly register it, retain its genuine source and population ancestry, and reconstruct/requalify it on cold and required replay before it qualifies a complete native run. This slice does not change that host's default graph or claim release readiness. + +## Tenure questions that remain + +The consumer source declares household `tenure_type` as `RENTED`, `OWNED_OUTRIGHT`, `OWNED_WITH_MORTGAGE`, or `NONE`. SPM-unit `spm_unit_tenure_type` separately declares `OWNER_WITH_MORTGAGE`, `OWNER_WITHOUT_MORTGAGE`, and `RENTER`; its documentation identifies Census `SPM_TENMORTSTATUS` codes 1, 2 and 3 respectively. These are different entity grains and code systems. + +Current qualified housing capture (`current_survey_housing.ASEC_HOUSEHOLD_COLUMNS` and `ACS_HOUSEHOLD_COLUMNS`) retains original ASEC `H_TENURE` and ACS `TEN`. ASEC `H_TENURE=1` identifies ownership but does not establish whether a mortgage exists. The legacy `housing_inputs._HOUSEHOLD_TENURE_MAP` maps it to `OWNED_WITH_MORTGAGE`. That historical convention cannot become a claimed mortgage observation. `H_TENURE=2` is cash rent and `3` is no-cash-rent occupancy; the legacy household mapper uses `RENTED` and `NONE`, respectively. NIU or missing values must remain separately unresolved, not silently mapped to `NONE`. + +ACS `TEN=1` identifies ownership with mortgage/loan, `2` ownership free and clear, `3` cash rental, and `4` no-cash-rent occupancy. The maintained `acs_inputs._HOUSEHOLD_TENURE` maps `TEN=4` to household `NONE`, while its separate `_SPM_TENURE` maps that code to SPM `RENTER` through `_map_tenure`'s membership join. `acs_release_predictors.ACS_TEN_TO_SPM_TENMORTSTATUS` also maps `TEN=4` into the SPM non-owner consumed bin 3. The official Census ASEC 2025 dictionary (PDF page 103, printed 6C-66) describes SPM_TenMortStatus=2 as “Owner without Mortgage or rent-free.” Thus the maintained ACS TEN=4→SPM code 3 mapping requires source reconciliation, and ASEC code 2 alone cannot establish household ownership. The consumer enum label is narrower than the source definition. Group-quarters and unknown/inconsistent records need explicit applicability and missingness handling. No tenure output or scientific convention is added by the state fragment. + +A separate source-completion proposal can capture `SPM_TENMORTSTATUS` from the same authenticated original ASEC person member, retain literals and source year, and require within-SPM consistency against the existing genuine SPM membership. `current_survey_spm_source` currently captures membership and role fields, not that tenure field. A later binding must retain the actual SPM source owner and applicability. It must not derive household-owner mortgage detail from unrelated or multiple SPM units, reinterpret all owners as mortgaged, or qualify an inherited canonical tenure column merely because it exists. + +Consumer declarations were read without imports from the pinned local `2.2.1+spmuniverse.20260919` consumer used by the native159 source audit: `variables/input/geography.py`, `variables/household/expense/housing/tenure_type.py`, and `variables/household/income/spm_unit/spm_unit_tenure_type.py`. No country-engine execution or consumer update is part of this work. + +## Enrichment host option + +`run_us_survey_enrichment(..., canonical_state_input=True)` opts into the canonical state binding after the existing terminal enrichment fragment. The default stays false. A visible keep-all FILTER creates `survey_geography.canonical_state_version` before `survey_geography.bind_state_fips`; it preserves every support row and weight and adds one structural ledger record. The separate version is necessary when earlier projections read an inherited `state_fips`: rewriting that column on the same population would redirect those earlier reads to the later writer and create a dependency cycle. + +When child support is selected, state takes the child-support version as its base and its typed final attachment as the ordering dependency. The state binding is then the returned terminal population; both keep-all transitions remain in the ledger. Selecting all people must preserve every entity axis, including otherwise orphan groups; the reconstruction refuses if selection would prune any. + +The host retains the genuine parent population and authenticates its source/geography through the existing cold and required-replay entry/final fences. The version kernel checks the exact projected person IDs, memberships and clone coordinates. The binding kernel checks the exact retained household IDs and qualified derived/observed states, then calls the deterministic state operator. The host independently reconstructs both complete population transitions and compares persisted artifacts, Frames, weights and ledgers on every execution and replay. These checks do not turn a detached receipt into source authority. + +An existing canonical `state_fips` must use `int64` or nullable `Int64` storage; unsupported inherited storage remains an explicit schema prerequisite. The option never silently casts that incumbent. ASEC household mortgage status remains unresolved by H_TENURE alone; this option makes no tenure inference. diff --git a/docs/us-native-health-completion.md b/docs/us-native-health-completion.md new file mode 100644 index 000000000..c392fe511 --- /dev/null +++ b/docs/us-native-health-completion.md @@ -0,0 +1,57 @@ +# Optional native health coverage completion + +`run_us_survey_enrichment(..., health_completion=True)` enables a development +model for seven coverage categories that ACS does not separately observe. The +default is `False`; the existing literal health projection remains unchanged. +This option does not qualify a release or establish complete input coverage. + +The model uses the full authenticated original ASEC donor roster and original +household DESIGN weights, independently of which survey households were selected +for receiving support. Training excludes people with an unknown target or required +predictor. The predictors are observed age, sex and observation-time state. A +missing required ACS predictor causes refusal; there is no filled predictor or +assigned-block-state fallback. + +The seven targets come from the existing coverage field contract: + +| Coverage category | ASEC item | +| --- | --- | +| Marketplace | `NOW_MRK` | +| Non-marketplace direct purchase | `NOW_NONM` | +| Medicaid | `NOW_CAID` | +| Other means-tested coverage | `NOW_OTHMT` | +| TRICARE | `NOW_MIL` | +| CHAMPVA | `NOW_CHAMPVA` | +| VA | `NOW_VACARE` | + +Valid allocated source answers retain their values and allocation status. Blank +or unrecognized codes remain unknown. In particular, broader ACS coverage items +and ASEC `NOW_MCAID` do not become aliases for these narrower categories. + +The graph exposes the full donor projection, target encoding and donor selection, +one original-ACS predictor matrix, seven fitted QRF target artifacts and seven +ordered apply steps. Each ACS original receives one set of draws, then its two +existing clones receive the same values. The attachment preserves every incoming +column, structural ID, membership, weight and source observation. + +The original source-recode nodes retain their nullable values. On the receiving +population, `__known` and `__source_status` continue to describe source knowledge, +while a separate `__imputed` flag identifies modeled cells. A modeled true/false +value does not make an unobserved ACS answer source-known. ASEC unknown responses +and unknown ESI/IHS observations are outside this seven-category completion. + +ASEC coverage is observed at the 2025 interview and ACS at the 2024 interview. +Transporting the relationship between those years is an explicit development +assumption. Scientific qualification is pending: held-out performance, donor +exclusion and allocation patterns, rare coverage categories, broader ACS category +consistency and sensitivity to the interview-year difference still need review. +The chained model does not guarantee that every predicted seven-category tuple +appeared together in a donor record. + +Source authority stays with the live retained preparation and enrichment owner. +The private projection, graph artifacts and descriptive receipts cannot recreate +that authority. The host checks exact fitted-donor and draw/matrix identities, +reconstructs complete outputs on replay and requalifies sources at its final I/O +boundary. The full combined country-owner lifecycle and actual-data fit require +their own separately admitted verification; small invented graph tests do not +substitute for those checks. diff --git a/docs/us-native-immigration-source.md b/docs/us-native-immigration-source.md new file mode 100644 index 000000000..408f88344 --- /dev/null +++ b/docs/us-native-immigration-source.md @@ -0,0 +1,102 @@ +# Native immigration source projection + +`current_survey_immigration_source` provides the original evidence needed for +the humanitarian immigration consumer reviewed at Microcosm PR #779 head +`7ee36ac1bea125218912996b1030b77a208db963`. It assigns no status and runs no +stock draw, fit, calibration or country model. It is not attached to the graph. + +The new sibling source intentionally continues native integration +`cf04225c2c2c5c2e3eafe8ef5f49aa4c5feabd8f`; the freshly fetched main lacked the +native owner APIs. No old PR source tree, source-stage manifest, schema or seed +identity was adopted. The original humanitarian checkout was inspected +read-only. Subfleet's live-session inventory had no matching humanitarian owner; +its recent ledger contained only a completed #779 gate review. No message was +sent and no other checkout changed. + +## API and evidence + +An authorized native build can call: + +```python +from microcosm.build.us_runtime.current_survey_immigration_source import ( + borrow_cloned_immigration_evidence, + qualify_current_survey_immigration, +) + +qualified = qualify_current_survey_immigration(preparation) +columns = borrow_cloned_immigration_evidence(qualified, receiving_frame.person) +``` + +`preparation` must be the live issued `AuthenticatedSurveyPopulationPreparation`. +The qualifier authenticates the retained current ASEC CSV and ACS person ZIP, +exhausts their bounded records, verifies exact digests and counts, and rechecks +the owners after I/O. It returns private tables plus an aggregate receipt: + +- `asec_full_raw`: all income-2024 ASEC persons from the issued coverage roster, + indexed by the original parent `source_person_id`. Exact `PERIDNUM`, `PH_SEQ` + and `A_LINENO` remain strings. The full native-key roster and ages must match + the independent coverage owner. Only identity fields and the exact 23-field + #779 readset are retained. This full table survives smaller support selections + so a later national stock assignment cannot accidentally run on selected rows. +- `asec_selected_raw` and `acs_raw`: original literals for selected people, + indexed by preparation person ID, bound to the separate `origins` table. + ACS retains `CIT`, `POBP`, `YOEP`, `AGEP` and its original keys. These support + #779's source-aware profile; they do not supply an ASEC-UA indicator set or + authorize ACS status assignment. +- `person_evidence`: namespaced `immigration_source_*` literals and observation + year. Blank source tokens remain blank strings; another survey's columns are + missing. Codes, decimals, whitespace and native string IDs are not filled or + coerced. Allocation/response provenance is not inferred. + +The exact ASEC readset replaces the older consumer's `WSAL_VAL` / `SEMP_VAL` +proxies with `A_LFSR`; its order was checked against the exact PR source by AST. +`A_LFSR` is current labor-force evidence at the **2025 interview**, whereas +income refers to **2024**. ACS observation year is **2024**. The qualifier does +not silently apply the old arrival-code midpoints or stock-year controls to +these literals. No prior-income columns or person-weight column are read here. + +The clone function verifies the existing clone-0/1 ancestry, source channel and +native IDs, then returns detached column Series without changing the source or +receiver. It refuses column collisions. It performs no draws. Exact output and +receiver seals are checked after the last source validation. A graph host must +still authenticate its own receiving population; these columns and the JSON +receipt are not receiving or release authority. Keep the qualified object and +call `validate()` after subsequent relevant I/O before consuming its tables. + +The physical index/Series digest helpers are live-bound through the retained +preparation: `survey_population_preparation._modules()` includes +`asec_2024_native_population._modules()`, whose `parent_owner` is +`asec_current_money_source`. `_pure_final()` checks that full live function seal +before trusting frame digests. Regression tests replace either helper to mask +a changed projection and require refusal even with a warm verification epoch. +The source module stays under the scoped provenance scanner; only its fixed +field/owner-map dynamic selectors are reviewed, with no blanket provenance +exemption. + +## Required next integration + +The receipt's `missing_rule_admissions` lists the concrete unfinished steps: + +1. Check 2025 ASEC observation/arrival-code semantics against the exact rules. + Reconcile the rule period and stock year explicitly. +2. Bind the full ASEC native-key roster to its authoritative household design + weights and approved controls. Run the reviewed deterministic status draw + once on original sources, then select and transport its results. The full + literal table alone does not qualify the national stock mass or weight basis. +3. Admit the ACS transfer/assignment method with its own source eligibility and + cohort rules; do not populate absent ASEC fields with defaults. +4. Add the graph attachment under the retained native source and receiving + owners, preserving original status assignments across clones and replay. +5. Qualify country-model/export inputs and run the written-H5 immigration + composition and four HR1 smoke probes. A source projection cannot satisfy + their national stock bands or dollar floors. + +`status_assignment_performed` and `national_stock_alignment_qualified` describe +this projection's actual scope. They are not a permanent veto on a later +separately qualified consumer. The existing immigration implementation and +legacy release gates remain unchanged. + +Tests use real source issuers and original-member capture paths over invented +archives. They exercise full-versus-selected coverage, literal/unknown-state +preservation, exact clone transport, copied-owner rejection and mutations during +final I/O. No real Census/ACS data or country model was used in this slice. diff --git a/docs/us-native-other-disability-completion.md b/docs/us-native-other-disability-completion.md new file mode 100644 index 000000000..6558ae0d1 --- /dev/null +++ b/docs/us-native-other-disability-completion.md @@ -0,0 +1,85 @@ +# Native other-disability completion + +This optional development fragment estimates `disability_benefits` for selected +original ACS people aged 15 or older. The target comes from the existing qualified +ASEC two-slot other-disability adapter: workers' compensation and Social Security +disability remain separate leaves. Under-15 people and unresolved ASEC reports +remain unknown. This is not consumer-completeness or release certification. + +`qualify_current_asec_retirement_detail(..., full_original=True)` and +`qualify_current_asec_other_disability(..., full_original=True)` expose the complete +current ASEC source through their actual retained owner. The default remains the +selected-source view. The implementation bytes change; this feature preserves +the default semantic/source values, not historical file hashes. + +`qualify_current_survey_other_disability_completion` uses original ASEC DESIGN +weights, age, current WSAL employment income, and current SEMP self-employment +income. Known positive and source-qualified zero amounts train the maintained +zero-aware regime-gated QRF. Unknown targets are excluded explicitly; unknown +required predictors on an otherwise eligible donor or recipient refuse rather +than silently truncate the sample. No prior wages, PUF income, geography, +poverty targets or guessed disability-status predicates enter the fit. + +The evidence records every source reason's count and DESIGN mass, including +affirmed recipients whose dollar amount remains unresolved, and their mass +relative to known-positive donors. Excluding these observations can bias modeled +incidence. The diagnostic does not authorize a correction. Transport from ASEC's +2024 calendar-year dollars to ACS's rolling prior-12-month income concept also +needs scientific qualification. + +The graph has eight nodes when recipients exist: full-source CREATE, model +columns, donor FILTER, recipient projection, QRF fit, QRF apply, receiver +keep-all version, and attachment. The no-recipient path has four nodes and no +model fit. Donor and recipient populations are separate; the apply node never +sits on the ancestor of the donor FILTER. A visible version barrier precedes +the canonical write. Fit cache invalidation is currently conservative because +source-custody parameters include preparation/recipient bindings; this fragment +does not claim fit reuse across changed receiving selections. + +One draw per original ACS person is copied exactly to its two clones, with +joins checking original ID, native ID, source channel and clone indices. Draws +are ordered by ascending original person ID. Receiving-row permutations do not +change assignment, but the sequential QRF stream is not invariant to changing +the original recipient roster. The later AGI `clone2` extension needs its own +reviewed variable-clone attachment contract before these features can integrate; +this fragment deliberately continues to require exactly clones 0 and 1. + +## Explicit graph storage projection + +Frame columns are `disability_benefits`, `survey_other_disability_known`, +`survey_other_disability_reason`, `survey_other_disability_model_applicable`, +`survey_other_disability_canonical_known`, and `survey_other_disability_value_origin`. +An ACS modeled zero remains source-unknown, with origin `modeled`. Known ASEC +positive/zero values have origin `observed`; unresolved source values have origin +`source_unresolved`; under-15 values have origin `outside_model_universe`. + +The graph currently does not support nullable `Float64` or `Int16` columns. +Original source reports are therefore retained in the declared +`microcosm.us.other_disability_attachment@1` artifact. Its `source_reports` +section preserves selected-original ASEC axes, dtypes, numeric backing bytes, +null masks and string values; `artifact_only_source_fields` enumerates fields +that are absent from the Frame projection. Its physical seal binds the original +source representation independently of the canonical output. Graph inspection +can locate those reports at `survey_other_disability.attach`; this is an explicit +projection, not a claim of unchanged report storage in Frame columns. + +New canonical storage is float64 with NaN for unknown. An incumbent float32 or +float64 column can be replaced only in the new version and only if every known +or modeled value round-trips exactly, including signed zero. Unsupported dtype, +report ownership collision, or lossy conversion refuses. No unknown is filled +with zero for compatibility. + +## Integration boundary + +`_CompletionBoundary` is private borrowed custody, not an issuer. The calling +host retains and validates the genuine preparation **and** the actual receiving +run before graph/store I/O and after final I/O, including required replay, with +pure checks last. It supplies the receiving version and exact typed terminal +artifact binding. A detached Frame, JSON receipt or fitted model confers no +source authority. + +This change supplies an optional fragment and registry; it does not switch on a +production host, modify default enrichment, or integrate later PUF placement. +Invented fragment tests use a genuine invented source owner and an explicitly +invented receiver. Host integration and actual-data qualification require +separate review and bounded admission. diff --git a/docs/us-native-other-disability-source.md b/docs/us-native-other-disability-source.md new file mode 100644 index 000000000..45d182730 --- /dev/null +++ b/docs/us-native-other-disability-source.md @@ -0,0 +1,149 @@ +# Native ASEC other-disability source + +`qualify_current_asec_other_disability(preparation)` reduces the two published +CPS ASEC disability income slots to one non-workers-compensation leaf. It adds +no raw reader of its own: it borrows +[`current_asec_retirement_detail_source`](../packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py), +which already qualifies `DIS_VAL1/2`, `DIS_SC1/2`, `DIS_YN` and their allocation +and topcode literals, and cross-checks the two retained `DIS_VAL` amounts — and +only those — against the authenticated money owner's own bits. The public entry point +calls that owner's qualifier, so the single member capture and its +requalification happen there and are not repeated or replaced here; the pure +`compose_other_disability` and `project_other_disability` entry points read no +member at all. This module issues no source capability, fits nothing, and +attaches no canonical consumer input. Callers retain the genuine owners and +requalify after their relevant I/O. + +## What the leaf is + +The retired eCPS pipeline summed the two annual slots wherever the slot's +source code was not 1, workers' compensation. That arithmetic lives in +`disability_benefits.derive_us_disability_benefits_from_asec`, and this adapter +reuses that function and its archived parameter dictionary rather than +restating them; a drift in either refuses with `ARCHIVED_ROSTER` or +`IMPLEMENTATION_CHANGED`. The exclusion is bound to the printed meaning of code +1, not to the bare integer, so renaming the published label refuses too. + +Social Security disability and workers' compensation are separate leaves and +are not folded in here. The `DIS_CS` and `DIS_HP` work-limitation answers the +detail owner keeps beside these slots describe leaving or limiting work; they +are not this leaf's dollars, and their allocation flags are deliberately +outside this family's provenance. + +## What stays unknown + +The archived arithmetic returned a number for every row whose four literals +it could read, and never an unknown; a row that was never asked the question +read whatever its literals happened to hold, which on a consistent +never-asked row is zero and on a contradictory one need not be. This adapter does not. Each slot is classified into one closed vocabulary +(`SLOT_KINDS`); only the first four resolve, and the rest leave the person +unknown: + +| slot kind | contributes | when | +| --- | --- | --- | +| `reported_source_slot` | the published amount | yes receipt, readable non-workers-compensation code, nonzero amount | +| `excluded_workers_compensation` | 0 | yes receipt, a readable code 1, and a readable amount, whatever that amount was | +| `unused_source_slot` | 0 | yes receipt, code 0, zero literal: no source in this slot | +| `nonreceipt_slot` | 0 | a "no" to `DIS_YN` with code 0 and a zero literal | +| `niu_not_observed_zero` | unknown | `DIS_YN = 0` with code 0 and a zero literal | +| `outside_age_universe` | unknown | under the printed 15+ reporting universe | +| `unresolved_slot_reporting` | unknown | missing, malformed, out-of-range or contradictory literals | + +A receipt answer alone does not settle a slot. A "no" or an NIU answer that +arrives beside a nonzero amount or a populated source code is a contradiction, +not a nonreceipt or an NIU: the owner labels it as such and the slot stays +unresolved. + +A workers' compensation slot whose amount cell is missing or unreadable is +`unresolved_slot_reporting`, not a known zero: a record that cannot be read +here is not evidence that its source code was read correctly. + +A person's leaf is known only when both slots resolve. A "yes" answer with no +populated source in either slot is `affirmed_receipt_without_reported_source` +and stays unknown rather than becoming a zero. A recipient zero under a +reported non-workers-compensation source stays ambiguous, because the printed +entry says `0 = none or niu`. Under-15 rows and NIU rows stay unknown +wherever the archived arithmetic would have read them; whatever it read — zero +on a consistent row, a positive number on a contradictory one, or nothing at +all when a literal will not parse — is recorded in +`other_disability_archived_arithmetic_amount` and +`other_disability_archived_arithmetic_evaluable`, and never adopted. + +That replay runs over the four fields **as the owner parsed them**, so it is +evaluable only where each read inside its own printed domain. The archived +function itself accepted any finite number, so a source code outside the +published table — `11`, say — is a row the retired pipeline would have summed +and this replay reports as unevaluable. It is a replay of the admitted +literals, not of every byte the old pipeline consumed. + +## Provenance is not knownness + +Every published allocation flag for this family (`I_DISYN`, `I_DISSC1`, +`I_DISSC2`, `I_DISVL1`, `I_DISVL2`) is evaluated against its own printed +conditional universe, read from the detail owner's entries rather than +restated. A flag outside its universe, unpopulated, or unreadable is labeled as +exactly that, in that order of precedence: a flag whose universe cannot be +resolved, or that sits outside it, is answered before its own literal is +judged, so an unpopulated or malformed literal is only reported where the flag +was in its universe to begin with. + +Two family-level answers are reported, under names that say which is which. +`other_disability_published_flag_origin` is the routing owner's reading, which +is universe-blind by that owner's charter: a nonzero flag read outside its own +printed universe can still count as a publisher allocation there — unless some +flag's literal is unresolved, which that owner answers first. +`other_disability_allocation_status` is this module's own reading, built from +the per-flag labels above, so a flag outside its universe does not allocate +this family. Its precedence is: an in-universe publisher allocation settles the +family; otherwise an unresolved universe or literal, then an unpopulated flag, +then a clear in-universe zero; and `no_flag_in_its_universe` only where every +printed universe — all of which are `X > 0` here — is false. Neither qualifies or disqualifies a receipt: +`other_disability_allocation_qualifies_receipt` is `False` on every row. +Topcode flags describe only the dollars this leaf admits, and +`topcode_corrected` stays `False`. + +## Clone transport + +`attach_other_disability_columns(values, receiving)` is the thin attachable +interface. It joins on the pre-clone source id, checks that each receiving row's +spine source identity matches the qualified row's `native_person_id`, requires a +uniform clone index set per original, and copies one qualified row to every +clone of that person. No draw is taken and no value is redrawn; the returned +receipt records `redraw_issued: False` and `draws_consumed: 0`. Rows on an arm +this source never observes — ACS today — carry a null amount, `known` false and +the reason `acs_source_unobserved`. They are never zero. + +The attachment returns the canonical leaf under `disability_benefits` and every +other column under the `survey_other_disability_` report prefix. It returns +columns to the host and mutates no receiving cell. + +## Boundaries + +The pure `project_other_disability` function accepts a supplied detail basis so +its semantics can be inspected and tested; neither it nor +`other_disability_values_seal` confers authority. `compose_other_disability` +seals the borrowed detail owner before and after projection and rechecks its +own active implementation dependencies. That fence binds every module-level +constant of this file, detached rather than aliased; every function defined in +it; the file's bytes as read at the end of its own import, so an edit after +that point refuses instead of being reported — the window between the loader's +read and that one is not attested — and an **enumerated** set of borrowed callables — the detail +owner's qualifier, literal projection, slot status, amount pair, member capture +and amount comparison and seal; the routing owner's receipt, literal, +code-frame, capture, digest and allocation helpers; and the archived arithmetic +with its own input guard and parameter dictionary. That list is deliberate, not +a transitive closure: it covers the callables this module's own path depends on +for the literals it reads, and it does not attest anything those callables in +turn call. `qualify_current_asec_other_disability` additionally rechecks the +retained preparation and native issuance. + +The detail owner cross-checks the retained `DIS_VAL1/2` amounts against the +authenticated money owner's bits, but `DIS_YN`, `DIS_SC1/2` and the allocation +and topcode literals reach this leaf from the member capture alone. That is why +the capture path is inside the fence, and it is a boundary a consumer should +know about rather than a property this module can prove on its own. + +ACS completion, SSI eligibility, tax treatment, topcode correction, host +composition, calibration and release remain separate work owned elsewhere. The +tests use invented literals; passing them establishes these contracts, not +actual-data quality, coverage or release readiness. diff --git a/docs/us-native-retention-seal.md b/docs/us-native-retention-seal.md new file mode 100644 index 000000000..d6817a263 --- /dev/null +++ b/docs/us-native-retention-seal.md @@ -0,0 +1,607 @@ +# US native build: replacing the replayed-population object comparison with a content seal + +Design authority for the change Max chose on 2026-09-17 — option (b) of the +transport lane's report §10 question 2 +(`experiments/native-scale-transport/out.md`), whose own authority is +`docs/us-native-scale-transport.md` §3d and §4. + +The problem, restated from that authority and re-derived here: the financial +runner compares its own replay against the executor's observation with +`atomic.same_replayed_population(expected[n], observed[n])` +(`graph_atomic_survey_financial.py:1796` and `:1948` **at the base**, now the +two `same_replayed_population_seals` calls at `:1882` and `:2042`), and the +replay cannot be built until `run_graph` has returned. So nineteen detached `Population` objects +must exist between observation and comparison — about 290 GiB at full source — +unless the comparison becomes a comparison of content seals. + +Every line number is at this branch's head unless it names another. Line +numbers move; re-derive the ones you rely on. In `survey_population_replay.py` +specifically, the seal's `import hashlib` shifted every line from 10 onward by +**one** against the base, so a citation into that file taken from the base +branch reads one line early. + +## 0. The report's proposed mechanism does not hold, and this is the first thing to say + +Report §10 question 2(b) says: + +> `_population_stamp` already folds everything `same_replayed_population` +> compares except the *type* assertions, which can be made on arrival without +> retaining. + +That sentence is wrong in **both** directions. Both halves are proved by +scripts committed under `experiments/native-retention-seal/`, run at this head. + +**`_population_stamp` is too strict — it would refuse a required replay.** +`probe_stamp_vs_comparison.py` builds a `US_SCHEMA` population whose +`nullable_integer` column carries `_data = [11, 71, 73]` under a mask +`[False, True, True]`, round-trips its frame through +`ContentStore.put_frame`/`load_frame` (which zeroes values beneath a null mask, +`store.py:456`), and reports: + +``` +expected masked _data: [11, 71, 73] +actual masked _data: [11, 0, 0] +same_replayed_population: ACCEPTS +_population_stamp equal : False eb6879921de1 0400e853a2dd +_frame_identity equal : True 774ed8184472 774ed8184472 +``` + +`same_replayed_population` **accepts** that pair, because +`NONCANONICAL_NULL_BACKING` (`survey_population_replay.py:80-83`) deliberately +permits the actual side's null backing to be canonically zeroed. The stamp +refuses it. `_population_stamp`'s own docstring says exactly this +(`survey_atomic_geography.py:231-237`): "Never persist it or compare it across +reconstructions: it folds physical storage parts (masked `_data` under nulls)". +A stamp-equality seal would turn every `resume="require"` replay red. + +**`_frame_identity` is too weak — it would miss real defects.** +`probe_frame_identity_gaps.py` finds three discriminations `same_replayed_frame` +makes that `_frame_identity` does not: + +``` +nan payload 0x...11 vs 0x...12 same_replayed_frame=REFUSES NATIVE_BITS +_frame_identity=same +flags allows_duplicate_labels same_replayed_frame=REFUSES +TABLE_TYPE_OR_FLAGS _frame_identity=same +quiet vs signalling NaN same_replayed_frame=REFUSES NATIVE_BITS +_frame_identity=same +``` + +`_cell` (`survey_population_preparation.py:732-745`) maps **every** NaN to +`None`, so `_frame_identity` spells all NaN payloads `null`; and nothing in it +folds `DataFrame.flags`. It is also inapplicable to a non-`US_SCHEMA` frame at +all (`FRAME_TYPE`, `:977-981`). + +**So the seal is purpose-built and lives beside the comparison it replaces**, in +`survey_population_replay.py`, folding exactly the bytes each comparison +compares — no more, no less. It reuses neither existing seal. + +## 1. Every field and assertion `same_replayed_population` compares today + +Read off `survey_population_replay.py` at this head. **Kind** is `unary` for a +predicate about one side that can be asserted on arrival, `binary` for a +comparison between the two sides that a seal must fold, and `directional` for +the one predicate that is not symmetric. + +### 1a. Population level (`:180-209`) + +| # | predicate | code | kind | +|---|---|---|---| +| P1 | `type(expected) is Population`, `type(actual) is Population` | `POPULATION_TYPE` | unary, both sides | +| P2 | `type(version) is str` (both) and `expected.version == actual.version` | `POPULATION_CONTEXT` | unary + binary | +| P3 | `tuple(sorted(owners.items()))` equal — order-**insensitive** | `POPULATION_CONTEXT` | binary | +| P4 | `all(type(v) is str for v in actual.owners.values())` | `POPULATION_CONTEXT` | unary, actual only | +| P5 | `tuple(weight_kind.items())` equal — order-**sensitive** | `POPULATION_CONTEXT` | binary | +| P6 | `all(type(v) is WeightKind for v in actual.weight_kind.values())` | `POPULATION_CONTEXT` | unary, actual only | +| P7 | `type(mass_ledger) is tuple` (both) | `POPULATION_CONTEXT` | unary | +| P8 | `all(type(r) is MassRecord ...)` over both ledgers | `POPULATION_CONTEXT` | unary | +| P9 | `canonical_json([asdict(r) for r in mass_ledger])` equal | `POPULATION_CONTEXT` | binary | +| P10 | `tuple(design_weights)` equal — key **order** | `POPULATION_CONTEXT` | binary | +| P11 | per entity `_array_bytes_equal(design_weights[e], …)`: `type is np.ndarray`, dtype equal, `not hasobject`, shape equal, `tobytes()` equal | `DESIGN_BYTES` | unary + binary | + +### 1b. Frame level (`:133-177`) + +| # | predicate | code | kind | +|---|---|---|---| +| F1 | `isinstance(expected, Frame)`, `isinstance(actual, Frame)` | `FRAME_TYPE` | unary | +| F2 | `schema` equal | `FRAME_CONTEXT` | binary | +| F3 | `entities` equal (ordered tuple) | `FRAME_CONTEXT` | binary | +| F4 | `links == () ` on **both** | `FRAME_CONTEXT` | unary | +| F5 | `canonical_json(_encode_frame_metadata(metadata))` equal | `FRAME_CONTEXT` | binary | +| F6 | `all(type(r) is MassChangeRecord ...)` over both mass logs | `FRAME_CONTEXT` | unary | +| F7 | `canonical_json([asdict(r) for r in mass_log])` equal | `FRAME_CONTEXT` | binary | +| F8 | `weighted_entities` equal | `FRAME_CONTEXT` | binary | +| F9 | `type(table) is pd.DataFrame` (both) | `TABLE_TYPE_OR_FLAGS` | unary | +| F10 | `left.flags == right.flags` | `TABLE_TYPE_OR_FLAGS` | binary | +| F11 | strata name: `_name_bytes` equal | `STRATA_NAME` | binary | +| F12 | per weighted entity `left.kind is right.kind` | `WEIGHT_BYTES` | binary | +| F13 | per weighted entity `_array_bytes_equal(values)` | `WEIGHT_BYTES` | unary + binary | + +### 1c. Axis (`:122-130`), applied to every table index, every table columns +axis, and the strata index + +| # | predicate | code | kind | +|---|---|---|---| +| A1 | `type(expected) is type(actual)` | `AXIS` | binary | +| A2 | `expected.identical(actual)` — pandas: `equals()` **and** every name in `type(index)._comparables` **and** exact class **and** dtype | `AXIS` | binary | +| A3 | `not isinstance(expected, pd.MultiIndex)` | `AXIS` | unary (expected; A1 carries it to actual) | +| A4 | `_name_bytes(name)` equal — `canonical_json(_axis_name_payload(name))` | `AXIS_NAME` | binary, plus unary encodability (`UNSUPPORTED_AXIS_NAME`) | +| A5 | `_series` over the axis array | the S codes | | + +### 1d. Series (`:49-112`), applied to every table column, every axis array and the strata + +| # | predicate | code | kind | +|---|---|---|---| +| S1 | `type(x) is pd.Series` (both) | `SERIES_DTYPE_OR_LENGTH` | unary | +| S2 | `type(expected.dtype) is type(actual.dtype)` | `SERIES_DTYPE_OR_LENGTH` | binary | +| S3 | `expected.dtype == actual.dtype` | `SERIES_DTYPE_OR_LENGTH` | binary | +| S4 | `len(expected) == len(actual)` | `SERIES_DTYPE_OR_LENGTH` | binary | +| **masked branch** — `BooleanDtype`, or `ExtensionDtype` and integer | | | | +| S5 | `_array_bytes_equal(lm, rm)` — the null masks | `MASKED_STORAGE` | binary | +| S6 | `lm.dtype is bool`; `lm.shape == (n,)`; `type(ld) is type(rd) is np.ndarray`; `not ld.dtype.hasobject`; `ld.shape == rd.shape == lm.shape` | `MASKED_STORAGE` | unary + shape | +| S7 | `ld.dtype == rd.dtype` | `MASKED_STORAGE` | binary | +| S8 | `_array_bytes_equal(ld[~lm], rd[~rm])` — the **present** values | `PRESENT_BITS` | binary | +| S9 | `_array_bytes_equal(ld, rd) or bool(np.all(rd[rm] == 0))` | `NONCANONICAL_NULL_BACKING` | **directional** | +| **string branch** — `StringDtype` | | | | +| S10 | `storage` equal and `(na_value is pd.NA)` parity | `STRING_POLICY` | binary | +| S11 | `_array_bytes_equal(isna(l), isna(r))` | `STRING_MASK` | binary | +| S12 | present cells: `type(a) is str and type(b) is str and a == b` | `STRING_VALUE` | unary + binary | +| **object branch** — `is_object_dtype` | | | | +| S13 | per cell `_encode_object_scalar(a) == _encode_object_scalar(b)` | `OBJECT_VALUE` | binary, plus unary encodability (`UNSUPPORTED_OBJECT`) | +| **native branch** | | | | +| S14 | `not isinstance(dtype, pd.api.extensions.ExtensionDtype)` | `UNSUPPORTED_EXTENSION_DTYPE` | unary | +| S15 | `_array_bytes_equal(to_numpy(l), to_numpy(r))` — type, dtype, `not hasobject`, shape, bytes | `NATIVE_BITS` | unary + binary | + +Forty-four rows, under the **twenty-two** refusal codes +`survey_population_replay.py` raises (counted off the module, not off this +table). The admitted dtype set is narrow, which bounds the problem: `_series` +refuses `CategoricalDtype`, `Float64Dtype` and `DatetimeTZDtype` with +`UNSUPPORTED_EXTENSION_DTYPE` (measured), so only masked integer/boolean, +`StringDtype`, `object` and plain numpy dtypes get through. + +The forty-four rows are a reading of the code, not a census of every mutation +that can reach it: §5's battery drives **564** comparisons, over twenty of the +twenty-two codes plus **42** pairs both paths accept, and two codes are +unreachable (`FRAME_TYPE`, because `Population` validates its own frame; +`STRING_POLICY`, because `StringDtype.__eq__` already compares storage and +`na_value`, so `SERIES_DTYPE_OR_LENGTH` fires first). Those figures are the +committed receipt's own +(`experiments/native-retention-seal/battery-receipt.json`: `comparisons`, +`agreements`, `by_verdict`), not a count taken by hand; they were 116 and +fourteen before the fix pass that closed three divergences in the object-axis +fold, whose new cases mostly drive the agreement driver too. + +## 2. Which of them `_population_stamp` already folds + +`_population_stamp` (`survey_atomic_geography.py:230-268`) folds, in order: +`canonical_json({"frame": _frame_identity(frame), "version", sorted owners, +weight_kind items, mass_ledger asdict})`; then, for every entity column in +order and then the strata, a length-prefixed `_storage_parts(series, +slice(None))` pair (physical values, null bitmap); then, per design-weight +entity, `canonical_json((entity, str(dtype), shape))` and `values.tobytes()`. + +| group | folded? | note | +|---|---|---| +| P2 version, P3 owners, P5 weight_kind, P9 mass ledger | yes | in the header `canonical_json` | +| P10 design key order, P11 design bytes | yes | the design loop | +| F2 schema, F3 entities, F8 weighted entities | yes | through `_frame_identity` / `encode_us_frame_context` | +| F5 metadata, F7 mass log | yes, but through `_normative_metadata` and `_json_data`, not `_encode_frame_metadata` | a **different** normalisation from the one the comparison uses | +| F12/F13 weights | yes | `_frame_identity` folds kind, dtype, shape and `tobytes()` | +| S5/S8 masks and present values | yes | `_storage_parts` | +| S15 native bytes | yes | `_storage_parts` | +| A4 axis names | partly | `_frame_identity` folds index `names` and the strata name; not through `_axis_name_payload` | +| **S9 directional null backing** | **folded the wrong way** | it folds the masked `_data`, so it **refuses** what the comparison accepts — §0 | +| **F10 `DataFrame.flags`** | **no** | nothing folds it | +| **A1/A2 exact index class and `_comparables`** | partly | `type(index).__name__` only; not the qualified class, not `freq` | +| all unary type assertions (P1, P4, P6, P7, P8, F1, F4, F6, F9, S1, S6, S12, S14) | no | they are assertions, not content | + +So the honest summary is: `_population_stamp` folds most of the *content*, gets +the one directional predicate backwards, misses `DataFrame.flags` outright, and +— unavoidably — folds none of the unary assertions. + +## 3. Which it does not, and how each is preserved + +Three mechanisms, applied in this order. + +**(a) Every unary assertion is made on arrival, in the seal constructor, with +the same refusal code.** `replayed_population_seal(population)` runs P1, P4, +P6, P7, P8, F1, F4, F6, F9, S1, S6, S12, S14, A3 and the two encodability +assertions (`UNSUPPORTED_OBJECT`, `UNSUPPORTED_AXIS_NAME`) against the one +population it is given, raising the identical +`SURVEY_POPULATION_REPLAY_` string. Four of them (P4, P6, and the +`actual`-side halves of the rest) are written today as assertions about +`actual` only; sealing both sides makes them fire on `expected` too. That is +strictly more closed, it fires earlier, and no code changes. + +Two are re-expressed rather than dropped, because they are not properties of +one side: + +> **Corrected 2026-09-18.** The two bullets below described a *dtype token* of +> strings and booleans, and a `_comparables` digest with a `repr()` fallback. +> **Neither is in the code**, and an adversarial pass over this note caught it: +> §4(1) of this same note argues that a spelling-based token would *miss* a +> defect the code catches, so the two sections contradicted each other and this +> was the wrong one. What the code does is the opposite of tokenising — it +> **retains the small objects** and applies the identical operator. The bullets +> now say that. + +* **S2/S3, the dtype pair.** `_series_seal` retains + `head = (type(dtype), dtype, length)` — the dtype **class object** and the + dtype **object itself**, not a spelling of either — and `_same_series_seal` + compares them as `expected_class is actual_class and expected_dtype == + actual_dtype`, which is `type(a.dtype) is type(b.dtype) and a.dtype == + b.dtype` with the operands retained rather than re-derived. Retaining the + objects is what makes the exact cases work, and §4(1) has the sharpest one: + `np.longlong` against `np.int64` has equal `str()`, equal `dtype.str` and + equal bytes, and a different dtype **class**. A token keyed on spellings + would accept that pair; this refuses it `SERIES_DTYPE_OR_LENGTH`. + `CategoricalDtype`, whose `__eq__` is not an equivalence relation, never + reaches here because S14 refuses it. + `test_the_dtype_head_is_faithful_to_the_predicate_it_replaces` sweeps every + ordered pair of the admitted dtype census through `_series` **and** through + the seal and requires the same verdict with the same code. Its earlier + version asserted `type(a) is type(b) and a == b` against + `(type(a), a) == (type(b), b)` — the same predicate written twice, calling no + seal function — which an adversarial pass caught; replacing the retained head + with `str(dtype)` now turns it red. +* **A1/A2, `type(...)` and `Index.identical`.** `_axis_seal` retains the + **class object** `type(index)`, the `_comparables` names, the raw comparable + **values**, the dtype object, the value fold and the name seal. + `_same_axis_seal` compares the class with `is`, the names with `==`, and each + comparable value with `bool(one == other)` element-wise — which is exactly + what `Index.identical` applies, and deliberately not a tuple comparison, + which would short-circuit on identity and accept a comparable whose own + `__eq__` refuses itself (pinned by + `test_a_comparable_whose_equality_refuses_itself_still_refuses`). Folding the + `_comparables` **tuple itself** closes a gap neither existing seal has: + `pd.DatetimeIndex._comparables == ['name', 'freq']` (measured at pandas + 3.0.3), so two `DatetimeIndex`es with identical values, dtype and name but + different `freq` are **not** `identical()` while their bytes are equal, and + an index class that declares a comparable this module has never seen refuses + `AXIS` rather than folding `None` for it. + +**(b) Every binary comparison folds exactly the bytes that comparison +compares.** Not an approximation of them, and not a different normalisation: +F5 folds `canonical_json(_encode_frame_metadata(metadata))` because that is +what `same_replayed_frame` compares — *not* `encode_us_frame_context`'s +`_normative_metadata`, which is what `_frame_identity` folds. S15 folds +`to_numpy(copy=False).tobytes()` with its dtype and shape, so NaN payload bits +and `-0.0` survive, which `_cell`'s `None` mapping destroys. F10 folds +`allows_duplicate_labels` — generically, over `type(flags)._keys` (measured: +`{'allows_duplicate_labels'}`), so a pandas release that adds a flag is folded +without an edit here. + +**(c) The one directional predicate, S9, is folded as three values rather than +one digest**, which is what makes an exact reproduction possible at all. Per +series the masked branch records + +* `mask_digest` — the null mask bytes (S5), +* `present_digest` — the present values' bytes with dtype and shape (S7, S8), +* `full_digest` — the whole `_data` buffer's bytes, +* `null_backing_canonical` — `bool(np.all(data[mask] == 0))` for **this** side, + +and the seal comparison requires +`present_digest` equal **and** +(`actual.null_backing_canonical` **or** `full_digest` equal), +which is `_array_bytes_equal(ld, rd) or np.all(rd[rm] == 0)` term for term, per +series, with the same direction and the same `NONCANONICAL_NULL_BACKING` code. +A single population-wide flag would **not** reproduce it: today's disjunction is +evaluated per column, and one non-canonical column must not force whole-frame +byte equality on the others. + +The seal record is therefore a nested structure of 32-byte digests and small +tokens whose size is **O(columns), not O(rows)** — for a US frame of six +entities and a few hundred columns, tens of kilobytes per node against about +1.5 GiB for the retained population at 1/10. + +## 4. What is no longer proved + +**Nothing about the content.** Every one of the forty-four discriminations in +§1 still refuses the same defect with the same code; §5's battery is the proof +obligation and it is discharged mutation by mutation, in both directions. + +Three things change in character rather than in coverage, and this note states +them rather than leaving them to be found. All three were established by +adversarial passes over the finished seal, not by writing it. + +> **Corrected 2026-09-18.** There was a fourth item here, and it was the one +> place this note said something false about what a run proves. It claimed that +> on an object-dtype axis the fold of the store codec's bytes "is never weaker +> than `equals` and is stricter on exactly those pairs", and that "every defect +> is still refused". A second adversarial pass ran both paths and found the +> fold **weaker** in one case and **stricter** in another it had not named, so +> for a period this change did *narrow* what a run proves. The fold was +> rewritten to reproduce the equivalence classes `Index.equals` actually has, +> the object-axis code no longer moves, and the withdrawn item is set out below +> the list rather than deleted. + +1. **Byte equality becomes sha256 equality.** Today `same_replayed_population` + compares the bytes; the seal compares a 256-bit digest of those bytes. This + is the same substitution the whole runtime already rests on — + `_population_stamp`, `_frame_identity`, `preparation_sha256`, every node key + and every store address are sha256 digests of content — so it introduces no + assumption the build does not already make. It is not nothing, and it is + written here so the owner can see it. Every predicate that is **not** a byte + comparison keeps the object it applies to and the identical `is`/`==`: the + dtype, its class, the index class, the axis name, the `WeightKind`, the + flags. That is why `np.longlong` against `np.int64` — equal dtypes, equal + `str()`, equal `dtype.str`, equal bytes, different dtype class — still + refuses `SERIES_DTYPE_OR_LENGTH`, which a token built from spellings would + have missed. +2. **Unary assertions fire earlier.** A `MultiIndex`, an unencodable object + cell, a `Float64` column or a non-`str` owner value is refused when the + population is observed rather than when it is compared, with the same code. + For `expected`-side defects that is unchanged in timing; for `actual`-side + defects the refusal moves from after `run_graph` returns to inside the + observer callback, which `run_graph` documents as refusing the run + (`executor.py:2453-2454`). +3. **A one-sided defect now fires before a two-sided difference.** When a + population carries both — an unsupported dtype *and* a changed version, say + — today's comparison reports whichever its own order reaches first, and the + seal reports the one-sided one, because the seal is built before anything is + compared. The same set of codes; a different precedence between them. No + pair in the battery carries two defects at once, and none is manufactured. +Every non-object axis kind is exact, including the three `array_equivalent` is +byte-tolerant for: `float`, `complex` and `bool` all collapse NaN payloads, NaN +sign, signed zeros and bool bytes outside `{0, 1}` before folding, so a +byte-only difference still reaches `NATIVE_BITS`. Since the second adversarial +pass, a **masked-integer** axis is exact too: `np.asarray` on a masked integer +array returns float64 with NA as NaN, so an `Int64` axis's fold was a float +fold and lost every distinction above `2**53`; it now takes the array's exact +object view. + +### 4a. The withdrawn fourth item, and why it is left here + +It said that on an object-dtype axis a difference reports `AXIS` rather than +`OBJECT_VALUE`, because `Index.equals` there is an element-wise `==` over +arbitrary Python objects that no digest reproduces, and that the fold of the +store codec's bytes is therefore "never weaker than `equals` and stricter on +exactly those pairs" — `True` against `1`, `-0.0` against `0.0`. + +**Both halves were false**, and each was reproduced by running the two paths: + +* **Weaker.** Two values with equal codec bytes need not be `==`, when one + carries its own `__eq__`. The comparison refused `AXIS`; the seal + **accepted**. That is a run accepting a replay the comparison refuses, and + for as long as it stood, this change did narrow what a run proves. +* **Stricter, in an unnamed case.** pandas holds `None` and every NaN + interchangeable on an object axis and the codec spells them apart, so the + comparison **accepted** and the seal refused — a green run turning red. + +The fold is now `_object_axis_equivalence`, which reproduces the equivalence +classes `Index.equals` actually has, **measured on this pandas pin** rather +than read off its source: `{None, NaN}` is one class, `pd.NA` and `pd.NaT` are +each their own, bool/int/float share one numeric class by exact value, and +`str` and `bytes` are each their own. The one thing no digest can represent — +a value's own `__eq__` — refuses `AXIS` when the seal is built, on either +operand. The byte-exact discriminations the codec makes and `==` does not are +not lost: they reach `OBJECT_VALUE` through the arm `_axis` itself falls +through to, which is what the comparison reports, so **the object-axis code no +longer moves at all.** + +So the honest answer to "what is no longer proved" is: nothing, now — and it +was not nothing between the first version of this note and the fix. That is +why the item is withdrawn in place rather than deleted. The report §10's +*stated mechanism* was wrong and §0 of this note says so and replaces it; the +*decision* — compare content seals, not objects — is implemented as chosen. + +## 5. The proof obligation + +For every row of §1, a test that mutates exactly that property in one of two +otherwise-equal populations and asserts: + +* `same_replayed_population` refuses, with its code — this half already exists + for most rows in `packages/microcosm-build/tests/test_us_survey_population_replay.py` + and is extended to cover the rest; +* the **seal** comparison refuses the same mutation with the same code. + +Plus, in both directions, the pairs today's comparison **accepts** — the store +round trip of §0, the object-scalar equivalences of +`test_object_scalar_equivalence_uses_actual_store_types` — which the seal must +also accept. A mutation the seal misses is a finding, not a test to delete. + +**And a seeded sweep beside the hand-built battery, because the battery tests +the cases its author thought of.** Two of the three divergences §4a records +lived behind a battery of 116 agreeing comparisons and a claim that every +defect is still refused. `experiments/native-retention-seal/agreement_fuzz.py` +walks a deterministic pseudo-random sweep over seven axis kinds, a 21-value +object pool and nine column mutations, driving every pair through both paths: +3,852 pairs, nine refusal codes, 466 mutual acceptances, **0 disagreements**. +It would have found two of the three. Any change that replaces a predicate with +a fold should carry one. + +## 6. Fail-closed after the change + +**Every refusal code that exists today still fires on the same defect.** All +twenty-two are raised by the seal constructor (one-sided) or the seal +comparison (two-sided), with the same `SURVEY_POPULATION_REPLAY_` prefix and +the same suffix. **None is retired**, and §5's receipt records which comparison +reached which. + +Six codes are **added**, none of them about a population's content, and this +note lists them rather than saying "none" and being wrong: + +| code | guards | +|---|---| +| `SURVEY_POPULATION_REPLAY_FRAME_SEAL_PROTOCOL` | a seal record that is not a frame seal of this protocol version | +| `SURVEY_POPULATION_REPLAY_POPULATION_SEAL_PROTOCOL` | the same for a population seal | +| `SURVEY_POPULATION_REPLAY_SEAL_TYPE` | `seal_identity` handed something that is not a seal record | +| `ATOMIC_OBSERVER_RETENTION` | the observed roster is not exactly the declared consumers, or an arrival seal's identity has moved | +| `FINANCIAL_SEALED_NODE_SCOPE` | a sealed node reaching the completion-boundary or manifest stamp arms, which the retention flag makes unreachable | +| `RETAIN_EVERY_NODE_POPULATION_FLAG` | the private retention flag is not a bool, or is set together with `child_property` | + +**Five of those six are pinned; the sixth is not, and this note says which.** +The three seal codes are pinned in +`test_us_survey_population_replay.py` — `test_a_foreign_frame_seal_record_refuses_frame_seal_protocol`, +`test_a_foreign_population_seal_record_refuses_population_seal_protocol` and +`test_seal_identity_refuses_anything_that_is_not_a_seal_record`, each verified +to go red when its own guard is reverted — and `RETAIN_EVERY_NODE_POPULATION_FLAG` +and `FINANCIAL_SEALED_NODE_SCOPE` in `test_us_graph_atomic_survey_financial.py`. +`ATOMIC_OBSERVER_RETENTION` has no test: its first conjunct +(`set(observed) == declared_consumers`) is reachable only from inside a real +nineteen-node run with an observer made to retain the wrong roster, and its +second conjunct re-derives `seal_identity` from the very record whose identity +it compares against, so it cannot fail while a seal record's `repr` is stable. +That asymmetry is stated rather than smoothed over, and it is a question for +the owner rather than a silent gap. +**`FINANCIAL_NODE_POPULATION_CHANGED` after the change** means: *for a node +whose `Population` object the run still retains — the declared-consumer roster — +the object's in-process `_population_stamp` differs from the one recorded when +the run was issued; and for every other node, the seal record the run retains +is not the one recorded at issuance.* Both arms refuse the same class of defect +the one arm refuses today — something the run retained about a node's +population changed after issuance — and neither arm is vacuous, because the +seal record is re-digested rather than merely re-read. + +**The declared-consumer roster.** `result.financial_population is +observed[final_node]` (`graph_atomic_survey_financial.py:1897` at the base, +`:1986` here) is an identity check, so the objects a caller holds must be the +objects the observer saw. The base financial run declares three: the final node (`financial.ATTACH_NODE`, or +`_tax_module().GATE_NODE` when the tax rebase is enabled), +`financial.ATTACH_NODE` itself (`legacy_population`, `:1821` at the base), and the property +population (`observed[final_node]` before the rebase moves `final_node`, +`:1818` at the base). Those three are retained as objects and stamped exactly as today. +Every other observation is sealed on arrival and dropped. + +**The completion host is a consumer of the base run's whole roster, and this +note says so because reading the code was not enough to find it — a test was.** +`graph_survey_completion_host` runs its own `run_graph` with its own observer +(`:781-794`), retains every observation, and pins their object ids in its own +identity fence (`:543-545`, `:572`). Those parts do not move. But at `:814-816` +it reads the **base** run's `state.node_populations` as its own expected +populations for the base-graph nodes, compares them against its own +observations, and hands them to `atomic._states`, which reads +`population.frame.table(entity)` for every structural node +(`graph_atomic_survey_population.py:85-96`). Seals cannot serve that. + +So the base run takes a private flag, `_retain_every_node_population`, whose +only caller is `run_atomic_survey_financial`'s own recursive base call when +`child_property is not None` (`:1356-1371`). With it set, the declared-consumer +roster is every node and the run retains exactly what it retains today. The +flag and `child_property` are mutually exclusive, which the runner requires. +`_node_population_seals` accepts both shapes. + +This is the one place where the memory saving does not apply, and it is the +completion path rather than the base 19-node path the measured wall belongs +to. Removing it means giving the completion host the same seal treatment and +finding a frame-free form for `_states`; that is a second change with its own +argument. + +**The residual risks this note will not hide.** An earlier version of this +paragraph declared one that **does not exist**: a `repr()` fallback in the +`_comparables` fold. There is no such fallback — `_axis_seal` retains the raw +comparable values and `_same_axis_seal` compares each with the same `==` +`Index.identical` applies, so the comparables arm introduces no divergence in +either direction. The three that are real, each reproduced before being written +down: + +1. **`seal_identity` depends on `repr`.** It is `sha256(repr(seal))`, and a + seal retains live dtype, class and `WeightKind` objects whose canonical JSON + does not exist. That is the *stamp*, not the comparison: two seals are + compared field by field by `same_replayed_population_seals`, and `repr` only + decides the digest a run retains per node for + `FINANCIAL_NODE_POPULATION_CHANGED`. Two objects that are `==`, of the same + class, with different `repr` would give different stamps — but the stamp is + only ever compared with another taken in the same process, from the same + object, which its own docstring says. +2. **An object-dtype axis cannot be folded exactly by any digest**, because + `Index.equals` there is an element-wise `==` over arbitrary Python objects. + `_object_axis_equivalence` reproduces the equivalence classes that operator + actually has — measured, not assumed — and refuses `AXIS` at seal + construction for a value that is not `==` to the plain builtin the codec + encodes it as, which is the only case a digest cannot represent. Before that + guard the seal **accepted** a pair the comparison refuses; §4 records it. +3. **A one-sided refusal now takes precedence** over a two-sided difference on + the same operand, because the one-sided assertions fire when the seal is + built. Both refuse; the code between them can differ. The clearest instance + is an object axis carrying a `datetime64` leaf and a value difference: the + comparison reports `AXIS`, the seal `UNSUPPORTED_OBJECT`. + +No index class in this runtime reaches the exotic comparables case at all: US +frames carry `Index` and `RangeIndex`, whose `_comparables` is `['name']` +(measured), and `graph_context.encode_us_frame_context` requires group entities +to carry a default unnamed `RangeIndex` (`:130-143`). + +## 7. The `microcosm-graph` half + +`_observer_snapshot` (`executor.py:356-408`) detaches by round-tripping every +entity and link table plus the strata through `pickle.loads(pickle.dumps(..., +protocol=5))` — one full independent copy per reached node, cache hits included +(the round trip is `executor.py:373`, inside `_observer_snapshot` at +`:356`; the call that takes one per reached node is `:2787-2791`, which at +the base was the single line `:2774`). An observer that only seals never retains and never mutates, so +it needs no detached copy. + +`run_graph` gains `_population_observer_detach: bool = True`, private and +paired with `_population_observer`. When it is `False` the observer receives the +live admitted population and no snapshot is allocated. Default behaviour is +unchanged, the keyword enters no key, no receipt and no cache record, and the +change imports nothing from the US runtime. + +**It is deliberately not called `population_retention`.** That keyword existed +on PR #893's original branch (commit `6c0f24c77`, layer 08) and was dropped; it +made the **manifest's attached population views** lazy against the content +store, which is a different target from the observer's per-node detached +snapshots (`docs/us-native-scale-transport.md` §4). Reusing the name for this +would invite exactly the confusion that document warns about. + +Invariants that do not move, each with the test that pins it +(`packages/microcosm-graph/tests/test_graph_executor.py`): + +| invariant | test | +|---|---| +| exactly once per reached node, in `compiled.order`, cold and on hits | `:4130-4166` | +| an observer's exception refuses the run with the observer's type | `:4172-4179` | +| a mutating, retaining observer cannot move a key, a manifest or a store object — **in the default detaching mode** | `:4183-4248` | +| with no observer, no snapshot is allocated | `:4368-4374` | + +The new tests are the shape of the last: with an observer **and** +`_population_observer_detach=False`, a monkeypatched `_observer_snapshot` that +raises must never be called; the run's manifest key and every store object must +equal a no-observer run's; and the observer must receive the same object the +executor holds, which is the mode's whole point and is what the caller is +declaring it will not mutate. + +That declaration is the mode's one honest cost: in this mode the executor no +longer *enforces* that the observer cannot reach execution state, it *trusts* +the caller's declaration. The default mode still enforces it, and this mode is +opt-in and private. + +## Integration note (native integration line, 2026-09-23) + +This note was written for `native-retention-seal` (#950). The native +integration line (`native-integration-20260923`) merges it into a tree that +already carried a second, opt-in answer to the same retention problem: +compact witness retention (`ee7049379`, `experiments/native-completion-retention/implementation.md`), +selected by `_population_retention="compact"`. The merge keeps both and +separates them by profile. Line numbers above are #950's and do not apply to +the merged file. + +- **`_population_retention="all"` (the default) is this note's seal.** Every + node is sealed on arrival with `replayed_population_seal`; only the declared + consumers are retained, detached with `_observer_snapshot`; the executor is + called with `_population_observer_detach=False`; replay comparisons use + `same_replayed_population_seals`; `node_populations` carries the seal record + for every dropped node. +- **`_population_retention="compact"` is unchanged.** It witnesses every node + with `_survey_population_witness.population_witness`, retains its own + compact roster, keeps the executor's detachment + (`_population_observer_detach=True`) and compares with the witness or with + `same_replayed_population` exactly as before. +- **The declared-consumer roster is the compact roster.** Both profiles take it + from `_compact_retained_roster`: `financial.ATTACH_NODE`, the development + attach node when source-qualified development inputs are enabled (the + integration line makes it `final_node`, so it is an identity consumer #950 + did not have), the property attach and the tax gate. +- **`_retain_every_node_population`** is accepted by `_run_survey_financial` + and by both public wrappers (`run_atomic_survey_financial`, + `run_pre_geography_survey_financial`). The recursive completion base call + sets it exactly when the profile is `"all"`; under `"compact"` the completion + host reads witnesses instead of the base roster, so the flag is refused + there under the existing `RETAIN_EVERY_NODE_POPULATION_FLAG` code. +- **Naming.** The section above explains why #950 avoided the name + `population_retention`. The integration line's `_population_retention` + keyword is `ee7049379`'s observer-retention profile, not the dropped #893 + keyword that made manifest views lazy. + +No run on actual data has measured either profile's memory effect at scale. diff --git a/docs/us-native-row-ceilings.md b/docs/us-native-row-ceilings.md new file mode 100644 index 000000000..8d394e012 --- /dev/null +++ b/docs/us-native-row-ceilings.md @@ -0,0 +1,379 @@ +# US native build: the row-count ceilings, and the rule their new values follow + +The [transport lane](us-native-scale-transport.md) lifted the 64 MiB +whole-roster receipt ceilings and, in its §5, recorded a second family it did +not touch: row-count bounds that a full-source native US build meets. This note +is that family's argument. + +It is one rule, not five decisions. Max decided (2026-09-17) that these are +lifted as one change with one argument rather than one build at a time, and a +rule is the only thing that makes the sixth case decide itself. + +## 1. The rule + +> **A bound moves only if a full-source native build meets it. A bound that moves +> becomes four times the measured full-source count of exactly what it counts, +> rounded up to the next whole million.** + +Three clauses, each load-bearing. + +**"only if a full-source build meets it."** Most of the bounds censused in §3 do +not bind, and most of those never will, because what they count is an upstream +file's own size rather than a roster this build chooses. A rule that raised +everything uniformly would weaken real checks for no gain. This one leaves them +alone by construction, and §4 says why each must stay. + +**"of exactly what it counts."** The commonest way to get one of these wrong is +to bound the wrong roster. The 19-node path carries at least five distinct +quantities that differ by more than an order of magnitude — the upstream ACS +file's rows, the selected ACS roster, the selected ASEC roster, the stacked +roster, and the combined clone — and a ceiling raised against the wrong one is +either dead or still binding. §2 measures each, and every moved constant carries +a comment naming the one it counts. + +**"four times … rounded up to the next whole million."** The multiple is the +transport lane's: `MAX_ROSTER_BYTES` is `64 × MAX_SEGMENT_BYTES` = 4 GiB, which +that lane measured at 3.9× a full-source preparation receipt. Adopting the same +headroom keeps one law across both families. The rounding makes the constants +legible and the rule checkable at a glance: divide any moved constant by the +count its comment names and the answer is between 4 and 4.6. + +The rule is not only written here. `test_us_native_row_ceilings.py` holds the +measured counts and asserts that each moved ceiling is exactly what the rule +produces from its count, so a later edit that drifts fails a test rather than a +build. + +### 1a. What the rule is not for + +**Byte transports take the transport lane's argument, not this one.** That +lane's answer to a payload that outgrows its cap is a segmented stream under one +explicit total, not a larger single cap, and it proved the segmented bytes +identical to the unsegmented ones. Raising such a cap here would contradict the +neighbouring argument rather than extend it. §5 names the byte ceilings that +bind and leaves them to it — including one that binds harder than anything this +lane moved. + +## 2. The counts, measured rather than extrapolated + +Every number below comes from the recovered 1/1000 pilot preparation artifact +(`_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json`, +sha256 `34b362d85d2f06acd255390976d76343aff45114958ba382edb10ffa3789a8a0`), and +specifically from its **catalogues** rather than its rosters. + +That distinction is the point. A roster at 1/1000 has to be scaled to say +anything about full source, and scaling a stratified sample introduces error. +The catalogues do not: they are counts of the whole upstream ACS and ASEC files, +and they are identical in a 1/1000 artifact and a full-source one. What makes +them answer the full-source question is this identity, which +`experiments/native-row-ceilings/roster_census.py` asserts before reporting +anything: + +``` +ACS occupied_hu 1,348,408 + + institutional_gq 84,422 + + noninstitutional_gq 98,784 + + ASEC households 55,762 + = 1,587,376 == selection.supplied_households +``` + +"The whole catalogue" and "what a full-source selection supplies" are therefore +the same set, and the catalogue's counts *are* the full-source counts: + +| quantity | full source | at 1/10 | +|---|---:|---:| +| ACS households (selectable; excludes 100,355 vacancies) | 1,531,614 | 153,161 | +| ACS persons | 3,422,888 | 342,288 | +| ASEC households | 55,762 | 5,576 | +| ASEC persons | 142,125 | 14,212 | +| **stacked households** | **1,587,376** | 158,737 | +| **stacked persons** | **3,565,013** | 356,501 | +| combined-clone households | 3,174,752 | 317,475 | +| combined-clone persons | 7,130,026 | 713,002 | + +The transport lane's §5 quoted ~3,471,000 stacked and ~6,943,000 cloned persons. +Those were the 1/1000 artifact's 2.186869 persons per household extrapolated, +and the catalogue figures supersede them — about 2.7% higher, which changes no +verdict but is the number each new constant is derived from. + +One further split matters, because two of the moved bounds are met by one +channel rather than the total: **`_normalized_source_copy` normalizes the ACS +and ASEC native frames separately**, and the ACS channel carries 96.0% of the +persons. A bound met per channel is met by 3,422,888, not by 3,565,013. + +## 3. The moved ceilings + +| constant | was | now | counts | full source | × | +|---|---:|---:|---|---:|---:| +| `acs_pums.MAX_EXACT_HOUSEHOLDS` | 1,000,000 | **7,000,000** | exact ACS household keys in one snapshot | 1,531,614 | 4.57 | +| `acs_pums.MAX_EXACT_PERSON_ROWS` | 1,000,000 | **14,000,000** | `NP` summed over the selected ACS households | 3,422,888 | 4.09 | +| `acs_person_coverage_columns.MAX_SELECTED_ROWS` | 1,000,000 | **14,000,000** | requested ACS person keys | 3,422,888 | 4.09 | +| `survey_observed_age.MAX_ROWS` | 2,000,000 | **14,000,000** | one channel's person rows | 3,422,888 | 4.09 | +| `survey_origin_budget.MAX_GROUPS` | 1,000,000 | **7,000,000** | allocation instructions = selected households | 1,587,376 | 4.41 | +| `current_survey_geography.MAX_HOUSEHOLDS` | 524,288 | **7,000,000** | selected households | 1,587,376 | 4.41 | +| `asec_demographic_source._MAX_PERSONS` | 600,000 | **14,000,000** | retained ACS persons (the larger of its two rosters) | 3,422,888 | 4.09 | + +Each refusal keeps its code, its exception type and its expression; only the +number moves. + +**One consequence, named so a reader does not have to find it.** +`MAX_SELECTED_ROWS` at 14,000,000 now sits *above* `MAX_ROWS` at 6,000,000 in the +same module. That is not an inconsistency: `MAX_SELECTED_ROWS` bounds +*caller-supplied* person keys, which can exceed what an archive holds, while +`MAX_ROWS` bounds the archive's own rows. But it does mean the effective ceiling +on that path is 6,000,000, by containment. Setting `MAX_SELECTED_ROWS = MAX_ROWS` +would be tighter *and* provable rather than chosen — and it would make the rule +two rules, which is why this note keeps one. + +| constant | refusal | raises | +|---|---|---| +| `MAX_EXACT_HOUSEHOLDS` | `"ACS exact selection requires bounded unique raw native keys."` | `ValueError` | +| `MAX_EXACT_PERSON_ROWS` | `"ACS selected complete roster exceeds native person budget."` | `ValueError` | +| `MAX_SELECTED_ROWS` | `"ACS coverage selected person count is outside the bound"` | `ValueError` | +| `survey_observed_age.MAX_ROWS` | `"SURVEY_OBSERVED_AGE_ROW_BOUND"` | `ValueError` | +| `MAX_GROUPS` | `"GROUP_COUNT_BOUND"` | `SurveyOriginBudgetError` | +| `current_survey_geography.MAX_HOUSEHOLDS` | `"CURRENT_SURVEY_GEOGRAPHY_HOUSEHOLD_COUNT"`, `…_PROJECTION_STORAGE` | `ValueError` | +| `asec_demographic_source._MAX_PERSONS` | `"ACS_ROWS"`, `"MEMBERSHIP_ROWS"`, `"CLASSIFY_ROWS"`, `"DEMOGRAPHIC_ROWS"` | `ValueError` | + +### 3a. `survey_origin_budget.MAX_GROUPS`, which the transport lane left open + +Its §5 recorded this one as "not established — the group count was not derived +here". It is established now, and it binds. `_initial` builds its groups with + +```python +instructions = graph.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] +) +_require(0 < len(instructions) <= MAX_GROUPS, "GROUP_COUNT_BOUND") +``` + +and `allocation_instructions` opens with +`_require(len(household_origins) == len(plan.selected), "ORIGIN_COUNT")`. One +instruction per selected household, so the group count *is* the selected +household count: 1,587,376 at full source against a 1,000,000 bound. + +Where it runs, stated precisely rather than implied. **No node in the +nineteen-node graph executes `freeze_survey_origin_budget`**, which is where +`GROUP_COUNT_BOUND` is checked — the recovered run's `graph.json` lists all +nineteen and none is a budget node, and the completion host has none either. +`graph_atomic_survey_financial` does call into the module, but only for +`_config_payload` and the `_live()` producer seal, neither of which reaches +`_initial`. `survey_age_calibration` and `graph_survey_budget` are what execute +it, over the same full-source selection, so the count and the verdict stand. + +## 4. Why a bound stays a bound, and why some must not move at all + +**Deleting a bound is not the cheap version of raising it.** Each of these +stands between a malformed or runaway input and an unbounded allocation, and the +refusal is what makes the failure legible and early rather than an OOM kill deep +in a multi-hour build with no statement of which roster was wrong. A ceiling at +4× the real thing still refuses a 10× runaway, which is the shape a real mistake +takes: a fraction argument off by a decimal place, a selection that forgot to +filter, a key list built from the wrong catalogue. + +Three classes must not move at all, and this note names them so a later reader +does not mistake a low number for an oversight. + +**An upstream file's real size.** `acs_person_coverage_columns.MAX_ROWS` and +`acs_native_coverage_binding.MAX_SOURCE_ROWS`, both 6,000,000, bound how many +records the reader will stream out of the ACS archive. The 2024 ACS person file +holds 3,422,888 of them. That bound is a structural assertion about a file this +build does not produce and cannot check any other way, and the roster it feeds +is separately bounded by `MAX_SELECTED_ROWS`. Raising it would trade a real +check for nothing: a selection cannot exceed the source it is drawn from. + +**A fixed-width encoding.** The `PAYLOAD_MAX_BYTES = len(MAGIC) + 4 + +HEADER_MAX_BYTES + ROW_BYTES * MAX_HOUSEHOLDS + 32` family in +`asec_housing_status`, `asec_housing_universe` and `asec_income_observations` +computes a byte budget *from* a row ceiling, so moving the row ceiling moves the +wire format a reader depends on. `survey_observed_age.MAX_EXACT_FLOAT64_INTEGER` += 2**53 is the same kind of thing one level down: the module's own docstring +calls it "a representation limit, not a scientifically permitted maximum age", +and it bounds an age's value, not how many rows carry one. + +**A quantity no selection fraction grows.** The ASEC codec's +`asec_current_money.MAX_PERSONS` (1,000,000) and `MAX_HOUSEHOLDS` (400,000) +count the ASEC source scope's own `person_rows` and `household_rows`. The +catalogue measures those at 142,125 persons in 55,762 households, and a +full-source survey selection does not change them. **The transport lane's §5 +listed `asec_current_money.MAX_PERSONS` among the bounds a full-source build +meets; it does not, and this note corrects that.** + +## 5. What still binds, and whose argument it is + +**Nineteen bounds bind at full source at the base branch's values.** §3 moved +seven of them. The other twelve belong to the transport lane's argument — a +segmented stream under one explicit total — not to this one. + +(The census in §6 reports fourteen rather than nineteen because its agents read a +moving tree: five of the seven were lifted while it ran, and they correctly +report the post-lift values at which those five no longer bind. Each of those +rows records both numbers. The two the census still shows as binding — +`current_survey_geography.MAX_HOUSEHOLDS` and +`asec_demographic_source._MAX_PERSONS` — were moved afterwards, on its evidence.) + +### 5a. The test the rule applies + +Three of the remaining bounds *look* like row ceilings, and one of them was +moved here after that test was applied to it. The test is a single question, +answered from code rather than from the constant's name: + +> **Does the binding site materialise a per-row payload, or does it stream?** + +A bound in front of a materialised payload cannot be usefully raised on its own, +because the payload's own byte cap refuses first and at a smaller number. A +bound in front of a streaming digest has nothing behind it, and the rule applies. + +| bound | value | what is behind the binding site | verdict | +|---|---:|---|---| +| `current_survey_geography.MAX_HOUSEHOLDS` | 524,288 | `_projection_digest` streams one `_encode(values, maximum=4096)` at a time into a `hashlib.sha256`; the module's only bytes are a 64 KiB summary receipt | **streams — moved in §3** | +| `current_survey_household_roles.MAX_PERSONS` | 2,097,152 | `_projection_bytes` is `table.reset_index().to_json(orient="table").encode()` — the whole per-person table in one string — under `graph_current_survey_household_roles.MAX_ARTIFACT_BYTES` = 64 MiB | materialises — transport's | +| `current_child_property_income_source.MAX_ROWS` | 600,000 | `_json` is one `json.dumps(value)` under `MAX_PROJECTION_BYTES` = 64 MiB | materialises — transport's | + +The middle row is the clearest case for why the test matters, and it is measured +rather than reasoned: through the module's own encoder, a person row costs +**320.08 bytes**, so that 64 MiB cap admits **209,661 persons — 5.88% of source** +against a row bound that admits 58.8%. The byte cap refuses **ten times earlier**, +so raising `MAX_PERSONS` alone would move nothing at all. +`experiments/native-row-ceilings/roles_projection_size.py` measures it; the table +it encodes is invented but faithfully shaped, and the per-row cost is a +difference between two row counts so the fixed schema preamble cancels. + +A fourth bound needed the same test plus one more question, and passed both. +`asec_demographic_source._MAX_PERSONS` (600,000) is enforced at five sites that +count **two different quantities**: a fixed ASEC three-cohort roster of 432,523 +rows, and the retained ACS person roster, which a full-source build grows to +3,422,888. One constant over two rosters takes the larger, so the rule produces +14,000,000 from the ACS arm. Three things had to hold before it could move, and +each was read rather than assumed: + +- **It is not a fixed-width encoding.** `rows` lives as a JSON integer in the + header; the module's only `struct.pack` is `" **`survey_origin_budget.MAX_PAYLOAD_BYTES` (64 MiB) admits 87,838 households — +> 5.53% of source.** That is below one tenth, and below the 96,860-household +> preparation-receipt ceiling the transport lane lifted. A full-source payload is +> 1.13 GiB, 18.07× the cap. + +Measured, not estimated: `experiments/native-row-ceilings/origin_budget_size.py` +builds one faithful origin record through the module's own `_reference` and +`_json` at full-source household-id widths and gets 764 bytes per group, +including the two `household_ids` and two `group_indices` entries the header +carries for each group's two clone roles. + +**It also cannot be raised on its own, and that is a structural fact rather +than a preference.** `survey_origin_budget._json` is +`graph._bounded_json(value, MAX_PAYLOAD_BYTES)`, and `_bounded_json` opens with + +```python +_require(type(limit) is int and 0 < limit <= 64 * 1024**2, "TRANSPORT_LIMIT") +``` + +so the shared encoder refuses any cap above 64 MiB before it encodes a byte. A +larger number in this module would not loosen the bound; it would refuse the +module. The 64 MiB is the encoder's, and moving it is the transport change. + +The consequence for this lane is stated rather than glossed: **lifting +`MAX_GROUPS` is necessary and not sufficient.** `GROUP_COUNT_BOUND` is checked +in `_initial` before the payload is built, so at full source it fired first; with +it lifted, the refusal moves to `TRANSPORT_LIMIT` in the same module. That is the +correct outcome of one argument applied to one class of bound, and the remaining +ceiling is a real piece of work for whoever carries the segmented transport into +this module. + +## 6. The census + +`experiments/native-row-ceilings/census.json` is the census. Every `MAX_*` row, +household, person, group or byte bound in +`packages/microcosm-build/src/microcosm/build/us_runtime/` reachable from the +19-node financial graph and the 45-node pilot graph was read at this head — **146 +bounds across five module families** — each with its enforcement site, its +refusal code and exception type, what it protects, the count it meets at 1/10 +and at full source, and whether it binds. Every verdict that claimed a +full-source build meets the bound, or that the bound guards an encoding width or +an upstream file's real size, then went through an adversarial pass that read the +code again and tried to refute it — **42 verdicts**, and no headline binding call +was overturned. + +The result, read as the agents read it: **14 bounds bind at full source, 6 of +them at 1/10.** Five of the seven §3 moved are absent from that fourteen because +they were lifted mid-census and the agents correctly reported the post-lift +values; each of those rows carries both numbers. **At the base branch's values +the figure is nineteen**, of which §3 moved seven, leaving twelve for the +transport argument. §5a and §5b say which and why. + +## 7. Pins + +These modules are hashed into stage implementation identities, so the pins were +re-derived through their generators rather than assumed. + +| pin | generator | result | +|---|---|---| +| `graph_implementation_inventory.json` → `contracts` (all 124) | `graph_implementation._dependency_contract(payload, name, _covered_imports(name, inventory))` | unchanged by these edits — a constant's value is not an import, an unbound use or a resource access | +| `acs_native_coverage_binding._ACCEPTED["acs_pums.py"]` | `sha256(.read_bytes())`, the same call the check makes | **moved**, `6ecf79f0…` → `e79a2a4e…` | +| the other three `_ACCEPTED` entries | same | unchanged; this branch does not edit those modules | +| all ten `implementation_manifest(stage)` | `graph_implementation.implementation_manifest` | built | + +`experiments/native-row-ceilings/repin.py` reports all of this in one run and +`regenerate_accepted_pin.py` produced the moved value. Neither pin was +hand-edited, and both generators are idempotent. + +What moves and is not a committed pin: each US stage's `implementation_hash` is +over its whole module roster, so editing any inventoried module moves it and +with it every node key and store address. #935 recorded that this is true of any +change to these files including a comment. diff --git a/docs/us-native-scale-transport.md b/docs/us-native-scale-transport.md new file mode 100644 index 000000000..cb7ef210e --- /dev/null +++ b/docs/us-native-scale-transport.md @@ -0,0 +1,480 @@ +# US native build: receipt transport and per-node population retention + +Design authority for the two hard limits the 2026-09-16 cost attribution found +in the native US graph build +(`experiments/us-native-executor-cost-attribution-20260916.md` on the +`microcosm-exec-profile` worktree, §5 and §9): a whole-roster receipt that +refuses above 6.1% of the US source, and a per-node population retention that +needs about 300 GiB at full source. + +Every line number below is at `5ff889814`, the branch point, unless it names a +later head. Line numbers in the cost-attribution report are its own tree's and +are offset from these. + +## 1. What refuses today, measured rather than inferred + +The preparation receipt is **one JSON byte string**, built by +`prepare_authenticated_survey_population` and handed to `_encode` in a single +call (`survey_population_preparation.py:1679`) under +`MAX_PAYLOAD_BYTES = 64 * 1024**2` (`:50`), which refuses `PAYLOAD_LIMIT` +before each chunk append (`:104-109`). Twelve top-level keys; ten of them are +fixed-size. The two that are not are `"selection": _plan_document(plan)` +(`:1698`) and `"origins": origins` (`:1699`), which carry **one record per +source household and per person**: + +| block | built at | per-row shape | +|---|---|---| +| `selection.selected`, `.excluded` | `_plan_document` `:240-253` | one `_value`-flattened plan row per supplied household | +| `origins.households` | `_origins` `:946` | one 7-field dict per stacked household | +| `origins.entities` | `_origins` `:870` | one `[new_id, channel, source_id]` row per row of **each** of the six US entities | +| `origins.persons.rows` | `_origins` `:965` | one 10-element row per stacked person | + +A second, independent budget guards the same lists: `_bounded_append` +(`:816-819`) `_encode`s each row on its own and refuses `ORIGIN_LIMIT` against +the same 64 MiB constant. `_plan_document` opens its own budget (`:247`) and +`_origins` opens another (`:844`), so there are three 64 MiB ceilings on one +document, plus the final `_encode`. + +**The size law, read off the pilot's own artifact rather than modelled.** The +1/1000 preparation receipt survives in `_recovered/pilot-runs/` — 1,136,063 B at +`native19-required-20260912/run/financial-artifacts/preparation.json`, byte +identical to `native19-v2`'s copy and to that run's +`graph-store/objects/04/0481…/payload.bin`, sha256 `34b362d85d2f06ac…`. It +records `supplied_households = 1587376` with 1,584 selected. Re-encoding it +through the repository's own canonical separators reproduces 1,136,063 exactly. +Decomposed: `origins` 718,800 B (households 269,000; entities 224,755; persons +210,924; storage_transitions 13,268; age normalization 762), `selection` +393,395 B (selected 385,122; excluded 7,250; cells 923), `producer` 21,135 B, +everything else 2,733 B. The blocks that grow with the draw are +1,096,821 B over 1,584 selected households: + +> **bytes(f) = 39,242 + 692.4375 × 1,587,376 × f** + +At f = 1 that is **1,099,197,911 B = 1.024 GiB, 16.4× the limit**. Encoding it +costs about 53 CPU-s at full source (the report's §5 measured 33.3 µs per source +household), so the ceiling is a transport-shape decision and not a performance +trade-off. + +**Five ceilings, in the order they bind.** Each is computed from the same +committed artifact, or from the allocation payload's own measured 501,597 B at +1,584 households: + +| # | ceiling | refusal | households | share of source | +|---|---|---|---|---| +| 1 | preparation `_encode` | `PAYLOAD_LIMIT` | 96,860 | **6.10%** | +| 2 | `_origins` row budget | `ORIGIN_LIMIT` | 150,916 | **9.51%** | +| 3 | allocation payload stream | `ALLOCATION_LIMIT` | 211,924 | 13.35% | +| 4 | `_plan_document` row budget | `ORIGIN_LIMIT` | 270,284 | 17.03% | +| 5 | allocation structural pre-check `ALLOCATION_MAX_BYTES // 128` | `ALLOCATION_LIMIT` | 524,288 | 33.03% | + +A 1/10 build is 158,737 source households, so **#1 and #2 both bind and #3 +does not**; a full build needs all five, because 524,288 is below the source's +own 1,587,376. #5 bounds the intermediate maps `allocation_instructions` builds +rather than the payload — its implied 127-byte row budget is below the payload's +220-byte structural overhead alone, so it can never be the binding *payload* +constraint — but it does refuse a full-source selection, so its ceiling moves +with the others and keeps its code. + +**The store is not the constraint.** `ContentStore.put_bytes` +(`store.py:984`) and `put_frame` (`:919`) carry no byte cap, and `put_frame` +already writes one `.npy` per entity and column under +`objects//` with a `meta.json` payload table recording sha256 and +size per file (`store.py:140-153`), re-verified on every read by +`_verified_meta` (`:227`). Every 64 MiB ceiling in this pipeline is owned by +the US runtime, not by the graph. + +## 2. The new transport + +### 2a. What must not move + +`preparation_sha256` is `_sha(payload)` (`graph_survey_population.py:493`, +`:513`), it is folded through `_digest` into the CREATE and ALLOCATION node +params (`:1226`, `:1235`, `:1254`), and `params` reaches `node_key` through +`normative(node)` (`keys.py:288`). So the document's bytes reach every node key +and every store address. That is why the transport changes and the **canonical +byte stream does not**. + +The stream is already defined independently of its materialisation. +`_chunks(value)` (`:97-102`) yields the canonical JSON of the document; +`_encode` (`:104-110`) concatenates those chunks under a bound, and `_digest` +(`:113-116`) hashes the same chunks under no bound. sha256 is a streaming hash +over exactly those bytes in exactly that order, so + +> `_sha(_encode(document)) == _digest(document)` + +for every document `_encode` accepts, and `_digest` keeps returning that value +for documents `_encode` refuses. The new transport digests the stream and +materialises it in bounded segments; it does not re-shape the document. + +### 2b. The segmented, content-addressed receipt + +`_segment_stream(document)` replaces the single `_encode` call for the two +whole-roster receipts. It walks `_chunks(document)` once and: + +* accumulates into a segment buffer, and whenever appending the next chunk + would exceed `MAX_SEGMENT_BYTES` (= the old `MAX_PAYLOAD_BYTES`, 64 MiB) it + closes the segment — so **no accumulation in the encoder is ever larger than + the value the old ceiling allowed**, and `PAYLOAD_LIMIT` still guards each + one, with the same code, at the same number; +* records each closed segment's sha256 and size, and updates one running sha256 + over the whole stream; +* refuses `ROSTER_LIMIT` above `MAX_ROSTER_BYTES`, an explicit resource ceiling + that is *not* a transport shape: it is the total the process is willing to + spend, set far above full source (see §2d), and it is the only new refusal. + +The result is a `_RosterReceipt` handle carrying `digest` (identical to +`_digest(document)`), `size`, and the segment table. The preparation path also +writes the segments to a content-addressed spill under the run's `snapshots` +directory — `/.segment` plus a small `/header.json` naming +the protocol, the whole-stream digest, the size, and the ordered segment table — +which is the same header-plus-bodies shape `ContentStore.put_frame` already +uses, and which gives the receipt a re-verifiable on-disk form independent of +the graph store. + +**The write surface, disclosed rather than left to be found.** This module +previously wrote nothing but `snapshots.mkdir`. The spill adds +`Path.write_bytes` for each segment and for `header.json`, and +`graph_implementation`'s `_RESOURCE_CALLS` (`:126-150`) covers *reads* — +`open`, `read_bytes`, `read_text`, `read_csv`, `load`, `files`, `joinpath`, +`import_module`, `exec`, `eval` and their neighbours — so a write is invisible +to `resource_accesses_sha256` by construction and no committed pin would notice +a redirected spill. Every component of the path is therefore checked here +instead: `root` is the snapshot root `_root()` already validated as symlink-free +and disjoint from the source root (`SNAPSHOT_LOCATION`), `name` must be a Python +identifier (`ROSTER_NAME`), each segment's own filename is the sha256 of its +bytes, the spill directory must not be a symlink (`ROSTER_SPILL_LOCATION`), and +an existing segment or header must be a regular file of the recorded size +(`ROSTER_SEGMENT_CHANGED`, `ROSTER_SPILL_LOCATION`). Presence is decided by +`lstat` and not by `Path.exists()`, because a broken symlink does not exist and +would be written straight through — a hole the first draft of this code had and +its own test caught. + +`read_all()` reassembles, re-hashes each segment against its recorded digest +and the whole against `digest`, and refuses `ROSTER_LIMIT` above +`MAX_MATERIALIZED_BYTES`. The two consumers that need whole bytes — the +`preparation` and `allocation` opaque artifacts, which reach +`ContentStore.put_bytes` through `KernelResult.artifacts` — call it. So +`AuthenticatedSurveyPopulationPreparation.payload` stays `bytes` and **every +existing consumer is unchanged**: `_sha(entry[1])`, `type(value.payload) is +bytes`, the `CANDIDATE_MISMATCH` byte comparison, and the artifact publication +at `graph_survey_population.py:510`. + +This is deliberately honest about what it does and does not buy. It removes +every ceiling that binds below full source and it keeps the encoder's peak +allocation at one segment. It does **not** remove the final materialisation: +1.02 GiB of bytes exists once, at full source, because +`KernelResult.artifacts` is a mapping of `bytes` and `put_bytes` takes whole +bytes. Removing that last copy needs a streaming artifact channel in +`microcosm-graph`, which is main-only and is §4's sibling. At full source +1.02 GiB against a ~26 GiB working set is not what stops a build. + +### 2c. The row budgets + +`_bounded_append`'s `ORIGIN_LIMIT` (ceilings #2 and #4) is a bound on the +Python row list, not on a transport. It keeps its code and its per-row +`_encode`, and its ceiling becomes `MAX_ROSTER_BYTES` — the same explicit +resource ceiling — so the refusal still fires, on the same rows, with the same +message, at a number a full-source build does not reach. + +### 2d. The allocation payload + +`_allocation_payload` (`graph_survey_population.py:344-369`) is already a +stream with a per-row bound: it emits `_bounded_json(_instruction_document(row), +4096)` per household into one bytearray with a pre-append check against +`ALLOCATION_MAX_BYTES` (`:362`) and a post-loop re-check (`:367`). Its bytes are +byte-identical to the canonical whole-tree encoding, which +`test_us_graph_survey_population.py:304` pins — an identity that holds only +because `"households"` sorts before all six metadata keys — so the same +streaming-digest argument applies. + +It is built inside `SurveyPopulationAllocationKernel.run`, which has no +filesystem path and must not write to the store itself, so it takes the +in-memory half of §2b: bounded segments in a list, each guarded by +`ALLOCATION_LIMIT` at the unchanged 64 MiB, under `MAX_ROSTER_BYTES` in total, +materialised once. `allocation_sha256` (`:441`) is unchanged in derivation and +in value. + +Ceiling #5, the `// 128` structural pre-check, moves to +`ALLOCATION_ROSTER_BYTES // 128` = 33,554,432 households. It bounds +`allocation_instructions`' intermediate maps, and its old 524,288 is **below** +the source's 1,587,376, so a full-source selection refused there too. The code +is unchanged. + +**One refusal narrows, and this design says so rather than leaving it to be +found.** `ALLOCATION_LIMIT` fired when the *whole* payload passed 64 MiB. Under +a segmented stream that condition no longer arises: a segment closes instead. It +still fires, with the same code and the same expression, on the condition a +segmented stream can still reach — **one instruction row plus the reserved tail +larger than one whole segment**. `test_us_graph_survey_population.py` drives both +that refusal and the new total ceiling, and asserts the payload's bytes are +unchanged at four different segment sizes down to the smallest one that holds a +row. The same narrowing applies to the preparation receipt's `PAYLOAD_LIMIT`: +`_chunks` yields one JSON token at a time and a segment never splits one, so the +refusal now means "one token larger than one segment", and because +`_check_scalars` has already bounded every string to 1 MiB the shipped 64 MiB +segment always holds any token. Both are tested at a segment size below the +longest token. + +### 2e. Size law after the change + +`MAX_ROSTER_BYTES` is set to `64 * MAX_SEGMENT_BYTES` = 4 GiB, which is +3.9× a full-source preparation receipt and 7.6× a full-source allocation +payload (measured 316.665 B per household × 1,587,376 = 502.6 MiB). The +refusals therefore still exist, still fail closed, and no longer bind below +full source: + +| | before | after | +|---|---|---| +| preparation receipt | 96,860 hh (6.10%) | 6,206,000 hh (391% of source) | +| `_origins` rows | 150,916 hh (9.51%) | 9,658,000 hh | +| allocation payload | 211,924 hh (13.35%) | 13,563,000 hh | +| `_plan_document` rows | 270,284 hh (17.03%) | 17,298,000 hh | +| allocation map pre-check | 524,288 hh (33.03%) | 33,554,432 hh | + +A fourth whole-roster receipt the cost attribution did not name is in the same +module: `_current_survey_wage_projection` builds one row per selected wage +earner and encoded it the same way. It takes the same transport. It does not +bind at 1/10 and does bind at full source. + +## 3. Retention and the frame seal + +### 3a. What the observer costs and where + +`run_graph` calls `_population_observer(node_id, _observer_snapshot(updated))` +at `executor.py:2773-2774`, under the single guard +`if _population_observer is not None:`. That is **outside** the cache-load +attempt (`if resume != "forbid":`, `:2575`), outside the cold path +(`if result is None:`, `:2607`) and outside the store write (`if not hit:`, +`:2776`), so every reached node pays it, cache hits included; only the +unreached-node `continue` (`:2510-2524`) skips it. `_observer_snapshot` +(`:356-408`) detaches by round-tripping every entity and link table plus the +strata through `pickle.loads(pickle.dumps(..., protocol=5))` (`:373`) — a full +independent copy, measured by the cost-attribution report at **9.6–9.8 bytes +per cell per snapshot**. + +The financial runner retains all of them: `observed[node_id] = population` +(`graph_atomic_survey_financial.py:1644`), never cleared, read at `:1656` +(node ids), `:1796`, `:1818`, `:1821`, `:1824`, `:1944`, `:1948`, `:1974` +(the objects), `:1897` (object *identity*) and `:1945` (the stamp alone). It +reaches the frozen `_FinancialRunState.node_populations` field (`:127`) through +`_issue_run` and `_node_population_seals` (`:500-515`), and `_pure_run` +(`:676-691`) re-stamps every retained population on **every** `checked_view()`. + +### 3b. Why the snapshot cannot be addressed to the store + +The obvious move — write each snapshot to the content store and keep the key — +is refused by the store's own normalisation, in three independent ways, and this +design records them rather than discovering them later: + +* `store.py:456` zeroes the values beneath a null mask (`values[mask] = 0`), + while `survey_atomic_geography._population_stamp` (`:230`) deliberately folds + the physical storage parts *under* the mask — its docstring says so at + `:231-237`. A store round trip therefore changes every stamp. +* `store.py:619` refuses a `MultiIndex` outright, and + `test_graph_executor.py:4254-4255` builds one into the observed frame. +* `store.py:486-487` refuses `CategoricalDtype` and `DatetimeTZDtype`, and + `test_graph_executor.py:4262` observes a categorical column. + +There is also no store API that accepts a `Population` (only `put_frame` for a +`Frame` and `put_bytes` for bytes), and `DataFrame.attrs` — asserted preserved +at `test_graph_executor.py:4320-4321` — is not persisted by `_write_frame`. +**A store-backed snapshot is not a drop-in for the pickle.** Anything that +spills a snapshot must spill the same pickle bytes the executor already trusts +for detachment. + +### 3c. Object identity is pinned, so re-materialisation is not free + +`result.financial_population is observed[final_node]` +(`graph_atomic_survey_financial.py:1897`) is an identity check, not an +equality check, and the completion host has two more of its own. A policy that +dropped a population and rebuilt it from a spill would produce a different +object and fail these. So the retention policy is **declared-consumer**: the +nodes whose objects a caller holds are named, retained in memory, and the rest +may be spilled. For the base financial run those are the final node, +`financial.ATTACH_NODE`, and — when the tax rebase is enabled — the property +population. + +### 3d. What still needs every population, and the question it raises + +Two loops need the whole `Population` object, not a seal: +`atomic.same_replayed_population(population, observed[node_id])` at `:1796` and +`:1948`. They compare the executor's observation against the runner's own +independent replay, field by field and byte by byte +(`survey_population_replay.py:132`, `:179`). The replay cannot be built until +`run_graph` has returned, because it reads the manifest's loaded artifacts, so +between the observation and the comparison **nineteen detached populations must +exist somewhere** — RAM (~290 GiB at full source) or disk (~285 GiB). There is +no third place, and choosing to replace the object comparison with a content +seal is a change to a verification contract, not an optimisation. §5 puts that +to the owner as a question. + +### 3e. The seal: a vectorised per-column pass, proven equal + +`survey_population_preparation._frame_identity` is the only remaining per-cell +frame walker in the US runtime. Its cost was literally + +```python +for value in series: + digest.update(_frame_cell_encode(value)) + digest.update(b"\n") +``` + +— one `_frame_cell_encode` call and two `digest.update` calls per cell, over a +quantity that is a pure function of the cell values plus dtype metadata. (The +two neighbours that look similar are already vectorised and are left alone: +`asec_current_money_source._series_digest` (`:98`) dumps `_data`/`_mask` bytes +for numeric dtypes and caches string tokens, and +`asec_2024_native_population._frame_identity` (`:184`) delegates to it.) + +The replacement builds, per column, **the exact byte string that loop would +have fed the digest**, and updates the digest once: + +| dtype | how | why it is exactly equal | +|---|---|---| +| `float64` | `_float_cells`: a fixed-offset `(n, 37)` uint8 matrix with a per-position keep mask, cut down with one boolean gather | the record is `["float","` + optional `-` + `0x1.`/`0x0.` + 13 mantissa nibbles + `p` + sign + up to 4 exponent digits + `"]`; the mask deletes the parts a value does not spell, so `float.hex()`'s variable spellings come out of one padded matrix | +| `int8`…`uint64` | `_integer_cells`: `"\n".join(map(str, values.tolist()))` | `str` on a Python int **is** `int.__repr__`, which is the fast path's encoder; every integer-dtype value lies inside `-2**63 <= v < 2**64` and spells ≤ 20 bytes, so neither `_encode` nor `PAYLOAD_LIMIT` is reachable | +| `Int8`…`UInt64` | `_nullable_integer_cells`: the same, with `pd.NA` positions overwritten by `null` | `_cell` maps `pd.NA` to `None` and `_frame_cell_encode` spells `None` as `null`; unsigned widths keep an unsigned carrier so `UInt64` above `2**63 - 1` still spells | +| `bool`, `boolean` | `_boolean_cells`: index into the three spellings | three possible cells | +| `string` | `_string_cells`: `pd.factorize`, then `_frame_cell_encode` once per distinct value | `StringDtype` compares and hashes exactly, so grouping equal values cannot merge two whose encodings differ | +| anything else | the cell-at-a-time walk, unchanged | `object` and `category` deliberately do **not** qualify: `1`, `True` and `1.0` are equal and hash alike there while `_frame_cell_encode` spells them three different ways | + +`_cell` refuses a non-finite cell with `FRAME_NONFINITE`; the float branch +raises the same code for the column rather than at the first offending cell, +and the digest is discarded either way. + +**The equality is proved against an independent oracle that already existed.** +`packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py` +carries `_legacy_frame_preimage`, a verbatim predecessor traversal, and +`_assert_exact_frame_preimage`, which records every `digest.update` call through +a patched `hashlib.sha256` and asserts the **concatenated byte preimage** — not +merely the digest — equals the oracle's. That is a byte-for-byte proof, and it +is joined by an exhaustive float proof and a per-dtype sweep this lane adds. + +## 4. What `run_graph(population_retention=...)` on main would need + +`#901` recorded that the shared executor has no retention mode, and that is +still true at this head: there is no eviction, pruning or population dropping in +`executor.py` or `store.py`. The executor's own `populations` dict is keyed by +**version**, not node, and a `StructuralDelta.NONE` node overwrites its entry +(`:2769`), so the executor itself already retains one population per version; the +per-node retention is entirely the observer's. + +**One such mode was written and dropped, and it is not the one this work needs.** +`run_graph(population_retention="lazy")` existed on PR #893's original branch, +staged as layer 08 GRAPH-ATTACHMENT-METADATA: commit `6c0f24c77` "Add opt-in lazy +population retention" adds `packages/microcosm-graph/src/microcosm/graph/attachments.py` +(210 lines: `_StoredPopulation`, `_LazyPopulations(Mapping[str, PopulationView])`), +81 lines of `executor.py`, 41 of `keys.py`, 31 of `manifest.py`, and +`tests/test_population_retention.py` (528 lines). The #893 reconciliation dropped +it (`experiments/893-reconciliation-amendments-19-20-20260912.md:192`) because +"no call site outside the graph package passes `population_retention`; its only +consumer was the branch's own `test_lazy_snapshot_metadata_integration.py`, +dropped with it", and recorded the way back: "if it is wanted it is a +self-contained later PR on top of piece B, which it imports." + +**Read what it actually did before reaching for it.** `attachments.py` imports +`ContentStore`, `_write_frame` and `_payload_table` and its docstring says +"Private, non-portable population attachments and execution lifetimes. These +store references never enter node records or portable manifest identity." Its +`_LazyPopulations` is a `Mapping[str, PopulationView]` — so it made the +**manifest's attached population views**, one per structural version, lazy +against the content store. That is a different target from the observer's +per-node detached `Population` snapshots, and §3b's three store refusals are why: +a `PopulationView` is a portable Frame that already goes through `put_frame`, and +a detached snapshot is not. **Restoring layer 08 would not remove the nineteen +pickles.** It is still worth restoring on its own terms, and it is still the +closest existing shape for the keyword, but the work §3d describes is a second +thing. + +A `population_retention` keyword would have to carry these, all of them +established above: + +1. **A mode that skips the detachment, not just the retention.** The cost is + `pickle.dumps` + `pickle.loads` per node (`:373`). An observer that only + seals needs no detached copy at all, because sealing is read-only. The + executor would hand it the live population and the mode would be the + observer's declaration that it will not retain or mutate it. + `test_graph_executor.py:4368-4374` already pins the converse — with no + observer, no snapshot may be allocated — so the shape of that test is the + shape of the new one. +2. **A declared-consumer roster.** `_observer_snapshot`'s detachment exists so + the observer may mutate what it keeps (`test_graph_executor.py:4183-4248` + asserts a mutated snapshot moves no node key, no manifest and nothing in the + store). A retention mode that hands out a read-only view instead would make + that mutation raise — *more* closed, but a behaviour change to a documented + contract in `microcosm-graph`, so it belongs to a main-only change with its + own test rather than to a country runner. +3. **Invocation invariants that do not move.** Exactly once per reached node, in + `compiled.order`, on cache hits and cold execution alike + (`test_graph_executor.py:4153`, `:4166-4167`); an observer's own exception + still refuses the run with the observer's exception type (`:4172-4179`); + `ATOMIC_OBSERVER_ROSTER` and `COMPLETION_OBSERVER_ROSTER` still refuse any + gap or reorder. +4. **A precedent in the same package.** `graph_survey_puf55` already declines to + retain everything: its issued run entry "retains the final output, not all 245 + executor-observed snapshots" (`graph_survey_puf55.py:638-639`). The financial + runner's `node_populations=observed` is the outlier among the six observers in + this runtime, and the shape it would move to already exists a few files away. +5. **A statement about `_pure_run`.** `state.node_populations` is re-stamped on + every `checked_view()`, so a retention mode that drops populations must say + what `FINANCIAL_NODE_POPULATION_CHANGED` means afterwards. Today + `_node_population_seals` already returns `()` when there are no observations + and no completion boundary (`:502-503`), which is the existing shape of + "nothing retained, nothing to re-stamp". + +Item 1 is the whole memory wall and most of the avoidable CPU. It is main-only, +it lands in `microcosm-graph`, and the executor hunks of PR #938 are its +neighbours. + +## 5. The ceilings this lane did not touch, and where they sit + +A transport ceiling is not the only kind. The 19-node path carries a second +family — row-count and row-byte bounds — and this section records where each one +sits so the next lane does not rediscover them one build at a time. The counts +come from the recovered 1/1000 artifact's own rosters: 1,584 selected households +carry 1,584 stacked households and 3,464 stacked persons, i.e. 2.187 persons per +household, and the combined clone doubles both. + +| bound | value | binds at | +|---|---|---| +| `survey_population_domains.MAX_HOUSEHOLDS` | 100,000 | **never** — it bounds one `classify_households` batch, and `survey_catalogue_selection` streams batches of at most `_BATCH_HOUSEHOLDS = 10,000` households or `_BATCH_PEOPLE = 100,000` people (`:187`, `:197-201`) | +| `survey_population_domains.MAX_TOTAL_MEMBERS` | 1,000,000 | **never**, for the same reason | +| `survey_population_domains.MAX_MEMBERS` | 20 | per household, unrelated to the fraction | +| `acs_person_coverage_columns.MAX_SELECTED_ROWS` | 1,000,000 | above 1/10; 347,137 stacked persons at 1/10, 3,471,383 at full source | +| `asec_current_money.MAX_PERSONS` | 1,000,000 | above 1/10, same counts | +| `acs_pums.MAX_EXACT_HOUSEHOLDS` / `MAX_EXACT_PERSON_ROWS` | 1,000,000 | above 1/10; 1,587,376 households at full source | +| `survey_observed_age.MAX_ROWS` | 2,000,000 | above 1/10; ~694,000 cloned persons at 1/10, ~6,943,000 at full source | +| `survey_origin_budget.MAX_GROUPS` | 1,000,000 | not established — the group count was not derived here | +| `graph_survey_age_artifact.MAX_PEOPLE` | 10,000,000 | above full source's ~6.94M cloned persons, so not at all | + +**So a 1/10 build meets no ceiling this lane did not lift, and a full-source +build meets at least four more** — all of them row counts rather than transport +shapes, and all of them in the 1,000,000–2,000,000 range that a 1.59M-household +source crosses by construction. They are a separate change with a separate +argument, and this lane does not touch them. + +## 6. Open, for the owner + +1. **Columnar format for the receipt bodies.** This design uses the store's own + existing shape — a `header.json` payload table plus content-addressed bodies, + as `put_frame` already writes `.npy` files — and keeps the bodies as canonical + JSON segments, so nothing derived from the byte stream moves. A true + Arrow/parquet body needs **no new dependency**: `pyarrow>=15` is already + declared by `microcosm-build` (`pyproject.toml:20`, "Parquet is the artifact + format for the us_trade margin and entry tables"), pinned at 25.0.0 in + `uv.lock`, and already named in this very stage's `STAGE_DEPENDENCIES` + (`graph_survey_population.py:77`), so its version already enters the stage's + implementation hash. What a columnar body would cost is the byte stream: it + changes, and therefore `preparation_sha256` and every node key and store + address move, once and deliberately. +2. **Whether the retention policy is a run option or the only behaviour.** §3d + shows that nineteen detached populations must exist between the observation + and the replay comparison unless the object comparison is replaced by a + content seal. That is a verification-contract decision, and §4's item 4 shows + the runtime has already made it once, the other way, in `graph_survey_puf55`. +3. **Whether the roster spill should be readable by anything but this module.** + It is written, verified, and then only reassembled in the same call today. A + consumer that wanted to stream it — a future `put_bytes` that takes an + iterator, or an auditor re-deriving `preparation_sha256` without loading + 1 GiB — would read `header.json` and the segments it names, which is why the + header exists at all. diff --git a/docs/us-native-soi-agi-targets.md b/docs/us-native-soi-agi-targets.md new file mode 100644 index 000000000..17c0b6954 --- /dev/null +++ b/docs/us-native-soi-agi-targets.md @@ -0,0 +1,77 @@ +# Native integration of SOI Table 1.1 targets + +This compiler-only port adopts the target representation from +[Microcosm #960](https://github.com/PolicyEngine/microcosm/pull/960), reviewed at +`b563d11be314bdd608f736c553f435b9236820df` and retrieved September 19, 2026. +[Issue #958](https://github.com/PolicyEngine/microcosm/issues/958) distinguishes +these target changes from the separate full-vector tail support work. + +The integration base is the clean native continuation +`59daeda3bf4a745eca2119e59520c6f691afe252`, descended from the active native API +source `5258a4c7f6d05cb1e68806d51b38996ea116bfb5`. At review, the compiler, its +existing tests and fact-to-target guide on that base were byte-identical to +fetched main `16c8e78d2f60d629da5d70294f643bce5c4597e2`. A sibling worktree carries +this port; the measurement and owner-PR worktrees are unchanged. Only compiler +source, compiler tests and documentation are adopted. The PR's population, +prototype reweighting and reform-score scripts are excluded. + +## Accepted facts and year alignment + +The rescue applies to stale `irs_soi.table_1_1` facts with all filing statuses, +national geography, measures `return_count` or `adjusted_gross_income`, and a +lower AGI bound of at least $100,000. Other tables, state/status slices and lower +cross-period bands remain excluded. The +[IRS table index](https://www.irs.gov/statistics/soi-tax-stats-individual-statistical-tables-by-size-of-adjusted-gross-income) +identifies Table 1.1 as all returns by size and accumulated size of AGI, and +Table 1.2 as a separate marital-status table. The $100,000 edge is the model's +existing tax-unit-count approximation threshold, not a statutory filing limit. + +Each stale class is represented as its share of the same-vintage Table 1.1 +national total, multiplied by the latest eligible national total of that +measure. Controls cannot come after the build year or before the class year. +An absent or zero same-vintage denominator leaves the class unbound. The +existing dynamic target selection chooses the latest eligible class vintage. + +For a 2022 class aligned to a 2023 control and aged to a 2024 build: + +| Field | Amount target after aging | +|---|---| +| `name` / source-record identity | Original 2022 source record | +| `ledger_fact_period` | `2022` | +| `uprating_from_period` | `2022` | +| `uprating_to_period` | `2023` | +| `source_period` | `2023`, the existing ager's alignment origin | +| `target_period` / `aged_to` | `2024` | + +The amount receives the realized 2022-to-2023 control ratio once, then the +remaining 2023-to-2024 CBO AGI ratio once. Counts receive the realized national +return-count ratio; the amount ager does not apply a CBO dollar factor to counts. +The nominal AGI edges remain the declared source edges. Same-period facts bypass +this rescue and preserve their existing behavior, including the existing +below-$100,000 behavior. No target IDs, API arguments or default compiler options +are changed. + +## Integration boundary + +The existing `compile_us_fiscal_target_registry` entry point consumes this code +when the separately admitted native release source includes the commit. No +Chronicle feed, the shared 32,867-target record, parity manifest, calibration +registry snapshot or frozen comparison surface is regenerated here. A future +build must record its actual newly compiled surface and requalify the relevant +frozen identities. The PR's separately reported target counts are not counts +verified on the shared native Chronicle artifact. + +These targets do not create high-income support or repair its income +composition. They cannot establish support adequacy, calibration feasibility or +scientific improvement. The separate full-vector tail and ownership work remains +a launch prerequisite under #958. No dataset, calibration, model or reform run +was performed for this port. The held-out top-rate benchmark remains held out; +this change adds no target family for it. + +Validation uses invented facts to exercise compiler acceptance, exclusions, +future/control-year selection, missing denominators, original identity and the +complete two-stage amount-aging chain. The official TY2023 workbook was acquired +and hashed during review, but no installed XLS reader was available, so the +upstream test's published numeric fixture was not independently cell-verified. +That source-review limitation does not change the values compiled from supplied +facts; production code introduces no hardcoded target amounts. diff --git a/docs/us-native-spm-amount-source.md b/docs/us-native-spm-amount-source.md new file mode 100644 index 000000000..0fce043ba --- /dev/null +++ b/docs/us-native-spm-amount-source.md @@ -0,0 +1,38 @@ +# Native SPM amount source + +`qualify_current_spm_amounts(preparation, spm_inputs)` projects current ASEC +`SPM_ENGVAL` and `SPM_CHILDCAREXPNS` through the existing authenticated money +owner and qualified original SPM membership. It returns descriptive person and +unit tables with evidence. It issues no source capability and does not attach +canonical consumer inputs. Callers must retain both genuine owners and +requalify after relevant I/O. + +The projection joins exact current-year native keys and IDs, checks original +household/person coordinates, and preserves complete literal SPM membership. +Amounts repeated on person records represent one unit amount; they are never +summed. Replicated amount bits, validity, status and zero origin must agree. +An absent money member leaves the whole unit unknown. Historical restated +amounts cannot fill current-year observations. ACS units remain unobserved. + +Missing money uses the maintained canonical positive-zero storage encoding, +with missing status and invalidity retained; that storage value is not an +observed zero. Frozen-storage zeros also remain unknown unless their original +evidence authenticates a Census-encoded zero. Unresolved annual membership +does not qualify a nonzero amount merely because it is present. + +The existing current-money domain and resource pins remain authoritative for +admission. In particular, this adapter does not resolve the conflicting energy +range in the ASEC dictionary or bypass the original owner's refusal. Childcare +is labeled reported uncapped expenditure. It is not claimed to be an observed +amount before subsidy. + +The pure `project_spm_amounts` function accepts supplied descriptive evidence +for inspection and testing. Its output and `spm_amount_values_seal` hash confer +no authority. The genuine borrower checks both retained owners, exact active +implementation dependencies and source bytes before and after projection. + +This layer performs no fitting, clone transport, graph attachment, calibration +or consumer export. Choosing unit rather than person donor weighting, completing +ACS amounts, and resolving consumer naming remain separate modeling work. Passing +invented fixtures establishes these contracts, not actual-data quality or release +readiness. diff --git a/docs/us-native-spm-input-bridge.md b/docs/us-native-spm-input-bridge.md new file mode 100644 index 000000000..287938570 --- /dev/null +++ b/docs/us-native-spm-input-bridge.md @@ -0,0 +1,98 @@ +# Native survey SPM source inputs + +The native SPM bridge qualifies two source inputs for income year 2024: +`is_spm_independent_minor_role` on people and +`spm_unit_spm_universe_status` on SPM units. It does not compute thresholds, +resources or poverty, certify a dataset, calibrate weights, or authorize a +release. A detached Frame or copied receipt does not carry source authority. + +## Source meaning + +ACS requires explicit native SPM construction and the named +`acs_spm_household_only_analysis_2024` profile. The profile retains separate +choices for admitting modeled and approved-inference membership. Its per-unit +classification uses the reviewed construction evidence, without regrouping. +Ambiguous secondary relationships make the household unresolved. Missing roles +remain missing. Group-quarters records are outside this **restricted ACS +household-only analysis**, not outside every possible SPM analysis. + +ASEC requires the versioned +`ASEC_2025_INCOME_2024_SPM_POLICY`. Census specifies an unrestricted SPM +person universe in the [2025 ASEC technical documentation, Appendix D, +D-3](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf). +That instruction supports inclusion only after authenticating the exact source +people and complete original SPM unit. It does not justify including arbitrary +rows labeled ASEC. Role and count reconciliation are additional integrity +checks; they do not supply the universe policy. + +The ASEC observations were collected in 2025 and refer to 2024 income. The +bridge preserves survey-time age and relationships. It does not backdate age, +infer residence history, reuse `POV_UNIV` as SPM scope, or transfer the ACS +household-only exclusions to ASEC. Omitting the ASEC policy leaves its annual +scope unresolved. A missing required role remains unknown and prevents engine +projection for that unit. + +## Retained source and final membership + +The source qualifier borrows the live preparation retained by the issued +enrichment run. It validates source ownership, implementation and source bytes, +and retains nullable roles, annual scope and reasons. Its validator must run +before and after consuming the borrowed values. + +The pure projection helper does not grant authority. It verifies the original +person axis, survey/source years and the receiving two-clone mapping. Every +receiving SPM unit and household must correspond to exactly one original group +and clone, with its complete person roster. Moving, mixing, splitting, +truncating or duplicating groups refuses. Existing output columns cannot be +overwritten. Projection returns new columns without changing the populations, +weights, strata or mass history. + +The source nullable role is preserved separately. Only an explicitly `OUTSIDE` +person may receive a chosen Boolean storage placeholder at the final engine +boundary. Null `INCLUDED` or `UNRESOLVED` roles refuse. Known roles with +`UNRESOLVED` scope retain that status for the country's `SPM_UNIVERSE_REQUIRED` +gate. The annual status is never carried into a different model year. + +The adapter honors the consuming country's explicit `spm.DATASET_SOURCE_INPUTS` +declaration, including both source inputs. Its runtime and import-free metadata +paths reject malformed, overlapping or unsupported declarations. Formula-owned +thresholds and resources remain rejected dataset columns, and the two source +inputs receive no invented defaults. + +## Enrichment graph + +`run_us_survey_enrichment` accepts three optional SPM arguments: +`spm_acs_profile`, `spm_asec_scope_policy` and +`spm_outside_role_placeholder`. Omitting all three keeps SPM disabled. Enabling +it requires a typed `ACSAnalysisProfile` for 2024 and an explicit Boolean +OUTSIDE placeholder. The ASEC policy may remain `None`, preserving unresolved +scope. Incomplete or untyped configuration refuses before enrichment fitting. + +The host qualifies its real retained preparation and preflights the complete +receiving membership. Three ordinary nodes then follow the hours attachment: +`survey_spm.source` retains private source evidence, `survey_spm.project` +transports it at the person and SPM-unit grains, and `survey_spm.attach` adds +only the two declared source inputs to the existing population version. +Source identifiers, nullable observations and mappings remain in private +artifacts. Public parameters and the final receipt record the chosen profile, +policy, representation and artifact digests. + +The host revalidates the retained source on cold and required-cache paths and +independently replays all three nodes against the complete population. Final +pure source and output checks follow the last source and artifact I/O. SPM +attachment does not make a run release eligible. Public qualification can still +inspect the issued enriched owner; projecting onto it again refuses because +the two input columns already exist. + +## Verification boundary + +Focused tests use invented source records and detached pure transport fixtures; +they do not constitute a population measurement. Genuine source-owner tests are +separate from pure classifier and projection tests. Optional-assembler skips or +expected failures are not evidence that native construction passed. + +The graph hooks and pure fragment checks do not replace a combined final-owner +cold/cache proof. That execution is a separate integration gate. Do not +interpret these helpers or the earlier development-cohort country result as +qualification of the full native population, a production wrapper, calibrated +estimates or an app deployment. diff --git a/docs/us-native-ss-report-completion.md b/docs/us-native-ss-report-completion.md new file mode 100644 index 000000000..f3b173971 --- /dev/null +++ b/docs/us-native-ss-report-completion.md @@ -0,0 +1,45 @@ +# Native Social Security report completion + +The optional `current_survey_ss_completion` qualifier and +`graph_current_survey_ss_completion` fragment complete **source reports** for +selected original ASEC and ACS people. They do not populate canonical individual +beneficiary inputs, join an enrichment host by default, or qualify a release. + +The qualifier borrows a genuine `AuthenticatedSurveyPopulationPreparation`. +Its donor population retains every original current ASEC person and the original +DESIGN weights, independently of receiving support selection. Eligible category +labels are positive reports whose source reason basis resolves completely to one +of the four maintained components. Known zero reports and below-15 NIU reports +are not labels. Publisher allocation codes remain in source evidence; no +allocation category is silently excluded. + +The approved development predictors are source age, current wage income, current +self-employment income, and the source Social Security report total. Required +unknown predictors refuse. The ASEC 2024 annual and ACS 2024 rolling-window +transport assumption remains scientifically unqualified. No age-only retirement +rule, assigned geography, prior wages, clipping or recipient fitting is used. + +When unresolved positive reports exist, seven graph nodes publish the full source, +model columns, resolved donor support, original recipient matrix, actual fitted +categorical artifact, raw probabilities, and a report-completion artifact. +`CategoricalConfig` and seed are explicit. The probability and report nodes own +no cells and join the donor version: the compiler makes a FILTER depend on every +member of its base version, so placing a fit consumer on the full-source version +would cycle through the donor FILTER. Every declared class needs positive +DESIGN-weighted support. Maintained categorical fit/probability kernels supply the +model, typed dependencies and cache behavior. Reason-allowed probabilities are +renormalized as conditional mean report shares by `complete_positive_basis`; +completed report components must conserve each known source total exactly. + +When no unresolved positive reports exist, only source and report nodes execute. +That path needs neither class support nor a fitted model. Both paths preserve +complete, zero and unknown source component bits. The versioned report artifact +keeps source arrays, raw probabilities and completed amounts separate and records +that reports may combine family payments. Its decoder reconstructs the completion +and verifies exact source and model bindings. + +The private boundary retains the genuine preparation. An integrating caller must +validate before graph/store work and after the final I/O, including required +replay, then perform the final pure seal. Detached Frames, projections, models, +receipts and cached artifacts cannot recreate that source authority. A production +host and canonical beneficiary interpretation remain separate future work. diff --git a/docs/us-native-survey-demographic-inputs.md b/docs/us-native-survey-demographic-inputs.md new file mode 100644 index 000000000..a2f26479a --- /dev/null +++ b/docs/us-native-survey-demographic-inputs.md @@ -0,0 +1,75 @@ +# Native survey demographic inputs + +The enrichment host offers two independent options, both disabled by default: +`demographic_inputs=True` binds `is_female`; `race_hispanic_inputs=True` binds +`cps_race` and `is_hispanic`. Each fragment borrows the original survey +preparation, binds values once per original person, and copies them through the +existing clone mapping. They do not add people, change weights, redraw values, +or assign geography. + +The race/Hispanic fragment captures source literals from the same pinned ASEC +person member and ACS person archive authenticated by the retained preparation. +It checks the complete captured members, source identities, original key roster +and final parent identity. Its projection is descriptive evidence, not a new +source credential. The host retains it and requalifies it after relevant I/O. + +## Source definitions + +`cps_race` represents the published CPS `PRDTRACE` categories. ASEC supplies +those codes directly. Hispanic origin uses the direct `PEHSPNON` response with +the `PRDTHSP` detail-universe crosscheck. The fragment preserves `PXRACE1` and +`PXHSPNON` allocation histories. These observations come from survey 2025; +their associated income cohort is 2024. See the [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), PDF pages 27–28 and 31–32. + +ACS 2024 Hispanic origin maps directly from `HISP`. Race uses `RAC1P` together +with `RACWHT`, `RACBLK`, `RACAIAN`, `RACASN`, `RACNH`, `RACPI`, `RACSOR`, and +`RACNUM`. The fragment checks agreement between the broad category, detailed +indicators and number of major groups. Native Hawaiian and other Pacific +Islander indicators form one major group. All combinations of the five groups +represented in CPS map to its published categories, including its residual +three-race and four/five-race codes. `FRACP` and `FHISP` preserve allocation +status. See the [2024 ACS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf), PDF pages 63, 106, 111, 127 and 129. + +“Some Other Race,” alone or in combination, has no exact CPS counterpart here. +It remains unresolved. A missing, malformed, unlabelled, contradictory or +allocation-unresolved source item also remains unknown. Allocation histories +that end without a published value do not establish a known response. Published +allocated values are retained with their allocation status; they are not +described as unallocated responses. No unknown is replaced by false or zero. + +## Graph inspection + +The visible `survey_race_hispanic` fragment has source, raw-column, canonical +binding and clone-attachment nodes. Source literals, observation year, +allocation and mapping-status columns accompany the canonical nullable outputs. +The graph's private source artifact retains source evidence without exposing +raw native person keys in public node parameters. A missingness mask belongs to +the canonical column; mapping status explains why a value is unresolved. + +The older `acs_release_predictors` mapping has a different contract: it uses +representative CPS codes for particular coarse donor-model categories. That +equivalence is not an exact source observation and is not reused here. + +The maintained SCF wealth model also uses an internal feature named `cps_race` +with its own coding. Its recipient helper maps known Hispanic origin to 3 and +non-Hispanic White/Black/Asian/Other to 1/2/4/7. Those are model-feature categories, +not canonical PRDTRACE. A future SCF graph must call or reproduce that explicitly +declared consumer transformation from canonical race plus Hispanic origin; it +must not feed the canonical 1–26 codes directly into that feature. This fragment +does not implement or change SCF consumption. + +## Possible future conditional harmonization + +An additional explicitly modelled operation could draw a CPS-compatible category +for an unresolved ACS person, conditional on reviewed common predictors and +the observed ACS category. Such an operation would need a scientific mapping +convention, authenticated donor population, model/seed declaration, consistency +constraints, validation and distinct imputation provenance. Original ACS +literals and unresolved-source status would remain available. Model output +must not overwrite the historical source claim or be relabelled an exact +crosswalk. This fragment implements no such convention or imputation. + +Tiny invented-source tests exercise capture, all supported category combinations, +unknowns, exact clone transport and cache replay. They do not certify national +data completeness, source comparability, a full enrichment host or release +eligibility. Native ACS required/default loader APIs remain unchanged. diff --git a/docs/us-native-survey-development-handoff.md b/docs/us-native-survey-development-handoff.md new file mode 100644 index 000000000..3d53b4885 --- /dev/null +++ b/docs/us-native-survey-development-handoff.md @@ -0,0 +1,204 @@ +# Native survey development handoff + +`tools/build_us_fiscal_refresh_release.py` now exposes +`prepare_native_survey_development_input(run, checkpoint_directory)`. Its input +is a **retained, issued `SurveyEnrichmentRun`** in the same process that ran +`run_us_survey_enrichment`. The output is a `NativeSurveyDevelopmentInput` +containing a usable `Frame` and a machine-readable coverage report. + +```python +from tools.build_us_fiscal_refresh_release import prepare_native_survey_development_input +from microcosm.build.us_runtime.native_survey_handoff import ( + load_native_survey_development_checkpoint, +) + +# enriched_run remains alive from the native PUF/enrichment continuation. +result = prepare_native_survey_development_input( + enriched_run, "/owned/new/native-development-checkpoint" +) +frame = result.frame +missing_inputs = result.report["missing_inputs"] +remaining_gates = result.report["required_release_evidence"] + +# Validated reuse while the original issuer remains alive: +reloaded = load_native_survey_development_checkpoint( + "/owned/new/native-development-checkpoint", run=enriched_run +) +assert reloaded.owner_live_verified +``` + +The writer creates a fresh directory containing `population.h5` (the maintained +Frame checkpoint format) and `handoff.json`. It performs complete readback +comparison: entity tables and row order, schema/linkage, dtypes and knownness, +non-owned fields, geography, strata, typed weights and mass history. Frame +metadata uses the graph store's typed metadata encoding. Nullable values follow +the maintained replay comparator's documented null backing-buffer normalization; +known values and masks retain their meaning. No additional support clones, +weights, default inputs, fitted values or legacy assembly receipts are created. +The final owner check must succeed before the API returns. Existing output +directories are refused. + +The loader also works without `run` for downstream development inspection. In +that case `owner_live_verified` is false. Byte identities bind the two files; +they do not issue source authority. The Frame checkpoint is not a +`USSingleYearDataset` H5 or simulation-ready multispine pool. Do not send it to +`--base-h5` or `--pool-manifest`. Its Frame can be used by existing Frame-based +consumers after those consumers' own input checks, while the source run is kept +alive whenever source qualification is required. + +The separate `prepare_native_survey_engine_input(run, declaration=..., consumer=...)` +entry selects an exact, closed projection from a live owner. Its optional +`PolicyEngineUSEngine` consumer calls `validate_input_representation(frame, +period=...)` against that adapter's selected live registry and country source +declarations. This checks owning entities, selected-year formula ownership, +annual/eternity periods, actual enum names, knownness and physical dtype +compatibility, including lossless integer IDs and memberships. It copies the +entity tables and materializes the typed household weight for inspection; it +does not apply defaults, rewrite inputs, construct a dataset or calculate. + +Success adds only `consumer_representation_compatible: true` to the projection +report. Runtime admission, source applicability, SPM universe qualification, +scientific gates, simulation readiness and release eligibility remain separate. +Ordinary float32 rounding and representable infinities are not scientific +certification; unsupported nulls, integer overflow, finite-to-infinite conversion +and string truncation refuse without filling or remapping. `UNRESOLVED` can be +a valid enum representation while still preventing SPM measurement. Omitting +the consumer preserves the previous import-free, unqualified projection path. +Both paths retain the final live-owner and exact-storage comparisons. + +The builder's private `_calibrate_native_input_frame` bridge can then solve +already-materialized targets with the maintained dense or L0/refit calibrator. +It requires an explicit formula-ownership provider and a closed solve-only +option set; warm starts, checkpoints and exact-k identities are not accepted. +Numerical option validation remains with the maintained calibrator; this bridge +does not define another solver parameter policy. +Before solving, the target frame must retain every projected input column, +structural ID, membership, row order, stratum and typed initial weight. Target +materialization may add calculated columns; those columns never enter the +exported input frame. + +The bridge works on detached copies, attaches weights using the maintained +helpers and verifies the result against its supplied parent. It reconstructs a +complete float64 weight vector in original household order. L0 keeps exactly +the maintained selected support, including zero post-refit weights, without a +second pruning threshold. The returned `comparison_specification` records the +exact solve options, registry and loss-weight identities, both frame identities, +and the supplied specification digest. The existing comparison `binding` hashes +those specification bytes, so the pair must be retained together. Target +materialization may have different metadata and mass history; the exported +input frame preserves its input parent's metadata and mass-history prefix. +Callback changes to caller configuration cannot +rewrite those captured inputs; changes to either supplied frame refuse. +The returned measured result and comparison binding do not issue native owner +authority, establish target/scientific qualification or authorize publication. + +The report preserves the owner's receipt and source content identities, the +native population version and graph manifest identity, and the amount model's +source/draw flags. Its default inventory merges the maintained release-source +column roster and historical input profile, retaining each named missing or +unknown-containing input and its declarations. `input_inventory_scope` records +these two checked rosters and the historical profile's source-manifest identity. +The legacy pool input surface is **not checked** and the live take-up ABI +inventory remains **unqualified**. The handoff does not import the legacy pool, +whose eager agreement/take-up registries instantiate the country engine. This +descriptive inventory does not establish a complete engine input contract; +historical take-up names are names only, and missing fields without a declared +grain retain `entity=null`. Presence is not proof of +applicability, source validity or non-default statistical signal. An optional +`export_contract=` accepts the consuming adapter's actual `ExportContract` and +reports missing, forbidden, formula-owned and unexpected columns without +projecting or default-filling the Frame. No country engine is imported or run by +this default handoff. The explicit engine-consumer projection described above +retains its separate runtime qualification. Release gates, source/applicability +checks and `simulation_ready=false` / `release_eligible=false` are unchanged. + +The approved native scope excludes the legacy prior-year income family: +`employment_income_last_year`, `self_employment_income_last_year` and +`previous_year_income_available`. The handoff does not generate, fit or require +these as native completion targets. Existing columns, if any, remain preserved +for inspection. The [input-coverage diagnostic](us-input-coverage-diagnostic.md) +now offers an explicit `USInputProfile.NATIVE_NATIONAL_CD` name profile with +159 inputs: it excludes the two prior-year fields still in the 161-name +national/CD roster; prior-year wages were already absent. This resolves only +the name-profile selection. Native scientific/source gates and historical +release requirements still need separate reconciliation; the handoff continues +to name that outstanding gate. + +## Follow-on release integration + +This closes the live-owner to exact development-Frame/checkpoint seam. Release +admission is still unsupported. Its concrete successors are: + +1. Attach and qualify each missing input named by this candidate's report. The + native hours source/transport work, SPM construction/role path, and remaining + source families must become part of the receiving owner's checked lineage. +2. Bind applicability and source-signal gates to that owner, including the + usual-hours gate, and reconcile the prior-year-income scope. The current + report's `release_eligible=false` is descriptive, not an irrevocable decision + about a future qualified owner. +3. Supply the selected engine's input projection and country runtime evidence. + The full checkpoint deliberately preserves fields that may be forbidden or + formula-owned in an engine export; drop them only in the owned projection. +4. Pass the resulting checked Frame directly into maintained target + materialization and common-population calibration. Replace the native + admission branch with checks for that actual evidence; do not route through + legacy assembly/clone stages or mint their receipts. +5. Run the maintained Chronicle/model, national/CD target fit, export + readback/input-mass/target parity, reform smoke, and matched incumbent/default + improvement gates. The builder's existing exact-k arm additionally expects + its agreement and ladder provenance, which this handoff does not fabricate. + +Serialization proves none of those scientific or publication gates. This entry +writes no release manifest, certification or publication pointer. + +## Development base and verification + +This continuation starts from `5258a4c7f6d05cb1e68806d51b38996ea116bfb5` after +fetching and inspecting `origin/main` on September 19, 2026. Main lacked the +native `SurveyEnrichmentRun` API and associated owners; the branches diverged by +41 upstream commits and 407 continuation commits. Work occurs in a separate +worktree. The active native measurement source and harness are untouched. + +Lightweight invented tests cover full storage readback, missing-input and +engine-contract inventories, unissued-owner refusal, changed artifacts and +output reuse. The genuine-owner integration test reuses the existing enrichment +suite's actual issued owner; run it with the compatible continuation source, +not against a frozen active measurement tree. No native data or country model +calculation is needed by the lightweight suite. + +### Optional veterans’ benefits development route + +The retained native enrichment host accepts `groups=("veterans_benefits",)` with +`full_original_amount_donors=True`. Existing default groups and donor selection +remain unchanged. This route qualifies `VET_VAL` and `VET_YN` from the exact +retained 2025 ASEC person member for 2024 income. Positive “yes” answers and +explicit “no” plus zero are known; NIU, recipient zero, missing answers and +contradictions remain unknown. Age 15 is the lower reporting boundary, without +an upper-age, veteran-status or VA-healthcare filter. + +`I_VETYN` and `I_VETVAL` retain their literals, published codes, conditional flag +universes and allocation status. A zero flag outside its universe does not +establish an unallocated observation. Allocation provenance does not create +income zeros or discard otherwise known publisher-allocated amounts. The +six-digit veterans amount range does not change the five-digit UC/WC ranges. + +The model uses full-original current ASEC DESIGN donors with qualified amounts +and predictors, including donors absent from the selected receiving support. +It draws once per eligible original ACS person and carries that draw to both +existing clones. ASEC originals and their clones retain the qualified source +amount and unknownness. The ASEC2025/income2024-to-ACS2024 transfer is an explicit +development assumption; held-out quality, donor adequacy and release equivalence +remain unverified. + +When veterans is selected, `survey_amounts.canonical.version` keeps all receiving +support after prior enrichment, and `survey_amounts.canonical.attach` writes only +selected veterans and child-support canonical outputs. Child-only runs retain +their existing node IDs. An enabled canonical-state stage follows this version. +Carried canonical values establish no source authority: supported float32/float64 +storage is replaced from qualified source/model values with exact-loss checks; +unsupported storage and unrelated ownership collisions refuse. The parent source +branch, other columns, entity axes, memberships, weights and ledger are preserved. + +Invented-source and tiny real-fit/replay tests qualify this component boundary. +They do not establish full financial/PUF/enrichment-owner acceptance, a completed +native input profile, actual-source quality or publication eligibility. diff --git a/docs/us-native-verification-once.md b/docs/us-native-verification-once.md new file mode 100644 index 000000000..abd74366b --- /dev/null +++ b/docs/us-native-verification-once.md @@ -0,0 +1,373 @@ +# Verifying native sources once per run + +A native US build spends most of its time re-proving things it already proved. +The 2026-09-15 pilot measurement on the nineteen-node financial graph at +`Fraction(1, 1000)` (3,168 expanded households, 5,362 s wall) attributes +**1,143 s to node execution and the remaining ~79 % to admission and +verification of the 3.48 GiB of staged source** — repeated. QRF fits inside +that run cost 0.06–0.07 s each. The graph should compile to roughly the +pre-graph runtime plus receipts; today it compiles to the pre-graph runtime +plus the source tree, once per accessor use and once per executed node. + +This note records the five mechanisms that cause it, what each one guarantees +today, what changes, why the guarantee survives, and the new cost class. + +## The rule the whole change obeys + +> Memoisation changes **when** a check runs. It never changes **what** the +> check computes. + +Every digest, key, seal and receipt value in this repository keeps its exact +value across this change. Nothing in `_validate`, `_validate_state` or +`source_content_key` produces a receipt field — all three only compare live +values against values frozen at issuance — so moving a comparison in time +cannot move a recorded value. The one place bytes are genuinely rebuilt +(`_update_series`, mechanism 5) is proven byte-for-byte against a verbatim copy +of the pre-change implementation. + +Two guarantees are stated up front because every section below leans on them: + +1. **Verified before first use.** Nothing is memoised until it has been + verified in full, exactly as today. +2. **Proven unchanged at the end.** Every memo is re-verified in full, + unconditionally, before the run hands anything back. A source or a live + object that changed while a memo was warm still makes the run refuse. + +## The verification epoch + +Memoisation is **opt-in and scoped**. `microcosm.build.us_runtime.verification_epoch` +provides a context manager. Outside an epoch there is no memo at all and every +capsule behaves byte-for-byte as it does today — which is why every existing +capsule test, including the two that trace `_producer` and `_file_identity` +firing inside a validation, keeps passing unchanged. + +Inside an epoch each capsule validation is split into two tiers: + +* **Tier A — every access, never memoised.** The issuance binding proof, the + live final authority and the attached owner payloads, the producer encoding + (`_encode(_producer())` / `_encode(_implementation())`), and a *signature*: + the stat identity `(st_dev, st_ino, st_size, st_mtime_ns, st_ctime_ns)` of + every file the memoised tier reads, plus the structural witness of every live + object the memoised tier digests (per table: the column axis, dtypes, row + count, and for each column and index the buffer address, shape, strides and + writeable flag). The signature is *input*, never a verdict: no Tier A check + refuses because a stat identity or a witness moved. +* **Tier B — memoised on Tier A's signature.** Everything else: the source + catalogues, ACS native coverage, the nested ASEC native validation, the + whole-roster `_source_files` re-hash, the roster stat comparison itself, and + the pure final seals other than the two Tier A repeats — the nested owner + seals, the plan document digest and the three frame identities. + +A signature mismatch is **not** a refusal. It is a memo miss: the complete +unmemoised validation runs, and whatever refusal it would have raised today it +raises now, with the same code, at the same access. This is why the roster's +stat identities are read into the signature rather than compared in Tier A: a +cheap comparison that refused first would answer an appended ACS archive with +`SOURCE_STAT_CHANGED` where an unmemoised borrow answers it with the ACS +catalogue's own refusal. +`test_an_in_epoch_refusal_carries_the_code_it_carries_today` pins that by +refusing the same mutation twice, once inside an epoch and once with no memo at +all: an appended `selection-request.json` raises `SOURCE_CHANGED`, an appended +`acs/csv_pus.zip` and an appended `asec/pppub25.csv` raise +`PREPARATION_VERIFICATION_REFUSED` (a foreign owner's own refusal, translated +by `_checked`), and a touched roster file raises `SOURCE_STAT_CHANGED`. + +On leaving an epoch — and on leaving every nested epoch, so a run that ends +early through an inner scope is still covered — every capsule the epoch +validated is re-validated **in full, with the memo bypassed**, and the result +is recorded. A close records a new signature only when nothing moved while it +was validating: `_validate` compares the roster stats and then runs a whole +trailing `_pure_final`, and at an **inner** nested close the memo it records +survives into the outer epoch, so a signature taken after validating would +absorb a file that moved in that window and answer every outer borrow up to the +outermost close from the memo. Each close therefore takes the signature before +validating and again after, and when the two differ it records no memo answer +at all — the next borrow is a miss that pays the complete validation and +refuses with the code that validation raises. The entry itself is kept, because +the outermost close re-validates every capsule the memo still holds. +`test_a_roster_stat_moved_inside_an_inner_close_refuses_at_the_next_borrow` and +`test_a_native_source_moved_inside_an_inner_close_refuses_at_the_next_borrow` +pin both capsules by moving a source from a profile hook as that close's own +validation returns; against the previous ordering both fail with DID NOT RAISE. + +Leaving the refusal to the next borrow is the whole answer only while a borrow +can still follow. At the **outermost** close there is none: the memo is cleared +as that close returns, so a source that moved inside its window — including one +removed, after which no signature can be taken at all — would end the run +without refusing. The outermost close therefore pays that deferred validation +itself, in the same place and with the same error translation as the +unconditional pass above, whenever its own signature moved or could not be +taken. The refusal is the one that validation raises, never a code the close +invents, and a close that leaves a clean signature — which is what a close +leaves when nothing moved — runs nothing extra. +`test_a_roster_file_removed_inside_the_outermost_close_refuses_at_the_close` +and `test_a_native_source_removed_inside_the_outermost_close_refuses_at_the_close` +pin both capsules by removing a source from the same kind of hook, and assert +the close raises exactly what an unmemoised borrow raises for that removal: +`SOURCE_ROSTER` and `NATIVE_BINDING_REFUSAL`. + +`verification_epoch()` yields the close's record: the protocol label, the +capsule count, the number of memo hits, the number of signature misses and the +number of unconditional final re-validations. It is filled in as the epoch runs +and completed as the epoch closes; `epoch_record()` returns the innermost open +epoch's record, or `None` outside an epoch. + +Both native runners bind the record they open and hand it to `run_graph`, which +attaches it to the manifest as `RunManifest.verification_epoch` — outside the +manifest key, outside its JSON, and outside every node receipt and cache +record, exactly like `source_identities`. The epoch closes before the runner +returns, so the record a caller reads there is the closed one: +`test_a_real_run_records_its_epoch_in_the_manifest` asserts that an actual +nine-node run over invented sources reports memo hits, at least one signature +miss, and one unconditional final re-validation for every capsule it memoised. +`test_nineteen_node_financial_cold_and_required_replay` asserts the other +runner, on both manifests it returns — its own epoch's record on the outer +manifest and the nested population epoch's on the prefix's — for the cold run +and the required replay alike: the epoch's protocol label, one final +re-validation for every capsule memoised, and nothing of it in `to_json`. + +## The five mechanisms + +### 1. `AuthenticatedSurveyPopulationPreparation._checked()` + +`survey_population_preparation.py:1138-1162`. Every accessor — `.frame`, +`.context`, `.selection_plan`, `.receipt`, `.to_bytes()`, `.checked_view()`, +`validate()`, and the module-level `verify_survey_population_preparation` that +the graph registers as a per-node population observer — calls `_validate(state)`. + +**Guarantee today.** At the moment of every borrow: the capsule is the one that +was issued and its payload is unchanged; both source catalogues, the ACS native +coverage binding and the nested ASEC native population re-verify their own +source bytes; the ten-file source roster re-hashes to `state.files`; the +producer modules re-encode to `state.producer`; the roster's stat identities +re-derive to `state.file_stats`; and the pure seals (live authority, attached +evidence, nested seals, plan document, prepared frame identity, both source +frame seals) match. + +**Cost today.** One `_checked()` re-hashes the ACS archive pair fourteen times +(four in the ACS catalogue's source checks, eight across the native binding's +four snapshot dictionaries, two in `_source_files`) — about 11.1 GiB of +SHA-256 — plus three passes over the seven ASEC files, plus the whole roster. +It runs once per executed node through the population observer, plus once or +more inside every kernel that borrows the preparation. + +**Change.** `_validate` becomes Tier A + Tier B as above. Tier A keeps the live +authority, the attached owner payloads and `_encode(_producer())` as refusals on +every access, and reads `_file_stats(state.root)` into the signature on every +access, adding the paths that `_file_stats` does not cover but the I/O arm +reads: the ACS catalogue's `source_dir` entries, its two private snapshot copies +and its projection path, the native binding's four snapshot dictionaries, and +the `source_files` rosters of both nested ASEC capsules. Tier B is the rest, +including the comparison of those roster stat identities against the ones frozen +at issuance. + +**Why the guarantee survives.** Producer tampering and live-callable rebinding +still refuse at the same access (`_producer()` calls `_live()`, and it is never +memoised). Every on-disk change to any file the I/O arm reads — truncation, +append, in-place rewrite, replacement, rename, roster addition or removal, +symlink substitution — moves at least one of `st_size`, `st_mtime_ns` or +`st_ctime_ns`, or the directory's own stat identity, so it is a memo miss and +the existing refusal fires at the same access it fires today. Every structural +change to a live frame — a replaced column, a reindex, a dtype change, a +rebound buffer, a flipped writeable flag — moves the witness and is likewise a +memo miss. What is deferred to the epoch's unconditional final re-validation is +exactly two things: an in-place value mutation of a live buffer, and an on-disk +rewrite that leaves all five stat fields identical. On this machine the second +is not reachable without privilege: an unprivileged same-length in-place +rewrite followed by `os.utime` restores `st_size`, `st_ino` and `st_mtime_ns` +but not `st_ctime_ns`, and `setattrlist(ATTR_CMN_CHGTIME)` refuses with +`EPERM` (measured 2026-09-15, APFS). That is an observation about one +filesystem, not a guarantee, which is why the final re-validation is +unconditional rather than conditional. + +**New cost class.** Per accessor use: O(modules) producer encoding + O(roster) +`lstat` + O(columns) witness — microseconds. Per run: one full validation at +first use plus one at each epoch exit, instead of one per accessor use, plus +two signature passes per memoised capsule at each exit, which bracket that +exit's own validation and cost the same `lstat` walk a borrow pays. + +### 2. `AuthenticatedAsec2024NativePopulation.frame` + +`asec_2024_native_population.py:449-499`. `.frame`, `.context`, `.receipt`, +`.to_bytes()` and `validate()` all route through `_state()` → `_checked()` → +`_validate_state(state)`: every frame access is a full re-authentication, +including a full SHA-256 of all seven ASEC source files. + +**Guarantee today.** On every borrow: the implementation encoding matches, the +descendant and parent frame identities and the frame context match, the parent +capsule re-validates, the coverage/anchor/field owners re-verify, all seven +source files re-hash to their pinned digests with a before/after `fstat` guard, +and — because a file check or a foreign owner call may yield — the producer and +both frame identities are re-checked *after* the file loop as well as before. + +**Change.** Same two tiers. Tier A keeps `_encode(_implementation())` and adds +the stat identity of the seven `state.source_files` paths and of their parent +directories (which `_file_identity` does not cover: it opens `O_NOFOLLOW` on +the final component only). `_State` here is a `NamedTuple` and a test pins its +immutability, so the memo lives in a side table keyed by `id(owner)`, holding a +weak reference to the owner beside the signature and the state. An entry +answers only when that reference still resolves to the same object, dead +entries are dropped as the epoch closes, and the whole table is cleared when +the outermost epoch exits. + +The owner's own close runs after this capsule's. At the outermost close this +capsule's memo is already cleared, so the owner's final validation reaches the +complete file check; at an inner nested close the memo is still live with a +refreshed signature, so the owner's validation is a memo hit — unless a source +moved while this capsule's close was validating, in which case that close +records no signature and the owner's validation is a miss that re-runs the +complete check. Nothing is skipped in net either way, because this capsule's +exit has just re-validated it in full. + +**Why the guarantee survives.** As in mechanism 1. The documented +yield-tolerance pattern is preserved intact: a memo hit runs neither the file +loop nor the checks that bracket it, and a memo miss runs the whole of +`_validate_state` unchanged, so the bracketing pair is never split. + +**New cost class.** Per access: O(modules) + seven `lstat`s + O(columns) +witness. Per run: two full validations, each close's own bracketed by the two +signature passes described above. + +### 3. Per-node source content keys in the executor + +`executor.py:2452-2457` re-derives `source_content_key(name, source_paths[name])` +for every source a cold-executed node declares, and `keys._directory_identity` +(`keys.py:31-51`) `read_bytes()`es every file of a directory source and hashes +the whole tree. Cost is O(executed source-declaring nodes × source bytes): the +3.48 GiB tree is re-read per node. + +**Guarantee today.** The check runs after the kernel returns and **before** +`_validate_result`, `_apply_result`, the population observer and `_write_node`. +Its real stake is the last one: no artifact computed from mutated source bytes +is ever persisted under a node key that claims the pre-mutation bytes, which +would poison the content-addressed store permanently. + +**Change.** `run_graph` gains a per-run cache in its own locals — never inside +`source_content_key`, which stays pure because four existing assertions call it +twice on one path with the bytes changed in between and require different +answers. The cache is keyed by source name and a *stat signature*: for a +regular file, its `_stat_identity`; for a directory, the sorted recursive +roster of relative names with each member's `_stat_identity` and the stat +identity of every directory in the tree, plus the resolved identity of any +member that is a symlink — `_directory_identity` selects members with +`is_file()` and reads them with `read_bytes()`, both of which follow the link, +so the signature follows it too and the two sides walk the same members. The per-node check derives the +signature (an `lstat` walk, no byte reads); on a match it reuses the cached +key, on a mismatch it re-derives the content key in full and raises the same +`NodeRejected` if it moved. Immediately before the manifest is built, every +source is re-derived **in full, cache bypassed**, and a mismatch raises. The +run-end re-derivation is written inline rather than through +`_source_paths_and_keys`, because a build test counts calls to that exact code +object. + +**Why the guarantee survives.** The refusal still precedes `_write_node` for +every mutation that moves any stat field — which is every ordinary mutation, +including one that adds, removes or renames a file inside a directory source, +since the roster is part of the signature, and including a change to the bytes +behind a member symlink, whose own five stat fields never move when its target +is rewritten. The narrowed case — a byte rewrite +that preserves all five stat fields — is caught at run end, before any caller +receives a manifest, but after intervening nodes have written store records. +That residual is stated rather than hidden: a run refused by the run-end check +may leave store entries behind, and those entries are self-consistent under +`_require_record_shape`, so the operator must clear the store for that run. +Against this, the executor gains a guarantee it does not have today: sources +are re-verified at run end even when the mutation happens during a node that +declares no source, which today is invisible. + +**Record.** `RunManifest` gains `source_identities`, an attached mapping modelled +exactly on `populations` and `mass_ledgers` — `repr=False`, `compare=False`, +excluded from `content_addressed` and absent from `to_json`. It carries the +run-end re-derived key of every source. It moves no manifest key, no manifest +JSON byte, no node receipt and no cache record. + +**New cost class.** Per node: an `lstat` walk of the source tree. Per run: two +full content derivations instead of one per executed source-declaring node. + +### 4. The ACS record fence + +`acs_person_coverage_authentication._records` fences raw logical CSV records +out of a zip member before any UTF-8 decoding or CSV allocation. It is a +pure-Python `for byte in block` loop over 4,096-byte blocks, and the member it +fences is the ACS person CSV inside `csv_pus.zip` — 2.4 GB expanded across +`psam_pusa.csv` and `psam_pusb.csv`. With `peek` and +`acs_person_coverage_columns.lines` it is 57–65 % of the admission phase in +every measured run. + +**Guarantee today.** For every byte, in this order: the record length must be +below its cap (64 KiB for the first record, 400 KB afterwards) or +`CSV_RECORD_BYTES`; an unquoted comma resets the token counter, any other byte +advances it and must stay within 64 KiB or `CSV_TOKEN_BYTES`; a double quote +toggles quote parity, which **persists across records**; an unquoted LF ends a +record, an unquoted CR ends it and absorbs a following LF. Record boundaries +are located by quote parity only; CSV validity remains the unchanged strict +literal parser's job. + +**Change.** The same state machine, driven by `bytes.find` and `bytes.count` +over 1 MiB blocks instead of a Python loop over individual bytes. Terminator +cursors are cached per block so an absent carriage return is searched for once +rather than once per record. Every completed record, and every in-flight record +at a block boundary, passes through a bound check: a record no longer than +64 KiB cannot have violated either cap, because the token counter resets at +each record start and so never exceeds the record's own length, and the record +cap is never below 64 KiB. Any record longer than that — which real ACS data +never produces — is replayed byte by byte through the original fence, so the +refusal code and the point at which it fires are exactly the original's. + +**Why the guarantee survives.** The yielded record sequence and the refusal +raised are proven identical, not argued: `test_us_acs_record_fence_scan.py` +compares the shipped `_records` against a verbatim copy of the pre-change loop +over 4,200 randomised byte strings, every boundary case (bare CR, CRLF split +across a block boundary, quoted terminators, odd quote parity carried across +records, BOM, empty lines, trailing CR at EOF) and every bound case on both +sides of every cap. The same comparison runs against the real staged +`csv_pus.zip`. + +**New cost class.** Linear in member bytes at C speed: measured 10.8 MB/s → +522.8 MB/s on the real archive, digests identical, a 48.5× reduction. + +**Pin.** `acs_person_coverage_authentication.py` is one of two modules whose raw +file SHA-256 is pinned in `acs_native_coverage_binding._ACCEPTED`, so this +change re-pins it. The new value is derived by `shasum -a 256` on the changed +file and recorded in the lane report. + +### 5. The kernel context digest + +`_context_digest` (`executor.py`) digests every column of every table twice per +node — once before the kernel runs and once after, to prove the kernel did not +mutate its input context. `_update_series` took a fast path only for masked +extension arrays exposing `_data` and `_mask` ndarrays. On pandas 3.0.3 an +ordinary float, integer or boolean column is a `NumpyExtensionArray` with +neither, so it fell through to `to_numpy(dtype=object)` and a per-value Python +loop with two `hashlib` updates per value. + +**Guarantee today.** The digest is a total function of the column: its dtype +string, the object projection's dtype and shape headers, and one framed payload +per value. + +**Change.** `_object_stream` builds exactly those bytes with numpy for the three +kinds whose boxed value is always one exact Python type — `float` packed to +eight native IEEE-754 bytes, `bool` rendered as `b"b1"`/`b"b0"`, and `int` +rendered as a variable-width decimal. It returns `None` for everything else, +including a `Categorical`, whose values array is an integer-coded view and +would otherwise be mistaken for an integer column. Both `_context_digest` calls +remain: the second one is the mutation guard, and reusing its result would +delete the guard. + +**Why the guarantee survives.** +`test_graph_executor_series_stream.py` compares the shipped helper against a +verbatim copy of the pre-change body **at the byte level**, not only at the +digest, across every column kind the projection can carry, the float specials +(signed zero, both infinities, the subnormal extremes), non-canonical NaN +payloads with their sign bit, both `int64` endpoints, empty, sliced and +strided columns, labelled indexes, and a 200-frame random sweep. + +**New cost class.** Measured 0.47 s → 0.09 s for a 6,928 × 240 frame; 183 → +18 ns per value for floats, 226 → 14 for booleans, 383 → 119 for integers. +Still linear in rows, but with no Python-level work per value. + +## What this note does not claim + +Nothing here is a statement about dataset quality, calibration or release +eligibility. The PR-CI / certification boundary is unchanged: these are code +contracts, and green checks do not certify any data artifact. diff --git a/docs/us-other-disability-host.md b/docs/us-other-disability-host.md new file mode 100644 index 000000000..53d95e8da --- /dev/null +++ b/docs/us-other-disability-host.md @@ -0,0 +1,41 @@ +# Other-disability enrichment continuation + +`run_us_survey_enrichment` accepts `other_disability_completion=True` with an +explicit nonnegative integer `other_disability_seed`. The default remains +disabled. The existing `n_estimators` controls its quantile forest as well. + +The host first completes the requested enrichment fragments and retains their +issued owner, population, terminal artifact, source declarations and kernel +objects. A second graph extends that exact predecessor with the other-disability +fragment. This ordering gives the fragment actual receiving values and terminal +bytes when it declares its dependencies. Every inherited node must hit its +authenticated cache; a node-scoped dispatch guard refuses inherited execution. +The continuation receipt binds that guard's separate policy and implementation. + +The disability model always qualifies full original current ASEC donors with +DESIGN weights, independently of `full_original_amount_donors`. It uses current +age, employment income and self-employment income. Known nonreceipt supplies a +zero target. Source-unknown ASEC dollars stay unknown and do not enter fitting. +Original ACS people aged 15 or older receive one draw each, copied to clone arms +0 and 1. Under-15 dollars remain unknown. Source knownness and modeled canonical +knownness have separate columns. The source artifact retains published routing, +allocation and topcode evidence. + +A keep-all population version precedes the canonical `disability_benefits` +write. The host compares every carried column, entity axis, weight and inherited +artifact against the retained predecessor. When `original_application_seed` is +also supplied, original PUF placement follows the disability attachment. Its +existing binding observes that real terminal population during graph execution. + +Inherited Supplemental Poverty Measure inputs and amount artifacts retain their +source/fragment scope. They do not assert quantities calculated on the final +candidate. Public consumers still receive a checked `SurveyEnrichmentRun` whose +parent is the genuine PUF run. No country engine runs inside this continuation. + +The maintained invented test covers the default enrichment terminal, cold and +required replay, source knownness, population conservation and mutation refusal. +Its replay monitor refuses QRF `fit_target` and `apply_target` calls; ancestor +verification can repeat other empirical calculations. +Combined original-arm placement and SPM settings need a separate integration +proof. Actual-data qualification, scientific acceptance, calibration and release +remain pending. Poverty comparisons neither select this model nor gate release. diff --git a/docs/us-pre-geography-survey-financial.md b/docs/us-pre-geography-survey-financial.md new file mode 100644 index 000000000..6fcc9c646 --- /dev/null +++ b/docs/us-pre-geography-survey-financial.md @@ -0,0 +1,45 @@ +# Financial completion before geography + +The US financial host now has two explicit entry points. The existing +`run_atomic_survey_financial` keeps its assigned-geography prefix and required +configuration. `run_pre_geography_survey_financial` uses the genuine survey +CREATE, allocation, whole-household clone EXPAND and clone ownership claim. +Its result retains the actual preparation, allocation and clone. It has no +placeholder geography population or configuration. + +The new variant lets support-changing financial and PUF work precede a future +completed-spine geography suffix. It does not implement that suffix, native52 +tail placement, selective clone2 receiving contracts or qualified GQ support. +Those remain separate work. Observation-time state and PUMA are source facts; +the new path can use qualified source state as a predictor without assigning a +synthetic block. + +Both paths use the same financial, property, tax and child-completion algorithms. +The actual issuer seals one of two exact result types and prefix modes. The +atomic checker still accepts only the atomic result. The common checker used by +PUF consumers accepts precisely the two issued types and repeats the same live +source, implementation, artifact, population and final-state checks. Public +result constructors and copied Frames grant no authority. New source and graph +identities require fresh compatible stores; old-store reuse is not asserted. + +For the new path, independent reconstruction verifies the complete allocation, +clone lineage, DESIGN anchors, weight halves, mass ledger and distinct +pre-claim/post-claim ownership. Its graph binds real preparation, allocation and +frame-context evidence. Optional status, development-input and child-completion +nodes retain that evidence; no geography gate or invented successful validation +receipt is supplied. The existing atomic receipt shape is preserved. The new +receipt explicitly says atomic geography is absent and release eligibility is +false. + +The old sampling-origin financial-successor bridge remains atomic-only. A +pre-geography result is not an admitted calibration population through that API. +Likewise this extension does not certify immigration, enrichment, engine input +completeness or a release merely because their parent types are descriptive. + +The first tests cover genuine invented-source cloning, source-qualified +predictor/property declarations, observed demographic state, altered-population +refusals and the unissued/invalid-option boundary. They execute no successful +financial model, PUF transfer or geography assignment. Cold/required fitted +financial and PUF composition require separate evidence before adoption. No +actual microdata or launch acceptance follows from source review or these tests. +Poverty remains comparison-only and prior wages remain excluded. diff --git a/docs/us-property-completion-routing.md b/docs/us-property-completion-routing.md new file mode 100644 index 000000000..debe0ee45 --- /dev/null +++ b/docs/us-property-completion-routing.md @@ -0,0 +1,82 @@ +# Property completion routing + +`current_property_completion_routing.build_property_completion_routing(qualified, clone_frame)` describes the remaining ordinary-interest and dividend work over the original ACS/ASEC source population. It assigns no amounts, fits no model and changes no Frame. Its `PropertyCompletionRouting` return value is descriptive; constructing it grants no source, complete-parent or release authority. + +The inputs are the existing `QualifiedPropertyIncomeSources` description and the complete initial clone Frame. The operation checks original source order, integer identities, source-native agreement, source ages, component knownness, the exact two clone roles and original household DESIGN weights. It preserves the complete input physical descriptions. Actual source custody remains external: a country host must retain and requalify the preparation and source issuers around relevant I/O. Neither the description's type nor a separately supplied DESIGN vector authenticates that custody. + +## Routes and retained reasons + +Each original person receives one route: + +| Route | Meaning | +| --- | --- | +| `unsupported_under15_measurement` | The source age is below15. No child amount or adult extrapolation is supplied. | +| `source_review_required` | A required source amount/receipt or ACS anchor is malformed, outside its published domain, inconsistent or has an invalid adjustment. | +| `carry_known_components` | Both independent ASEC components are known. Joint-fit exclusion does not erase them. | +| `existing_acs_anchor_decomposition` | The adult ACS aggregate anchor is observed and qualifies for the existing decomposition branch. The separate components remain unresolved at this stage. | +| `acs_anchor_completion_review` | An adult ACS anchor remains missing. This operation supplies no fallback. | +| `asec_component_completion_review` | At least one adult ASEC component remains unresolved. Any independently known other component is preserved. | + +Under15 is the first route; malformed, positive-outside-universe and contradictory observations remain visible as overlapping reasons. Adult source errors precede generic missingness. Joint-donor exclusions remain separately named reasons and never by themselves turn otherwise known O/D into unknowns. Unknown new source statuses refuse classification instead of silently entering a generic completion branch. + +The long component table preserves amount, individual knownness and the exact reporting/literal statuses. `observed` denotes a qualified observed amount; `derived` distinguishes a qualified known-nonreceipt zero; `unresolved` retains missing amounts. An observed ACS aggregate is recorded separately from its unresolved individual components. No component receives a `modeled` origin here; only a later operation consuming actual checked model outputs can assign that origin. Allocation and disclosure flags remain source provenance rather than donor filters. + +## Outputs and support + +- `person`: one row per original, with source/native identity, source age, original household, independent O/D knownness, separate ACS anchor and exclusive route. +- `components`: two rows per original for the two required component families. +- `reasons`: all overlapping required-source and joint-fit diagnostics. +- `clones`: exact original/native/source identity and clone roles mapped to the initial receiving person IDs. +- `summary`: raw original-person and unique-household counts, household-inherited DESIGN person mass and union-household DESIGN mass for each route/reason. Clones do not multiply these summaries. Zero-weight support remains visible in raw counts. +- `payload`: deterministic JSON with the tables and explicit statements that it neither assigns amounts nor authenticates a source or complete parent. Integer IDs remain integers, including those above2⁵³; unknown amounts serialize as null. + +These are support diagnostics, not prevalence estimates or calibrated totals. Overlapping reason masses must not be added as if the reasons were disjoint. + +## Integration boundary + +`property_completion_artifact_output()` supplies the typed `completion_routing` artifact declaration. This additive slice does not connect a new node or change the accepted38-node host. The next host integration should emit the payload at the existing property source-projection boundary, bind the actual retained source and clone descriptions, and recompute/compare it during requalification and reconstruction. That host change requires its own source/graph identity and observation checks. + +This operation does not create a child-income measurement model, fill an unknown adult input, assert that the PUF covers a missing source population or relax the existing completeness gate. + +## Opt-in host artifact + +`PropertyIncomeOptions(completion_routing=True, ...)` requests a fourth typed +artifact, `completion_routing`, from the existing +`survey_property.source_projection` node. The default remains disabled and +retains the previous canonical option payload. Neither setting changes the +numerical model declarations, source universes, clone attachment or tax split. +An implementation source edit does change executable cache identities. + +The private artifact contains the complete original-person, component, reason +and initial-clone tables. The source receipt contains an explicitly allowlisted +aggregate summary and its private artifact digest. The actual graph HTML and +text views expose receipts, so private identifiers and per-person values must +never enter this summary. Aggregate DESIGN support is descriptive; it is not +calibrated representation, a release gate or a disclosure certificate. + +Source execution reuses its existing qualification and builds routing once. +Independent host reconstruction checks exact diagnostic bytes and the expected +aggregate receipt on cold execution and required replay. Ordinary branch checks +retain their existing three source-artifact checks and do not recompute routing. +The host retains all declared artifact hashes and canonical options; its checked +parent/source lifetime remains the authority boundary. The pure routing result +and the artifact grant no source or complete-parent authority. + +The bounded integration controls exercise qualified invented source descriptions, +option/declaration behavior, tamper refusal and the real public serializers. +Actual 38-node host acceptance is a separate test scope; this feature does not +claim native completion, PUF readiness or a released dataset. + +The repository's existing 38-node tax-host fixture now enables completion +routing on both its cold and required calls. Its controls retain full population +and unknown-input checks, verify the private typed artifact and aggregate receipt +on replay, and refuse changes to the retained artifact bytes. The local native +harness predicates can share those fixtures through a separately pinned acceptance +wrapper; they are not a machine-specific dependency of repository tests. + +The bounded acceptance passed 27 focused controls and three actual invented +cold/required host tests at source `c06d29ea8`. The latter reuse one 38-node pair, +check the full saved Frame after final source revalidation, and reject changed +retained artifact bytes. They preserve the fixture's false tax-input completeness +verdict. Exact scope and receipt hashes are in the +[13 September integration record](../experiments/us-launch-consolidation-20260913.md). diff --git a/docs/us-property-income-graph.md b/docs/us-property-income-graph.md new file mode 100644 index 000000000..d06b4c869 --- /dev/null +++ b/docs/us-property-income-graph.md @@ -0,0 +1,58 @@ +# Property income model graph + +`microcosm.build.us_runtime.graph_property_income.property_income_nodes` +declares ten numerical nodes: four conditional QRF fits, four chained draws, +one operation that writes the raw draws as columns, and one signed +reconciliation. It uses the existing QRF and reconciliation implementations. +The fragment consumes separately supplied original donor and recipient +populations; it does not qualify native sources or attach results to clones. + +The shared columns in `property_income_constants.py` describe ordinary +interest, retirement-account interest, dividends and signed broad property +receipts. The first three are nonnegative; broad property receipts may be +negative. `property_reported_total` is a required predictor and the recipient +reconciliation anchor. The source bridge supplies the ASEC reported aggregate +for donors and the adjusted ACS INTP amount for recipients. It must preserve +their different reference periods and account for unresolved income routes. + +The training wrapper requires finite float64 predictors and components, +original DESIGN weights, and a component sum equal to the supplied donor +aggregate. It delegates actual fitting to `LegacyQRFTrainKernel`; it does not +filter, repair or reconcile donor observations. The owning source preparation +must establish original household membership, design-weight mapping, eligible +donor support and source knownness before invoking this fragment. A weight +kind label alone does not establish original source authority. + +Recipient draws preserve the complete target order and earlier raw draws as +conditioning inputs. The column operation binds every raw value to its apply +checkpoint, exact recipient index, sibling producer, and final model history. +The graph executor supplies the authenticated artifact edges. Direct calls or +copied checkpoint bytes do not authorize source or model admission. Person IDs +remain int64 independently of the pandas index used by QRF checkpoints. + +Reconciliation retains raw draws in separate `draw_` columns and exposes each +component's adjustment, active bound, residual and objective. Scales and +tolerances are required parameters. Changing them recomputes reconciliation; +changing recipient anchors recomputes recipient draws while reusing donor fits. +Negative or zero net income never forces all positive components to zero: +positive interest and a property loss can coexist at either total. + +Missing anchors and under-15 source blanks require an explicit upstream +decision; this fragment does not manufacture zeros. Original ASEC observations +are not reconciled to ACS respondents. The next country integration must join +the complete source-qualified basis, draw once per original ACS person, retain +the result on both initial clones, and derive the declared tax splits before +running the dependent PUF successor. Retirement-account earnings are not +ordinary taxable/exempt interest. Broad property receipts are not yet a +rental-income tax leaf. PUF-owned alternatives must not subsequently be forced +back onto the survey anchor. + +The initial invented test uses 44 donor persons and four recipients, including +negative/zero totals, large int64 identities, original membership, metadata and +a zero-weight household. It executes the real twelve-node graph including two +source nodes, compares draws with direct weighted QRF, compares reconciled +values with the pure projection and reopens the store in required-cache mode. +Twenty-two tests also cover delegated training-code identity, dependency reuse, incorrect weight kinds, +unknown/inconsistent donors, altered draw history and invalid declarations. +They establish numerical and graph contracts, not native source acceptance, +statistical adequacy, tax correctness or release readiness. diff --git a/docs/us-property-model-receipts.md b/docs/us-property-model-receipts.md new file mode 100644 index 000000000..1c16e9618 --- /dev/null +++ b/docs/us-property-model-receipts.md @@ -0,0 +1,65 @@ +# Property model receipt verification + +`verify_property_model_receipts(nodes, donor_population, recipient_population, +artifacts)` in `graph_property_income_receipts.py` reconstructs the existing +four fit and four apply receipt dictionaries. The caller passes the ordered +property fragment, the two original `Population` branches, and authenticated +artifact bytes keyed by `(node_id, output_name)`. Additional non-model fragment +nodes and their artifacts may be present; the country host verifies those. + +The caller must first bind the store bytes, typed producer outputs and exact +population dependencies. This helper loads the existing trusted model format, +which contains pickle. A self-consistent digest or a returned receipt does not +authenticate arbitrary model bytes or establish source authority. The owning +host retains responsibility for source qualification, complete population +identity, physical seals, borrowed inputs and final I/O checks. + +## Checks + +The helper reconstructs the shared graph declarations for the four ordered +property targets and compares their complete normative bytes. This binds input +slices, population versions, predictors, target order, seed, phase, fit options, +model/training sibling edges, prior raw-draw edges and output artifact types. +Training stays donor-only. The existing model protocol initializes its exact +chain-start state against the original design-weight donor without fitting. +Each trusted model must match the complete ordered training history, actual +consumed float64 donor bytes, donor pandas index, resolved weights, configuration +and preceding/succeeding training states. The first three property components +remain nonnegative and the four-component sum must equal the observed donor +aggregate. Both branches require finite float64 model columns and unique int64 +person IDs. + +Ordinary population apply checkpoints have a pandas-index identity but no +independent recipient-feature digest. The verifier therefore performs four +additional `apply_target` operations using the retained fitted models, current +recipient features, raw prefix and exact initial draw RNG. It compares the +resulting raw bytes and complete application states with the published +artifacts. It validates every model/raw hash in the ordered application history. +It never calls `fit_target` or a model-fitting convenience method. + +This replay establishes output consistency with the supplied feature values; +a model can produce the same output for different inputs. Exact feature and +person-ID provenance remains the caller's authenticated population binding. +The existing protocol uses pandas row labels for model state and raw draws; +the verifier does not substitute entity IDs or invent a new identity digest. +Graph owners must continue to bind the distinct int64 person axis themselves. + +## Scope and cost + +The return value uses the existing raw kernel receipt schema; the host adds and +checks the executor capability/input-writer envelope through its existing +`_states` path. It adds no status, +source issuer, receipt artifact type, model format or host framework. The +helper does not fit, reconcile, attach clones, assign tax treatment or modify +its input frames. Cost per call is one chain initialization, four trusted model +decodes and four deterministic apply replays over the recipient branch. The +owning host decides when to perform this validation; no native-scale runtime +claim follows from the small tests. + +Tests reuse the actual twelve-node property graph fixture with two trees, +44 invented donor persons, four recipient persons, non-default pandas row +labels, int64 IDs above `2**53` and zero-weight records. Original cold and +required-cache kernel receipts match the verifier. Separate controls alter +donor/recipient axes and values, design weights, graph declarations, model and +training bytes, model/raw history and draw RNG. Country engines, native sources, +the full PUF host and publication are outside these checks. diff --git a/docs/us-property-tax-leaves.md b/docs/us-property-tax-leaves.md new file mode 100644 index 000000000..afc520911 --- /dev/null +++ b/docs/us-property-tax-leaves.md @@ -0,0 +1,124 @@ +# Property tax leaf rebase + +`graph_property_tax_leaves.py` provides three deterministic graph nodes for an +already checked population with property components and incumbent tax leaves. +It does not qualify sources, fit models, attach clones, grant a checked-run +handle or wire the country/PUF host. The owning host verifies the original +component observations, clone pairing and complete frame before using it. + +## Numerical operation + +`split_property_tax_leaves(person)` returns four float64 columns indexed by exact +int64 `person_id`. It imports the maintained fractions from `cps_carried`: + +| Input component | Primary leaf | Complement leaf | +| --- | --- | --- | +| `property_ordinary_interest` (`O`) | `taxable_interest_income = O * 0.680` | `tax_exempt_interest_income = O - taxable_interest_income` | +| `property_dividends` (`D`) | `qualified_dividend_income = D * 0.448` | `non_qualified_dividend_income = D - qualified_dividend_income` | + +The complement uses subtraction after computing the primary. This is a declared +operation order, including for the dividend complement that legacy code computes +by multiplying the complementary fraction. These fractions are modelling +assumptions; they do not establish a person's observed tax treatment. + +Interest and dividend knownness are separate. A finite nonnegative input produces +its two leaves; a missing input produces two NaNs, replacing any earlier tax +predictions. Zero stays an observed numerical zero. A negative or infinite O/D +input, an infinite retirement component, incompatible dtype or ambiguous person +identity refuses. `property_retirement_interest` is retained as an auxiliary +component and never enters either tax-interest leaf. Its missing value does not +invalidate known O/D values. Broad property receipts, net anchor sign, source age, +donor-fit eligibility and earlier predictions never supply missing O/D values. +No under-15 zero completion or missing-adult prediction occurs here. + +## Three graph nodes + +Call `property_tax_leaf_nodes(frame, population=base_version, +projection=projection_input, reconciliation=reconciliation_input, atol=..., +rtol=...)`. Both tolerances must be explicit finite nonnegative Python floats. +The two `ArtifactInput` aliases must be `projection` and `reconciliation`; the +host supplies their actual producer IDs and nominal types. Register +`PropertyTaxReceivingKernel`, `PropertyTaxLeavesKernel` and +`PropertyTaxLeafGateKernel`. + +When compiling before the earlier financial/property nodes execute, the host +may pass `anticipated_outputs=tuple_of_owned_descriptors`. The factory combines +those existing declarations with the real checked Frame's entity/axis/group +shapes and dtypes. It creates no placeholder values or synthetic population. +It rejects duplicate descriptors, unknown entities, structural ID fields, +masked outputs and incompatible or unclaimed rewrites. At execution the host +re-declares the nodes from the actual complete post-property Frame and requires +the same normative declarations before reconstructing the results. The default +empty descriptor tuple still requires a complete actual input Frame. + +1. `survey_property.tax_receiving` opens a new population version using + FILTER-all. This inherits the prior owners and permits explicit replacement of + the four incumbent leaves. The executor adds a real FILTER conservation record + to the population mass ledger. It preserves the existing `Frame.mass_log`. +2. `survey_property.tax_leaves` replaces the four leaves on that version and + emits `diagnostics` of type `microcosm.us.property_tax_leaf_diagnostics/v1`. +3. `survey_property.tax_leaf_gate` recomputes the leaves and their knownness, + checks exact leaf bits, partition sums within the declared tolerances, and the + complete diagnostics payload. It emits `verification` of type + `microcosm.us.property_tax_leaf_verification/v1`. + +The gate binds ordered person IDs without floating-point conversion, current +O/D/retirement bytes, all four leaf byte digests, fractions and operation order, +individual knownness, and the typed projection/reconciliation artifact keys and +payload hashes. It includes every physical person, including dependents and +zero-weight members. `complete` is false if any person lacks O or D; a successful +numerical check with missing inputs reports `evidence_absent`. Complete numerical +inputs report `pass`. Neither outcome establishes source, model, tax-validity or +release authority. The artifact explicitly leaves full-frame verification to +the host. + +## Supported receiving shapes and host responsibilities + +`Frame.select(all)` currently refuses link tables and normalizes group indices. +The declaration factory therefore refuses links, nondefault group indices, +orphan groups, empty person frames and groups lacking a readable non-ID column. +The last condition ensures the kernel context exposes group IDs for the +receiving check. Orphans are already invalid at Frame construction; the factory +also detects a later membership mutation. The kernel rechecks exposed group +membership and logical group indices immediately before returning its keep mask. +It never silently prunes a group or switches structural operation. + +For supported frames, person rows/order/IDs, entity tables, weights and their +kinds, schema, strata, metadata and frame mass log are preserved, apart from the +four declared leaf replacements. The new population version, owners and +conservation record are intentional changes. Kernel contexts do not expose the +complete schema, metadata or population ledger, so the host must retain and +verify those independently. It must also qualify the complete supported shape +before adding these nodes, bind actual artifact producers and stored payloads, +and recheck the original owners and output after its final relevant I/O. + +For independent expected-result reconstruction, the host can call each kernel's +`run(KernelContext)` on the retained current population and authenticated typed +artifact aliases. The existing executor `_project_context` and `_apply_result` +provide the canonical context projection and result application. FILTER results +must be materialized through the validated keep mask before `population.patch`; +passing its raw `keep` result directly to that lower-level patch function omits +the executor's frame selection. Ordinary rebase/gate results use the existing +patch path. These deterministic kernels consume no random draws. Expected +artifact bytes and receipts can then be compared with the actual store and node +states, and expected full populations with the host's retained observations. +This fragment adds no separate executor, authority registry or public run issuer. + +Future PUF integration must depend on this verification artifact and require +complete tax leaves over every physical member the PUF consumer needs. The +fragment does not add that host dependency itself. Matching checked inputs give +matching outputs on paired clones; the fragment does not infer or repair clone +pairing. Capital gains retain their earlier conditioning and are not rebased by +this change. + +## Verification scope + +The 32 invented controls exercise a real four-node graph including its source, +ContentStore cold execution and required-cache replay. They cover negative, +zero-net and positive anchors with loss offsets, independent O/D unknowns, +known O/D with unknown retirement/broad components, integer IDs above `2**53`, +row permutations, empty pure input, malformed values, altered leaf bits, +diagnostics and artifact bindings, full-frame preservation, zero-weight records, +the population conservation record and unsupported receiving shapes. No native +survey, country engine, PUF execution, fitting or publication is part of this +acceptance. diff --git a/docs/us-puf55-original-development-placement.md b/docs/us-puf55-original-development-placement.md new file mode 100644 index 000000000..e3e33833e --- /dev/null +++ b/docs/us-puf55-original-development-placement.md @@ -0,0 +1,90 @@ +# Original-arm PUF55 development placement + +The survey enrichment host exposes optional original-arm application through +`run_us_survey_enrichment(..., original_application_seed=...)`. Omission preserves +the existing path. An explicit integer seed must differ from the retained +clone-one seed. The original arm reuses the genuine route fits and runs every +selected profile target with that application seed; it does not refit the donor. +This development path does not qualify a release. + +The selected PUF55 profiles contain **52 person outputs and 3 tax-unit outputs**. +The six SCF mortgage-detail fields in the full65 compatibility roster are not +PUF55 targets. The person outputs form three disjoint categories: + +- Five possible new singleton values: `long_term_capital_gains_on_collectibles`, + `non_sch_d_capital_gains`, `partnership_income`, `s_corp_income`, and + `partnership_self_employment_net_earnings`. +- Twelve qualified fixed values remain unchanged. Eight are survey-preserving + financial leaves; four use the explicitly recorded development assumptions + for pension taxability, regular IRA routing, rental proxy, and farm amount. + These are not newly claimed observed taxable amounts. +- The other 35 person outputs receive no new placement in this slice. This is + not a claim that all 35 lack existing values. + +The three unit outputs are `domestic_production_ald`, +`unrecaptured_section_1250_gain`, and `health_savings_account_ald`. + +`puf55_original_placement.placement_result` returns detached full-axis columns +and a descriptive receipt. Its checks require exact ordered entity IDs and +complete memberships, all twelve fixed values known for every original member, +unchanged known person values, and exact fixed conditioning totals. Mixed-known +units remain unresolved. Singleton means exactly one member in the complete +retained person table, not one eligible member or a capped source return size. + +A candidate output must be absent in the genuine pre-PUF canonical parent and +introduced by the checked arm-one attachment. Its current owner must still be +that attachment, with original-arm nulls intact. A preexisting column is left +alone even if null. Nullness is an integrity check on authenticated ownership; +it is not source authority, a source NIU-to-zero rule, or legal applicability. +The new values are explicitly modeled development transport. Source knownness +and that transport's scientific acceptance remain distinct. + +The pure helper preserves inherited dtypes and requires exact representability +of each new value. Signed outputs are not clipped. A draw outside the maintained +nonnegative domain remains unresolved for that output; no clipping, alignment, +tail cap, multi-person allocation, QBI-support placement, or tuition allocation +is performed. Blocked cells, clone-one values, all other columns and weights are +preserved. Per-output write counts and reasons are descriptive measurements. + +`graph_puf55_original_placement.original_placement_nodes` declares a keep-all +version after the supplied current terminal, then explicit full-column rewrite +ownership for the candidate outputs. The attachment consumes every route's +55 model/training/raw-draw/conditioning/v2-state dependencies, plus qualified +fixed inputs and the exact original recipient matrix. Its result helper invokes +the existing strict complete conditioning merger. There is no fabricated +completed prefix or finalizer-only restoration of survey values. + +The existing enrichment owner admits and rechecks the genuine financial and +PUF owners and qualifies the original fixed inputs. Its private original-arm +binding is not a new issuer. Declarations use predecessor metadata, while the +executor observer supplies the actual completed receiving Population. The +central terminal selector follows state, canonical child-support/veterans, +race, sex, immigration, SPM, then hours according to the enabled fragments. +The receiving version comes from that selected node, including later versions; +it is not assumed to be the earlier PUF version. + +The binding compares declared kernel contexts with the executor's projection of +that complete terminal, including typed weights and strata. The existing host +checks every artifact against actual compiled producer keys. It reconstructs +all preceding branches, compares the actual terminal, rebuilds the keep-all +and placement, compares persisted attachment bytes and complete Populations, +and retains its source requalification, live-code and final storage checks. +A caller-supplied digest, a descriptive placement receipt, or the private +`_capture_checked_puf_ancestors` helper alone cannot admit a receiving run. + +Whole-host qualification still requires all real route fits/applications and a +required replay on admitted source owners. The bounded integration verification +is staged: actual three-target training/application/replay, the complete 110 +application-node declarations for two profiles with explicitly synthetic +metadata suppliers, and invented terminal/context/placement preservation tests. +The declaration check is not evidence of 110 fitted or executed applications. +The genuine full51 parent previously exceeded the bounded test's CPU budget by +itself, so that parent and a whole-host run are reserved for a separately +admitted integration run. Applicability, coverage, and the development +assumptions also need the eventual no-worse comparison before release. + +The local tests use invented Frames for eligibility and patch arithmetic. A +separate complete-envelope test uses synthetic 55-target payloads derived from +an actual three-target test template solely to exercise the strict codec path. +It is not evidence of 55 fitted models, a genuine financial owner, a complete +host run, or a calibrated dataset. diff --git a/docs/us-social-security-beneficiary-convention.md b/docs/us-social-security-beneficiary-convention.md new file mode 100644 index 000000000..54981ceaa --- /dev/null +++ b/docs/us-social-security-beneficiary-convention.md @@ -0,0 +1,109 @@ +# Development Social Security beneficiary convention + +`microcosm.build.us_runtime.survey_social_security_beneficiaries` provides a +pure allocation contract for an explicitly restricted development subset. It +does not qualify actual survey sources, attach graph or Frame columns, run an +engine, or make a release ready. Its public dataclasses contain descriptive +claims, not source-owner capabilities. Validation establishes internal +consistency only. + +## Report evidence and beneficiary income + +The existing Social Security report classifier remains unchanged. Its category +probabilities and conditional mean report shares do not identify beneficiaries +or their benefit incidence. In particular, converting four positive scores +into four positive canonical benefit amounts would manufacture component +incidence. The new core has no such conversion. + +Reporting identity need not equal beneficiary identity. ASEC permits combined +family reports; its reason 7 does not distinguish disabled, dependent, and +surviving children. ACS combines Social Security and Railroad Retirement and +uses a rolling twelve-month period. See the [2025 ASEC dictionary, pages +48–49](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +and [2024 ACS questionnaire, page +18](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf). + +Tax treatment can depend on the entitled beneficiary even when another person +receives the payment. [IRS Publication 915, “Who is +taxed”](https://www.irs.gov/publications/p915). Children can receive family +benefits; exclusion from a survey's reporting universe is not benefit +ineligibility. [SSA family eligibility](https://www.ssa.gov/family/eligibility). + +## Implemented convention + +`resolve_singleton_asec_retirement` requires both keyword assumptions to be +explicitly true: + +- `closed_benefit_unit`: this report includes no nonresident's benefit, and + this resident has no benefits reported elsewhere. +- `stable_roster_over_income_year`: the interview roster is an adequate + benefit allocation boundary throughout the income year. + +The helper accepts only a complete, one-person ASEC housing-unit roster whose +sole person is the reporter; a positive finite annual OASDI report with receipt +code 1; age within the ASEC 15+ reporting universe; known reasons containing +only retirement and optional NIU; and no publisher allocation. This is a +declared source convention, not statutory eligibility logic. Interview age +does not override the reported retirement category. + +The full report amount is assigned to that person's retirement component in +one exact edge. The other three canonical components are zero **by the +convention**, not observed beneficiary zeros. The output records +`resolved_by_convention`, the two assumptions, definition resolution and +calendar-year resolution. It makes no observed-beneficiary, source-admission, +engine-compatibility or release claim. + +ACS, reason 7/8, other/missing reasons, unknown/incomplete rosters, group +quarters, publisher allocation, unknown money, report zero, unresolved +definition and noncalendar periods cannot use this shortcut. These cases need +separate source evidence or an accepted future allocation model. + +## Artifacts and consumer boundary + +- `ReportLot` retains the original survey/vintage/reporter coordinates, + amount or `None`, raw reasons, receipt, publisher allocation, definition and + period. Amount and period changes do not create a new lot identity. +- `BeneficiaryCandidate` is a possible person/component edge with a basis; + it has no money. It can describe a child or unknown/nonresident possibility + without assigning incidence or an equal share. Candidates are descriptive; + this slice does not authenticate relationships or construct a family graph. +- `LotAllocation` keeps candidates, assignments and explicit resolution. + `unresolved_report_lot` leaves the complete reported amount unresolved. + Unknown (`None`), zero, and positive unresolved amounts are distinct. + Zero unresolved dollars do not mean resolved beneficiary assignment. +- `validate_development_consumer_subset` revalidates every supplied lot and + returns four-vectors only for an exact, nonempty requested person subset and + calendar year. Any unresolved lot fails. Repeated original lot coordinates + fail even if their payloads differ. Multiple reports for one beneficiary + fail because overlap has not been resolved. The vectors use the existing + `survey_social_security.COMPONENTS` order and retain convention origin. + +The consumer check cannot discover omitted lots or prove that a caller's +closed-benefit-unit assumption is true. The returned object is a structural +development contract, not an admitted engine input. Do not drop uncertainty +flags or pass unknown zeros to the engine. A parent combined report plus a +child's independent report must not be automatically added or subtracted. + +## Integration still required + +A later graph adapter must bind the live source owner and physical digest, +original source-person identities, the full qualified roster and readset, and +source period and definition. It must preserve report evidence separately from +the existing classifier, expose each candidate/allocation edge, validate the +consumer subset, and attach only through the exact supported original/clone +mapping. Renamed report keys cannot replace that source ownership check. + +No graph adapter, actual-source qualification, clone fan-out, cold/required +replay, consumer engine execution, or national/CD completeness proof is +implemented by this core. It neither changes weights nor synthesizes missing +persons or reports. Family allocation, report overlap, ACS Railroad Retirement +separation and calendar conversion remain separate prerequisites for broad +coverage. + +The invented test module covers the accepted convention, refusal boundaries, +report-zero/unknown distinction, candidate-only children, exact single-lot +conservation, duplicate/overlapping reports, constructible-artifact +revalidation, and the positive-class-score incidence regression. Those tests +do not establish scientific validity or source authority. Poverty remains +comparison-only: no target, tuning, candidate selection or release gate; no +prior wages are introduced. diff --git a/docs/us-survey-completion-host.md b/docs/us-survey-completion-host.md new file mode 100644 index 000000000..432662161 --- /dev/null +++ b/docs/us-survey-completion-host.md @@ -0,0 +1,139 @@ +# Optional US survey completion host + +This source implementation adds opt-in completion to +`run_atomic_survey_financial`. It has not been imported, collected or executed +in this task. Component acceptance does not establish combined-host acceptance. + +Select an exact `ChildPropertyOptions` with explicit scenario ID, scope, +`SupportRequirements`, `JointTransport` and keyed stream. Also select +`property_income` and `rebase_property_taxes=True`. `household_roles=True` +requires child completion; its default is `False`. Existing `person_status` +remains independent. Unsupported combinations refuse before source execution. +There are no scientific defaults or new public issuers. The result remains an +existing-issued `AtomicSurveyFinancialRunValues`. + +The host first issues and retains the actual property parent (35 nodes, or 39 +with status), with tax rebasing disabled. It retains its preparation, source +objects, original legacy/property populations, manifests, store, keys and kernel +registry. Registries are copied before extension. The union executes in the same +store and requires every already-issued parent node to hit cache. + +| Selected path | Union nodes | +| --- | ---: | +| Existing financial / property / property plus tax | 19 / 35 / 38 | +| Existing status variants | 23 / 39 / 42 | +| Property, child, tax | 45 | +| Property, status, child, tax | 49 | +| Property, status, roles, child, tax | 51 | + +A full-row completion FILTER creates a distinct version. Qualified household +roles optionally add two nodes after incumbent reconciliation. Child completion +adds six nodes, including the generic empirical fit and draw. A separate tax +FILTER, split and gate follow child verification. Tax declarations use the +independently reconstructed completed frame, including its actual role/status +and imputation columns. All three tax nodes bind the actual typed child +verification artifact. The country child verifier owns canonical child evidence; +the tax seam binds its payload hash. + +Before compilation, the helper binds exact ordered node identities for receiving1, +optional roles0/2, child6 and tax3. Compensating child/tax count drift refuses even +if the union total is unchanged. The portable child count comes from retained +nodes. Its source-knownness flag comes from the independently reconstructed +child verification bytes that the materialized verifier checks against actual +store evidence; the document retains that same payload's SHA-256. + +The private helper independently applies the maintained receiving FILTER and +role reconstruction before constructing `ChildPropertyBoundary`. Its parent +callback returns that same retained Population after pure checks of the issued +base, source ownership, executable closure and complete receiving/role stamps. +Full base/source and role qualification occurs at host boundaries. Ordinary +child borrows do not repeat role parsing. The child fragment itself still +requalifies its source on each `_current()` call. + +Both cold execution and required replay load artifacts through the actual +compiled producer, manifest, store, keys and numeric scope. Both invoke the real +child materialized verifier with actual executor-completed output, both actual +support Populations, every actual verification input and the actual verification +output. Required hits skip kernels, so that final verifier is mandatory. Expected +full-population states, receipts, owners, weights and ledgers are compared using +the maintained graph state checker. Final source/store I/O precedes pure seals. +A failed enabled-host check permanently revokes its completion boundary. + +Original property verification always receives its original property output. +Status preservation follows the actual FILTER lineage; structural operations +legitimately change inherited owners. Qualified survey reference persons confer +no tax/SPM role, filing status or eligibility. Existing property completion +routing is labeled `pre_child_source_gaps`; those counts are not post-child +completion counts. Public completion metadata is aggregate only. + +Only qualified jointly unmeasured eligible children receive paired modeled O/D, +using full original ASEC teenage donors and original household design weights. +Source observations and all nonrecipient/complement cells remain unchanged. +Modeled zero remains `imputed_zero`, distinct from observation. Tax completeness +requires finite nonnegative ordinary interest and dividends for every physical +person, including dependents and zero-weight rows. Unknown adults remain +incomplete and the maintained PUF admission gate must refuse before fitting. + +The authored finite tests include separately selectable cold/required 49-node +(roles disabled, status enabled) and 45-node (roles/status disabled) pairs, one +shared cold/required 51-node fixture, and one incomplete-adult fixture. The positive +fixture composes status/role/money literals and attachments before issuance, +keeps its teenage donor outside the selected sample, and includes housing roles +only. It does not erase unsupported GQ incumbents. It covers exact declarations, +paired draws, unchanged complement/weights/geography, real tax completeness, +detached custody, altered artifacts/identity and final-I/O revocation. The 45/49 +pairs explicitly select the partial ACS+ASEC fixture, preserve incumbent role +columns without role writes, call actual checked views, and assert count, +declaration/manifest/payload parity and required hits. They do not parameterize +the 51-node adverse suite. All 16 functions / 32 source-expanded cases are UNRUN, +including those fixture-composition assumptions. + +Lineage assertions cover all nine prefix nodes and actual financial, property +and enabled status writers before receiving. Both FILTERs must preserve entity +dimensions, tables, strata and weights, append one ledger record with +`operation="filter"` and `policy="conserve"`, and conserve total, stratum and +partition mass. Assertions follow actual structural owner/version transitions +through role/child attachment and tax-leaf writes. FILTER uses `Frame.select`, +whose Frame constructor copies tables; role column patching additionally uses +explicit deep table copies. The parent-mutation adversary checks the retained +base stamp immediately after mutation and retains permanent child revocation. + +The 49-node control also replaces a retained completed Population with equal +values under a different identity and checks refusal and permanent revocation. +Other controls reuse real observed 51-node evidence to challenge a base cache +miss (which does not change manifest content identity), missing observations, +and final source/key/implementation/artifact identities. Final identity controls +re-read the authentic store and sources without executing kernels. The child +artifact and verification-identity adversaries can reconstruct/refit; they are +not fit-free checks. Source-construction adapters are restored before issuance +to prevent later fixture composition from wrapping an earlier fixture's closure. + +Remaining review scope: T4's financial-only parent refusal needs an authentic +19/23-node run; the retained tax-rebased 51-node parent refusal is authored. T5 +does not exhaust all initial base-key/artifact, final declaration/manifest or +role-reconciliation adversaries. T6's actual source composition and checked-view +behavior need guarded execution; source confirms PUF calls the completeness gate +before source admission. T7 GQ and zero-weight scientific coverage stays open: +the positive fixture excludes unsupported GQ and its source helper fixes positive +weights. A coherent new source fixture requires separate owner review. Unknown +values must remain unknown. The status+tax42 complement check now uses the retained pre-tax property +population, where the original status writer still owns its columns. Final +completion and tax populations retain separate full custody and reconstruction +checks. The isolated repair passed source review; its guarded runtime check and +combined-host acceptance remain pending. + +A new guarded resource proposal is still required. Source-visible work includes +base issuance, one receiving/role reconstruction, independent child numeric +reconstruction, union execution, artifact loading twice during extension, and a +mandatory materialized child verifier. Each host requalification invokes the +base checker, optional full role qualification and source-closure reads. Child +construction has its initial qualification plus `_current()`; each executed +country child kernel calls `_current()` twice, and materialized verification +calls it twice even on required replay. These are call-site descriptions, not a +measured file/read/memory budget. The child 64 MiB artifact and status 2 GiB +serialization limits remain upstream constraints. No national capacity, native +PUF, fiscal, scientific age-transfer or release acceptance is claimed. + +The combined source and runtime-module classification passed independent source +review. Root owns the final source closure, identity/golden reconciliation and +independent runtime gates. Source approval does not establish runtime acceptance. diff --git a/docs/us-survey-development-inputs.md b/docs/us-survey-development-inputs.md new file mode 100644 index 000000000..a68969951 --- /dev/null +++ b/docs/us-survey-development-inputs.md @@ -0,0 +1,46 @@ +# Source-qualified survey development inputs + +The native financial host accepts +`source_qualified_development_inputs=True` to publish four current ASEC income +mappings before PUF application. The default is `False`. + +The option adds `survey_development_inputs.source_projection` and +`survey_development_inputs.attach`. The first node publishes the authenticated +routing basis, source literals, knownness and named development assumptions. +The second maps each qualified original person to its complete pair of survey +clones. Both nodes bind the declared parent inputs and genuine producer keys. +Existing canonical columns cause refusal. + +| Person output | Maintained development mapping | +| --- | --- | +| `taxable_private_pension_income` | Apply the maintained pension fraction to qualified pension and annuity amounts when both are known. | +| `taxable_ira_distributions` | Preserve the qualified regular-IRA route amount when the distribution and routing are known. | +| `rental_income` | Use the qualified net-property amount as the existing rental proxy. | +| `farm_operations_income` | Preserve the qualified farm amount, including a known negative amount. | + +The implementation reuses +[`_development_person_values`](../packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_observed.py) +and the current ASEC income-routing qualifier. Graph parameters and typed +artifacts record the existing rule names and values. They retain the raw totals +and unresolved source taxability or component statuses. These development +mappings do not establish observed taxable amounts or release-science acceptance. + +American Community Survey (ACS) rows remain unknown. Ambiguous, off-route and +otherwise unqualified ASEC values remain unknown as well. A not-in-universe code +does not establish a zero. The attachment preserves identifiers, other columns, +typed weights, strata and population accounting. + +The new attachment follows the selected base financial/property/tax output. +When child completion is enabled, the completion receiving version carries the +four additions from the genuine base. Household roles, child property and the +final tax gate remain the later writers. The output selector returns that final +tax gate for completed runs. Compact retention keeps the development attachment +in the base and retains the existing completion owner. + +The tests execute genuine invented source preparation and routing, then real +source and attachment kernels with cold execution, required replay and +independent artifact/Population reconstruction. They also exercise the actual +child, household-role and tax declaration factories through the compiler and +check the compact roster. These component tests do not execute the complete +financial/property/PUF host. That continuation requires separate verification +before dataset-release claims. diff --git a/docs/us-survey-person-status-graph.md b/docs/us-survey-person-status-graph.md new file mode 100644 index 000000000..87efa31bf --- /dev/null +++ b/docs/us-survey-person-status-graph.md @@ -0,0 +1,90 @@ +# Descriptive person status in the common US graph + +`run_atomic_survey_financial(..., person_status=True)` adds four operations +after the complete survey/PUF-support clone families and atomic geography, before +financial enrichment. The default remains `False`. The existing nine-node +spine/geography prefix and its returned populations remain separate and unchanged. +The optional graphs have 23, 39 or 42 nodes, compared with the respective +19-node financial, 35-node property and 38-node property/tax paths. + +| Node | Input and output | +| --- | --- | +| `survey_person_status.source_projection` | Borrows the actual retained preparation and source qualifier. Creates a private branch of original support and a typed artifact containing exact original identities and literal items. | +| `survey_person_status.literals` | Exposes the complete union of ACS/ASEC raw items and distinct allocation/edit flags on original support. | +| `survey_person_status.recode` | Calls the existing descriptive recoder and checks its result against the retained qualifier. Declares every parse/code field, difficulty/student observation, knownness, universe and period field. | +| `survey_person_status.bind` | Joins the descriptive observations and context onto both initial clone families by exact original-person, native-person, survey-arm and clone-index identity. | + +The receiving Frame carries ten descriptive observations and their explicit +context, rather than repeating all raw strings per clone. Raw/edit/parse fields +remain on original support. The projection artifact is therefore required for +the audit lineage: its typed producer edge, digest and the clone-origin linkage +must be retained by a subsequent export. A receipt alone cannot authorize the +source. Row values, private paths and native identifiers are absent from public +node parameters and receipts. The private JSON artifact represents integers as +decimal strings under `integer_literal`, never lossy JavaScript numbers. + +All column declarations are fixed, including when the selection contains only +one survey arm. Missing opposite-survey fields remain null. Code columns use +nullable `Int64` and observations use nullable Boolean storage, with knownness +separate from the value. No `is_blind`, `is_disabled`, annual full-time student, +prior-wage, sex or household-role writer is added. The +[source definitions and reference periods](us-survey-person-status.md) remain +unchanged. + +The shared host adds a typed status-binding dependency to its existing financial +donor node. A narrow adapter checks that dependency, then supplies the exact +original declaration and artifact slice to the existing donor kernel. This +ordering is visible in the graph. The original source qualifiers continue to +receive the untouched geography population. + +Verification covers the complete output. The status boundary reconstructs every +status value, dtype, null mask, writer and position from qualified source literals. +The existing financial/property verifiers independently verify the remaining +population, including unrelated values, structural links, weights, design +weights and mass ledger. The host retains complete output seals and rechecks +them after its last source requalification. It refuses both a status-only change +with an unchanged financial complement and an unrelated financial-cell change. + +When property-tax rebasing creates a receiving version with FILTER, that +structural node owns the carried status columns in the new version. The host +checks the original status writer on its retained pre-tax parent, then verifies +the complete tax reconstruction and final population seal. It does not accept +arbitrary replacement writers or treat the carried columns as new source facts. + +The fragment validates the actual qualified object on admission. Before and +after each executed kernel callback it checks live code, defaults, closures, +declared artifacts, input/output seals and the original preparation/student +owner relationships. The final host borrow requalifies the original source +members before the last pure checks. Cold execution and required replay use the +same declarations and full materialization comparisons. + +## Validation boundary + +The private projection has an explicit 2 GiB serialized-byte ceiling. Each raw +survey token retains the source qualifier's 64-character bound; the existing +ASEC student-controls issuer also retains its 600,000-row ceiling. There is no +claim that these limits admit the complete ACS/ASEC universe; the inherited +survey graph's preparation/allocation artifacts each retain their 64 MiB limits. +Projection JSON is +materialized in memory before its byte-limit check, and its working memory can +exceed the final serialized size substantially. The small invented fixture will +check a conservative 16 KiB-per-original serialized bound; that is an authored +test expectation, not a measured native-data size. Before a maximum-local or +full-national claim, measure the actual bytes per original and peak memory on +successive small samples and consider a separately reviewed columnar/chunked +private artifact. Repeating that artifact in every clone is not an option here. + +Authored tests include pure schema/binding controls, an actual 13-node +spine/geography-plus-status fixture, optional 23-node common-host replay, and +39/42-node property/tax preservation using invented authenticated sources. +These tests are unexecuted in this source-only change pending review and a +separately pinned bounded run. The later PUF and survey-enrichment hosts are not +modified here; their combined end-to-end preservation remains a separate +integration acceptance. Input-coverage counts describe the input registry, not +the number of Frame columns. + +The root previously reported acceptance of 77 person-status source tests +(75 pure/source controls and two actual owner cases), and 100 canonical-sex +tests (88 plus 12). Those are root-reported source-boundary results, not this +author's independent acceptance, validation of this new graph, a native-file +build or release certification. The frozen native V4 graph is unaffected. diff --git a/docs/us-survey-person-status.md b/docs/us-survey-person-status.md new file mode 100644 index 000000000..31310cda3 --- /dev/null +++ b/docs/us-survey-person-status.md @@ -0,0 +1,100 @@ +# Survey person-status observations + +The additive person-status qualifier retains published difficulty and enrollment +items from the original ACS 2024 and ASEC 2025 person sources. It does not assign +`is_blind`, `is_disabled`, or `is_full_time_college_student`. It is independent of +the existing legacy eligibility operator. The optional +[four-node graph fragment](us-survey-person-status-graph.md) exposes these +descriptions on the common US population; it does not add release coverage or +establish eligibility. + +`qualify_current_survey_person_status(preparation)` accepts the actual retained +`AuthenticatedSurveyPopulationPreparation`. It captures its pinned original +members, selects people through exact original household/person/line identities, +and checks each original age against the unchanged `A_AGE` column. It does not use +a normalized or imputed `age` as the item universe. Twenty-two-digit ASEC native +person keys remain strings; stacked person IDs retain their integer index. + +The result contains the original Frame reference, an origin table, literal source +columns, descriptive observations, and an immutable receipt. The result owns no +source authority. Its `validate()` method rechecks the actual preparation and +student owners and compares all retained value seals after the last owner I/O. +A consuming host must keep those owners, validate after its own last relevant +I/O, and independently verify any materialized descendants and replay. Serialized +rows or a copied receipt do not authorize a source or a graph execution. + +## Meaning and period + +| Observation | Definition | What it does not establish | +| --- | --- | --- | +| `survey_vision_difficulty` | The published answer to blindness **or serious seeing difficulty**, with source and period retained. | Statutory blindness, measured visual acuity or visual field. | +| Other `survey_*_difficulty` items | The particular hearing, cognitive, ambulatory, self-care and independent-living questions. | A program-specific disability determination. | +| `survey_any_applicable_difficulty` | Any known yes among applicable items; false only when every applicable item is a valid no. Completeness and universe contradictions are separate diagnostics. | A medical, duration or eligibility finding. | +| `survey_publisher_disability_recode` | `PRDISFLG` or `DIS`, retained separately from the item-derived battery. Agreement is reported only when both are known. | An independent clinical observation. | +| `survey_full_time_college_student_last_week` | A coherent ASEC enrollment, college and full-time combination for the preceding survey week in 2025. | Annual student status in the 2024 income year or five calendar months of attendance. | +| `survey_college_attended_last_3_months` | ACS school attendance with undergraduate/graduate level in the preceding three months. | Full-time workload. ACS does not measure that here. | + +ASEC difficulty items use the published `PRPERTYP = 2` universe, rather than a +replacement age threshold. The CPS publication may retain an earlier rotation +response; its survey year does not imply a fresh March interview answer. ACS +hearing/vision apply at all ages, cognitive/ambulatory/self-care at age 5+, and +independent living at age 15+. Out-of-universe items remain unknown, not false. +A positive or other non-NIU code outside that universe prevents an item-derived +battery answer and remains visible as a contradiction. + +For ASEC student items, the printed enrollment universe is ages 16–54; the +named zero explicitly includes NIU, children and Armed Forces. The descriptive +recoder therefore also rejects positive enrollment routes for those named +person types and leaves an unresolved type unknown. This is an additional +published-code consistency check, not an alteration of the existing control +issuer's age-only assertion. Missing or unreadable student literals remain +unresolved even outside the age universe. + +All values retain literal/code/parse states. Printed but unnamed numeric range +members remain unresolved. ASEC `PXDIS*` transitions distinguish no change, +longitudinal retention, allocation and changes to blank; nonzero is not a common +allocation flag. Valid published yes/no values survive unknown edit provenance +without becoming claims about an unallocated original answer. ACS `FSCHP` retains +its enrollment scope and the separate `FSCHGP` flag records grade-attending +allocation. The initial source-review packet overlooked `FSCHGP` on dictionary +p.129; this implementation corrects that omission using the same pinned PDF. + +## Source contracts + +- [ASEC 2025 dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`: student values pp.23–24, difficulty values pp.26–27, person type and summary p.28, student allocation pp.28–29, edit transitions p.30 (one-based PDF pages). +- [ACS 2024 dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf), SHA256 `929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff`: difficulty p.36, attendance/level pp.43–44, summary p.56, allocation pp.124–125 and129–130. +- [ACS 2024 household questionnaire](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf): attendance items10a/b, p.12; difficulty items18–20 and their age gates, p.14. This does not claim separate verification of the group-quarters instrument or every interview mode. +- [BLS disability FAQ](https://www.bls.gov/cps/cpsdisability_faq.htm): rotation-period retention and the distinction between the statistical battery and SSA eligibility. + +The minimal readsets are declared once in `current_survey_person_status.py` and +recorded in the result receipt. `SCHL` is excluded because it measures completed +attainment. ASEC `A_ENRLW`/`A_FTPT` are retained as current-cohort literals as well +as cross-checked against the maintained `asec_student_controls` issuer. Calling +that issuer consumes its existing **three-cohort** contract (income years +2022–2024); only the matched 2024 cohort is selected into this projection. The +extra current-cohort `A_HSCOL` and allocation fields are authenticated from the +pinned original member rather than assumed to be covered by the control issuer. +Where the parent retains known `A_HSCOL` or `PEDIS*` cells, the qualifier also +checks their numeric representation against the captured member. Missing carried +cells impose no observation; matching NIU or unnamed numeric codes only establish +representation consistency. They remain unknown under the named observation +domains. Per-field comparison counts are recorded without changing the parent. + +Code columns have nullable integer dtype even when every code is unknown. The +receipt records actual table columns, named observations, difficulty mappings, +allocation vocabularies and the applicable source periods alongside the table +digests. This metadata describes the projection; a stored receipt cannot replace +the retained owner or its validation callback. + +The source modules perform no downloads, no fitting, no tax/benefit calculations, +no cloning, and no graph or dataset publication. A later legal or annual bridge +must be a separate declared operation with its assumptions and validation. + +## Validation scope + +The committed tests use invented literals and privately pinned small source +fixtures. They cover universe boundaries, partial knownness, named allocation +transitions, enrollment routing, exact keys/ages, copied owners and mutation +seals. This patch was prepared under a source-only boundary: static checks passed, +but these new tests have **not yet been executed**. Bounded runtime acceptance and +any integration into the country graph remain separate work. diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md new file mode 100644 index 000000000..e01ab94ee --- /dev/null +++ b/docs/us-uk-release-path.md @@ -0,0 +1,379 @@ +# US and UK release path + +Planning snapshot, 14 September 2026. This is a release work plan, not a +certification of either country. Component tests, historical candidates and +current-candidate acceptance are distinct evidence. + +## Products and common construction + +The US product must support national and congressional-district analysis. The +UK product must support national, constituency and local-authority analysis. +Each country should have one fully constructed population from which full and +smaller analysis files are derived. Record the survey, monetary, policy and +boundary reference periods explicitly; do not infer them from a filename. + +After constructing and harmonizing the complete multispine, including its +initial clones, assign one atomic area to each resulting household and derive +larger geographies from versioned mappings. Distinct clones may receive different +areas, constrained by observed source geography and keyed by stable clone +identity. Subsequent enrichment retains the assigned location. Once construction +and enrichment are complete, +calibration changes household weights; scope selection and pruning retain the +original values, missingness and relationships for every retained household. + +For a shared set of weights and consistent measurement definitions, additive +local totals must reconcile to national totals. Separately refitted compact +views need declared approximation tolerances and their own diagnostics. A +common source population alone does not guarantee identical estimates. + +## Release sequence + +| Stage | US work | UK work | Exit evidence | +| --- | --- | --- | --- | +| Source and period coverage | ACS and ASEC survey multispine; PUF tax detail; SCF and other required auxiliary sources; declared aging | Raw FRS 2024–25; SPI/HMRC, WAS, LCFS/ETB and other required stages; declared uprating | Source register, licensed-input boundaries, source definitions, missingness, units and periods; every applicable input has a producer | +| Complete population | Harmonized relationships and tax/benefit units; geography; financial and PUF enrichment; remaining benefit, disability, housing and asset inputs | Integrated FRS stages and current relationship/claimant definitions; income, wealth, consumption and benefit inputs | Small real end-to-end build; source-channel coverage; plausible distributions and joints; no silent engine-default substitutions | +| Geographic support | Block assignment after initial multispine cloning; versioned county/PUMA/CD derivation; enough household types in each district | Shared atomic-area assignment after full spine cloning, and derivation for each nation; adequate constituency/LA support | Observed geography respected; distinct stable clone draws; subsequent location invariance; support and fit measured at advertised levels | +| Model and calibration | Locked US model; national/state/CD target matrix from identified facts | Locked UK model and companion input changes; national/local matrix with corrected UC and other target definitions | Baseline engine evaluation, calibrated population, target-level errors, support/weight diagnostics and explicit outstanding failures | +| Independent quality | Held-out distributions and geographic cells; tax/benefit totals and representative reforms; comparison with the incumbent | Current release battery, held-out areas and complete incumbent comparison; resolve remaining support and target-fit failures | Candidate-specific reports; explained regressions; thresholds agreed before choosing a candidate | +| Scale and compact views | Increase local build size after smaller checks; derive and refit the intended full/sparse family | Choose a size that meets local support and quality needs; complete compact/exact-count release evidence | Measured time, memory and cache replay; retained-input invariance; full/compact error and performance comparisons | +| Release and consumption | Exact-file dashboard, methods/papers, immutable package and real PolicyEngine loading | Exact-file dashboard and certification; immutable cut, loader/pointer integration and PolicyEngine-UK adoption | Fresh export/readback, reproducible manifest and checksums, consumer calculation smoke test, reviewed release and rollback path | + +## Current position + +The 45-node US combined completion-host test stopped at its 1,800-second CPU +limit without a completion receipt. Its 49- and 51-node status/role variants +remain on hold while we investigate repeated verification costs. The child property-income component has passed all 58 +pure tests and all 14 actual-owner cases on invented inputs. The local host +repair now also passes all 50 small custody tests, including private-support +mutation and permanent-revocation checks. These tests establish component and +boundary behavior; the combined host and a native release file remain unverified. + +The latest native property/tax attempt reached its CPU limit without a final +acceptance receipt. Paired measurements did not justify adopting either the +scanner or digest candidate: each slowed relevant workload classes. The +existing implementations remain. Repeated whole-frame and source verification +are a plausible bottleneck from source inspection. A small experiment using +the actual functions passed seven correctness controls and measured a median +18.36 ms per frame-identity call on 2,048 invented people. Actual call counts +and whole-build attribution remain unmeasured. No whole-build speedup, native +rerun or larger resource budget follows from these measurements. +Native PUF qualification still requires completion of applicable unknown inputs. +Adult routes outside the child model need aggregate source-incidence counts +before choosing whether another producer is necessary. Retirement +candidate accounting has passed 80 controls but does not yet supply fiscal +inputs. The +[dated evidence](../experiments/us-launch-consolidation-20260913.md#14-september-completion-checks) +records the accepted checks and remaining work. + +UK work proceeds independently on the shared same-kind weight update and +metadata/mass-log/column-order interfaces needed by its country graph. The +separate [UK contract proposal](https://github.com/PolicyEngine/microcosm/pull/918) +has passed 101 invented runtime checks. Native FRS integration and an accepted +UK file remain unverified; that proposal establishes no new UK candidate or +release acceptance. + +The US housing participation graph is incorporated in the integration branch, +with 64 focused and nine actual invented full-host tests passing. It preserves +source observations, original household design weights and paired clone draws, +with awards and caps owned by the country model. Native housing outcomes and +the remaining fiscal-input inventory still need verification. See the +[housing integration evidence](../experiments/us-launch-consolidation-20260913.md#other-integrated-source-work). + +| Stage | US demonstrated result | UK demonstrated result | +| --- | --- | --- | +| Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; maintained two-route PUF and subsequent amount/health hosts pass scoped invented controls; native PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | +| Geography | Small native post-clone block assignment and financial extension passed on 11 September; strict required replay passed on 12 September; complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | +| Calibration | Declared national/state/CD measurement and dense grouped calibration stages pass on invented inputs; fully enriched native solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | +| Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | +| Full/compact files | Small prefix replay and 36 supplied-parent full/pruned/local export checks pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | +| Delivery | No accepted new-architecture release or default consumer adoption | Assembly implementation and historical assembled cuts exist; current certification and default promotion pending | + +The 11 September native pilot passed the corrected survey → full initial +support clone → atomic block assignment → financial-enrichment order. At a +1/1000 source sample, its 1,584 source households and 3,464 people become +3,168 households and 6,928 people after cloning. The nineteen-node graph fits +and attaches seven financial fields. Its closed receipt is tied to frozen +source `2ca11c85a`, and reports 5,362 seconds elapsed and 9.31 GB peak RSS. +This supersedes the earlier assignment-before-cloning milestone for ordering; +it does not establish full PUF enrichment or national/CD calibration. + +On 12 September, a strict required replay of that frozen source passed with a +fresh output directory and a read-only retained store. It verified the source +bytes again, reused all nine prefix and nineteen financial-graph nodes, and +reproduced the four non-manifest exports byte for byte. The portable manifest +also matched after excluding only operational timing and cache-hit fields. +The closed run took 3,981 seconds and peaked at 10.99 GB RSS; source, resource, +native-input, owned-file and thread-control checks remained unchanged, with no +unexpected access refusals. See the +[replay record](../experiments/us-financial19-required-replay-20260912.md). +Meanwhile the maintained graph's observer-mutation defect has been fixed and +independently reviewed. The shared graph consolidation merged in +[PR #913](https://github.com/PolicyEngine/microcosm/pull/913) at +`a9cc63e737fd4619a304117dc2ec7dcd86d97901`; the release branch has incorporated +that main revision without changing its existing graph implementation. Those +software changes still need their +own native candidate verification; this replay certifies only its frozen +financial stage, not the newer source or a release. + +The maintained PUF55 host is now adopted. Its 31 small controls and two actual +full-host controls pass on invented inputs: both conditioning routes, 110 fits +and 110 applications, cold execution, required replay, whole-population +preservation and numerical finalizer equality. The full-host run took 742 +seconds and 0.74 GB peak RSS. Those 64 invented donors do not establish the +runtime or quality of the full native donor. A separate current-source native +packet uses a fresh writable store and the actual retained financial run. +See [the PUF host record](../experiments/us-puf55-host-adoption-20260912.md). + +That native packet's second attempt closed with an assertion failure after +about 82 minutes, during its financial-stage acceptance and before any PUF +execution. The source and resource pins remained unchanged and no unexpected +access refusal was reported. Source review identified an invalid harness +assumption: the atomic host first materializes CREATE and ALLOCATION, so those +two nodes legitimately hit when its expanded graph runs in the same store. +Requiring every prefix node to miss rejects that valid execution. The refusal +record does not identify its precise assertion frame, so this diagnosis does +not establish completed native financial acceptance. The corrected v3 harness +passes two actual invented financial cold/replay controls and thirteen bounded +source and diagnostic controls. After independent review of its exact packet, +one fresh native run started with unchanged calculation source, native inputs +and resource limits. On 13 September, the owned process was no longer present +and neither a final receipt nor a PUF manifest existed. Its last recorded +progress was about 5.76 hours wall time, 6.41 hours process CPU and 28.29 GB peak +RSS. The exit code and stopping cause are unavailable; the CPU total exceeding +the configured budget does not establish that cause. This attempt is closed +without verified native PUF output. A successor requires a fresh reviewed +packet; the preceding source, input and failure evidence remain preserved. + +The maintained financial graph now retains an eighth field: the tax-exempt +interest remainder of its existing observed or modeled total. The split remains +a modeling assumption. Twenty-five financial/source controls and three full +invented PUF replay, finalizer and retained-output controls pass; the PUF stage +preserves the original-channel remainder. The historical native financial +receipt above still covers seven fields. See +[the interest correction](../experiments/us-survey-interest-conservation-20260912.md). +A separate reviewed source-hashing optimization preserves exact digest bytes +and passes thirty source tests. Its repeated-string benchmark improvement is +not a measured native build improvement; see +[the scoped benchmark](../experiments/us-source-string-seal-20260912.md). + +The new fiscal measurement stage passes all 66 interpreter/stage checks on the +combined integration, including an actual US-engine dividend calculation. It +uses declared target bindings, household-aligned CSR measurements and explicit +national/state/CD masks. Missing consumed predicates refuse, indicator sums +are numeric, and local calculation helpers participate in cache identity. +Specialized unsupported bindings refuse; this is not full fiscal-registry +coverage. + +The dense grouped calibration stage is now integrated. Independent review of +its numerical adapter found no actionable defects, and all 167 selected +calibration tests passed. The combined integration also passed all 42 tests +for the measurement and dense stages together. The adapter uses the existing +grouped Adam solver, keeps initially zero weights fixed, enforces group bounds +and refuses a result exceeding the declared original row caps. It returns +calibrated weights and schema-8 diagnostics. Numeric bound arrays do not prove +their source: the complete-population host must authenticate the original +sampling references and retained parent before and after use. Native fit, +full-size memory and calibration acceptance remain unestablished. + +The dense fiscal kernel now forwards an optional host-owned target-snapshot +observer to the actual grouped solver. Tiny invented graph tests verify equal +weights, artifacts, receipts and cache identity with observation on/off, plus +silent required replay and final mutation refusal after the sink. The integrated +suite has 473 passes and one explicitly recorded historical thread-configuration +skip; five final calibration-identity checks also pass. Source admission, +production cadence costs and a dashboard consumer remain separate. See +[the fiscal snapshot evidence](../experiments/fiscal-target-snapshot-host-20260912.md). + +Fiscal leaf-policy infrastructure now distinguishes required producers from +explicitly documented proposed assumptions. It passes 91 source/mock and +invented fiscal graph checks. The default remains strict all-producer; +assumption execution and inactive-leaf exemptions remain unsupported until +complete-parent admission and exhaustive dependency proof exist. No US +assumptions or new dataset inputs are enabled. See +[the policy contract and boundary](fiscal-leaf-policy.md). + +Complete-population ancestry remains an explicit requirement. The PUF host now +retains its actual checked result for downstream consumption through +`check_survey_puf55_run`. Constructing a dataclass or decoding its receipt cannot +issue that authority. Consumers must recheck the original run before use and +after their last relevant I/O. The corrected checker passes 25 controls on +actual invented cold/required execution, including extra manifest attachments, +source/store corruption and late output mutation. Independent review approved +the source and mechanical test colocation; 45 cases share one expensive fixture. +See [the retained-output evidence](../experiments/us-puf55-checked-output-20260912.md). + +The fixed post-PUF country host now assembles source-qualified unemployment, +three health-cost amounts and nine current health-coverage inputs. It retains +the checked PUF owner and attaches both fragments to the same receiving +population, preserving existing columns, geography, design anchors and mass +ledger. ASEC observations and unresolved values are retained on both clones; +modeled ACS draws are deliberately shared within each original person's clone +pair. Reporting universes remain explicit; no prior-wage or age-only zero +fallback is introduced. The narrower ACS coverage concepts remain nullable +where the source does not establish equivalence. + +The accepted invented fixture reports 74 present input names out of 161, up +from 61, leaving 87 missing. It executed 26 new enrichment nodes over a validated +245-node prefix, then required all 271 replay hits and complete Frame readback. +Its six logical controls passed across five tests in the main attempt and a +corrected focused parent-revocation follow-up, not one all-green full run. +Independent review approved that scoped evidence. After integration with the +eighth interest field and merged shared graph, 80 cheap source, attachment and +health-fragment controls pass. Declaration checks still give 271 nodes: the +eighth interest field adds an output to the existing financial attachment, +not a new fit or graph node. This integration did not repeat the full combined +host or measure fresh native coverage. See +[the combined integration record](../experiments/us-survey-enrichment-integration-20260912.md) +and [source/model decisions](us-current-survey-amount-successor.md). + +Actual graph tracing also confirms that the legacy full ASEC carry and ACS +gap-fill are not executed by this host. Remaining pension, retirement-account, +property, farm and other-income inputs therefore need source-qualified producers +and appropriate ACS completion. The ACS property-income aggregate should +condition a joint decomposition before the dependent PUF fits; separately adding +rent afterward would not conserve that observed anchor. Source measurement +bridges and overlapping income categories must be resolved explicitly before +claiming reconciliation. The broader ACS retirement total is not merely pension +plus regular-IRA income. + +Two reviewed building blocks for that completion are adopted in the release +branch. The [ACS anchor qualifier](../experiments/us-current-acs-income-anchors-20260912.md) +preserves original INTP/RETP amounts, adjustment factors, original ages and +allocation flags through the retained source owner; its 35 invented-source +controls pass. The [signed reconciliation helper](../experiments/signed-income-reconciliation-20260912.md) +projects joint component draws onto a caller-qualified total while allowing +declared signed components and preserving loss offsets; its 45 numeric controls +pass. An explicit property-income option now integrates the source bridge, +four conditional component models, signed reconciliation and clone attachment +into the existing financial host. Its actual invented 35-node construction and +required replay pass, preserving the full preceding population. The new columns +remain available beside the retained earlier population. A further explicit +three-node extension rebases four interest/dividend tax inputs and checks their +conservation and completeness. Its 38-node cold/required host passes eleven +invented-source controls; unknown inputs remain unknown and an incomplete gate +refuses PUF qualification. Retirement completion and complete PUF/native adoption +remain separate work. The focused PUF +handoff retest passes after correction of an invalid invented source record; +see the [scoped host evidence](../experiments/us-property-financial-host-20260912.md). +The [13 September integration record](../experiments/us-launch-consolidation-20260913.md) +also covers retirement source qualification, UK lineage and the CI repairs. + +The remaining critical sequence is complete PUF/input integration, a small +real full build, model and calibration evaluation, then progressive scale +and release verification. The existing age-development runner is a separate +prefix demonstration, not calibration of the nineteen-node financial result. See +[the US review guide](us-launch-review.md) for scoped acceptance records. + +UK work already includes raw-source enrichment, national/local calibration, +size experiments and release assembly. Its recorded P50/P95b 55,000-household +candidates fail area-support and target-fit checks; the skipped holdout does +not establish an incumbent-comparison pass. Historical dense assembly also +does not satisfy today's release battery. See +[the size evidence](../experiments/355-uk-dataset-size-receipts.md) and +[the current dense release runbook](uk-dense-release-assembly-runbook-762.md). + +The current UK route clones for geographic support and then uses the existing +OA ladder. It has not adopted the new shared atomic geography graph keyed to +stable post-clone household identity. Its existing placement after cloning is +consistent with the user's 10 September ordering correction; the source, +identity and shared-operator migration remain separate work. Maria's +[PR #900](https://github.com/PolicyEngine/microcosm/pull/900) updates Chronicle +household targets and the Northern Ireland lookup; that source work does not +by itself migrate the assignment owner. Anthony's +[PR #896](https://github.com/PolicyEngine/microcosm/pull/896) provides staging +telemetry and synthetic smoke evidence, not a calibrated release. + +The shared UK adapter now has accepted supplied-array tests for England/Wales +Output Areas, Scottish Output Areas and Northern Ireland Data Zones. These +checks cover source conventions, observed-region constraints, stable post-clone +identity and row-order/subset invariance. Ordinary UK imports also preserve the +public export contract without eagerly loading unrelated stages or country +engines. All 52 checks passed with unchanged source bytes and no unexpected +access attempts; see the [scoped acceptance record](../experiments/uk-atomic-area-lazy52-20260910.json). +No native UK area source, whole-spine build or calibration result is certified +by those checks. Maria can use these shared primitives in the existing UK graph +integration rather than starting a second country runtime. + +UK release coverage must be re-established on the chosen integrated build and +locked model. The declared 145-input contract's older candidate evidence must +not certify the current stage set. The +[migration epic](https://github.com/PolicyEngine/microcosm/issues/665) tracks +source and calibration work; +[exact-count release parity](https://github.com/PolicyEngine/microcosm/issues/898) +and [default promotion](https://github.com/PolicyEngine/microcosm/issues/823) +remain separate consumer-release requirements. Existing open issues are leads +for candidate-specific verification, not proof every historical defect remains. + +## 15 September decisions + +- The native property and tax graph ships without the child-property + completion. Persons under 15 carry zero interest and dividend income, as the + incumbent default does: in Build P, 0 of 37,248 persons under 15 have + taxable interest or dividends, against 918 of 8,669 persons aged 15 to 17 + with taxable interest. The completion's transfer from 15-to-17-year-old + donors is an unmeasured assumption that claims no source observation. It + returns only as a count-calibrated policy anchored on an external tabulation + of children's unearned income, such as IRS SOI's Form 8615 statistics. + Turning it on runs the ten completion nodes; the 35 base nodes replay from + the store. A pilot may declare a candidate-scope policy to exercise the + nodes; a release does not. +- Native runs keep the 21,600-second CPU cap. A cap hit closes the run with + its complete CPU-by-code-chain ledger and authorizes no larger rerun. The + cost is removed at its source before any native run repeats: sources are + authenticated once per run with memoised validation, source keys are derived + once per run and re-derived at run end, the ACS record scan runs at C speed, + and column digests hash buffers rather than values. Measured on the + nineteen-node pilot, node execution is 1,143 of 5,362 seconds; the rest is + repeated admission and verification of 3.48 GiB of sources. +- Main must demonstrably build the certified default from raw sources and pass + its preflight and certification gates. The last from-scratch build is Build P + (28 July 2026, policyengine-us 1.764.6); every later default is a + supplied-parent enrichment of that file. The from-scratch build and + certification run on this integration tree before #893 merges and on main as + a standing check afterwards, and a pull request of this size carries that + receipt. The complete native dataset is not the merge gate; it earns default + status separately by beating the incumbent on held-out cells. + +## Team meeting decisions + +This document is the shared US/UK release-path reference. The country review +guides and run receipts provide supporting detail; update the sequence and +ownership here when they change. + +The 10 September coordination sequence is: + +- Maria's #900 source/target reconciliation merged on 10 September at 14:54 UTC; + Anthony's #855 target hierarchy/diagnostics and corresponding dashboard + consumer follow it. Its merge clears that dependency, not all release gates. +- Maria owns registering every existing UK stage as a graph node, stacking on + #893. Changes to UK stage order follow in a separate PR. The shared atomic-area + adapter work supports that integration and does not create a second UK graph. +- Anthony's UK staging #896 can proceed independently. His earlier runtime + proposal #885 is superseded in intent, not another required implementation. +- US construction and verification continue alongside these changes. Complete + graph coverage in both countries is required for the broader advertised + launch; passing tests for individual stages does not establish that coverage. + +The latest UK full rebuild and evaluation were reported as underway at the +meeting. Its new report must replace historical quality evidence only after the +actual candidate and results are available; the approximate four-hour runtime +reported for that build is not a US or release ETA. + +1. Confirm the release years, advertised geographies and full/compact products. +2. Assign one accountable owner to each country candidate and one owner to each + shared dependency: runtime, calibration, diagnostics and release consumers. +3. Review every row above for the latest actual candidate. Name the missing + artifact and next measurable result, rather than reporting code completion + as population acceptance. +4. Agree calibration, holdout, incumbent and representative-reform criteria, + including which differences need explanation and which block release. +5. Set dates for the next complete small candidate, the full quality review and + the production-sized candidate using measured runs rather than linear timing + extrapolation. + +Country data work, target/source reconciliation, independent validation, and +dashboard/consumer integration can run in parallel. Each final dashboard and +release verdict must refer to the same candidate bytes. Cross-sectional release +does not require retirement dynamics, every possible forecast year, or a +complete Chronicle/Orrery redesign. Keep those workstreams active without +making them prerequisites for the first accepted cross-section. diff --git a/docs/us/puf-original-application.md b/docs/us/puf-original-application.md new file mode 100644 index 000000000..208c2b1bd --- /dev/null +++ b/docs/us/puf-original-application.md @@ -0,0 +1,37 @@ +# Original survey-arm PUF application + +`us_runtime.puf55_original_application` declares all 55 actual conditional +applications per selected original-arm route. It reuses the exact maintained +route fits and consumes the source-qualified, typed fixed-value artifacts. +The fitted-model seed remains the clone-one seed. The caller must choose a +separate, explicit `original_application_seed`; the helper invents no offset. +Existing clone-one declarations and v1 decoders are unchanged. + +Every target still draws for every recipient. Its raw draw is retained; the +conditioning output substitutes only cells whose qualified fixed-input mask is +true. Later targets consume that conditioning output. The v2 history binds the +matrix bytes and producer, actual model/conditioning/fixed-input producers, +complete ordered target prefix, raw/conditioning hashes, known counts and the +separate fit/application seeds. + +`merge_puf55_original_conditioning` checks the exact closed 55-target routes and +returns a detached tax-unit table and a descriptive receipt. It does not +unpickle models, issue source authority, attach columns, or certify scientific +coverage. Its caller must authenticate each typed artifact against the genuine +compiled graph and retained run, and requalify the original financial/fixed +inputs at the maintained pre/post I/O fences. Merely constructing an +`ArtifactValue` or a matching digest does not satisfy those duties. + +Eight shared financial inputs are qualified survey-preserving values; they can +include earlier modeled or split values. Optional pension/IRA/rental/farm fixed +inputs retain the explicit development rules and unresolved source statuses +from `puf55_survey_observed`. None becomes a claim of observed taxability. +An aggregate mask is true only when all members are qualified. A false mask +never permits overwriting known people in a mixed unit. + +Population attachment is deliberately separate. This slice does not implement +mixed-unit allocation, pruning, tail caps, tuition allocation, QBI finalization, +a new host entry, or release-science acceptance. Its component proof runs three +real invented-data fits/applications and required replay. Separate synthetic +codec-only 55-target envelopes exercise roster and merger refusal checks; +those envelopes are not evidence of 55 fitted models or a native owner. diff --git a/docs/validation-cost-and-borrow-boundaries.md b/docs/validation-cost-and-borrow-boundaries.md new file mode 100644 index 000000000..ff809a960 --- /dev/null +++ b/docs/validation-cost-and-borrow-boundaries.md @@ -0,0 +1,67 @@ +# Validation cost and borrow boundaries + +The US graph must detect changed sources, substituted artifacts and changes to retained Populations without repeating the same expensive work at every nested call. This note records the current decision and the proposals still under review. It does not change dataset release requirements. + +## Adopted: reuse source compilation + +The ACS native coverage owner keeps a bounded process-local cache of compiled code. Its key includes exact source bytes, filename, mode, flags, inheritance setting, effective optimization level and compiler identity. Every source read and loaded-function, global, alias and closure check remains fresh. Cached code is neither source authority nor a portable receipt. + +The change passes 25 focused controls and the original invented financial fixture, including cold execution, required replay and final checks. One paired profile took 216.82 wall seconds versus 308.26 seconds before the change, a 29.66% reduction. CPU time fell from 303.90 to 214.69 seconds. These instrumented observations establish an improvement for that fixture; they do not establish native-data performance. See [the experiment](../experiments/us-acs-compilation-cache-adoption-20260910.json) for exact source, test and verification identities. + +The native pilot already in progress retains its original frozen source snapshot. + +## Next: remove duplicate work within a check + +A source review confirmed nine direct Frame storage traversals in a healthy `verify_acs_native_coverage` call: three calls to `_verify_frame`, each performing three storage hashes. Some repetitions occur across supplemental axis and resolved-weight checks, so collapsing all nine into three is not automatically equivalent under the existing mutation checks. + +The next candidate will share the first storage calculation with the supplemental fingerprint calculation, while preserving the independent final prepared-Frame storage pass and all three outer verification points. That would reduce nine direct traversals to six. It needs a regression that mutates a cell during the supplemental check, unchanged receipt and fingerprint comparisons, and a paired measurement before adoption. + +## Keep separate: code, files and retained objects + +Three different facts currently appear together in nested validation: + +- Loaded-code identity: the interpreter's functions, code, globals, aliases and closures. +- Agreement with files: source bytes, archive snapshots and stored graph artifacts still match their recorded identities. +- Retained-object identity: the actual Frame, Population, weights and ancestry still match their issued state. + +An independent Fable 5.1 review recommended making these checks explicit and performing expensive file verification primarily at public entry and emission boundaries. Moving file checks changes when an altered archive is detected, so it needed a documented entry contract and tests at the boundaries before implementation. + +**Adopted 2026-09-16, as a scoped epoch rather than a boundary rule.** `survey_population_preparation.verification_epoch()` opens a verification epoch around a graph run. Outside one, nothing changes: every borrow re-authenticates in full. Inside one, each borrow still pays the cheap tier — loaded-code identity through `_encode(_producer())`, the attached owner payloads and the live authority — and the expensive file and retained-object tier is skipped only while a signature is unchanged: the stat identity of every path the four foreign verifications and `_source_files` re-read, including `_file_stats` over the whole source roster, the identity and length of every borrowed payload, and a read-free witness of each live Frame's storage. The roster's stat identities are read on every borrow, but into that signature rather than as a comparison of their own: a signature that moved is a memo miss, not a refusal, so the complete validation the miss runs is what refuses, with the code it raises today. Leaving the epoch, at every nesting level, re-validates every memoised capsule in full with the memo bypassed, so a change no signature can see still refuses before the run returns anything; a close whose signature moved while it was validating records no memo answer, and at the outermost close, where no borrow follows, it pays that validation itself. The contract, the residuals it accepts and the argument for each are in [verifying native sources once per run](us-native-verification-once.md); the boundary tests are `packages/microcosm-build/tests/test_us_native_verify_once_epoch.py`. + +**Adopted 2026-09-19: cache immutable declaration summaries.** The ACS owner now +retains ordered tuples of top-level definition names and imported aliases, keyed +by exact source bytes. It does not retain mutable ASTs. Every check still reads +the current source and compares its declarations with the current loaded objects; +the existing loaded-code checks also remain. Source changes therefore select a +different entry, and a warm entry cannot hide a changed live alias. + +The cache accepts only the original parser function, code, compiler, exact typed +defaults and AST flag values. Altered parser context bypasses it. Up to 128 source +keys of at most 128 KiB bound retained source bytes to 16 MiB; this is not a total +memory bound. Failed parsing is not cached. Like the existing code check, this +operates within a trusted Python process, not against arbitrary private-state +tampering. + +The candidate passes 210 focused source, coverage, membership and row-ceiling +tests, including warm-cache parser drift, changed source and alias refusals. +Eight repeated code-only producer checks took 2.033 CPU seconds with declaration +caching disabled and 0.385 seconds with it enabled. Their complete producer +documents matched within this candidate. This does not establish a native-run +speedup or equality with producer identities from earlier source versions. The +active native retention experiment keeps its separate frozen source. See +[the declaration-cache experiment](../experiments/us-acs-declaration-cache-adoption-20260919.json) +for reviewed source and verification identities. + +## Composition review + +The same review raised kernel-registry sharing, registration order and recipient ordering. These require owner-level checks before being treated as bugs. For example, each financial run currently constructs a fresh survey prefix internally; it does not accept an existing prefix for a second extension. The PUF raw-route merger separately derives and verifies the complete receiving tax-unit axis before restoring row order. Neither observation alone proves every composition boundary correct. + +The budget returned-view correction is now adopted. Detached documents and grouped bounds are constructed after the last borrowed I/O, then checked against the issued payload and retained owners. Four targeted controls passed, including nine callback branches; the post-run verification rechecked 501 source/owned files, 5,983 model-code files and 12 resources. The earlier eight financial-successor controls remain evidence for their original source revision. These are separate component runs, not twelve tests of one integrated revision. See [the correction experiment](../experiments/us-budget-detached-view-correction-20260910.json). + +A second Fable source review found no confirmed correctness or ownership bug in the public PUF recipient interface. It recommended explicit cold-cache reuse assertions, exact comparisons with the original issuers, and clearer documentation of the original-source reads required by the recipient kernels. The frozen two-positive pilot remains unchanged; those additions belong to subsequent controls. + +The immediate implementation priorities remain actual public PUF recipient and graph checks, authenticated donor/model binding, one whole-cohort PUF finalization and attachment, then native calibration and release verification. Performance work proceeds alongside those tasks. + +## Review scope + +Fable 5.1 completed a read-only source review on September 10, 2026, using the saved Hivesight login. Its requested scratch-cache read was denied and that limitation was disclosed in its result. It supplied architectural advice, not runtime approval or a release verdict. The recommendations above are distinguished from independently verified source findings and adopted changes. diff --git a/experiments/893-reconciliation-amendments-19-20-20260912.md b/experiments/893-reconciliation-amendments-19-20-20260912.md new file mode 100644 index 000000000..f1627eb2b --- /dev/null +++ b/experiments/893-reconciliation-amendments-19-20-20260912.md @@ -0,0 +1,722 @@ +# #893 reconciliation to main's amended graph interface — lane report + +> **12 September Codex continuation:** this is the historical layer proposal +> at `80fef2217`, not an approved extraction plan. The final battery +> `bt0nx8fcc` was killed during its first pytest group: CI-group verification +> and coverage check exited zero, but no final consumer result was produced. +> Independent review found that G4 exposes mutable admitted populations to +> observers and that the layer order omits hard packaged-resource dependencies. +> Fix and review both before extracting or landing layers. PR912 is also held +> for fixes from Astra's executed review; the branch-head assumption below is +> superseded until its corrected head lands. +> +> Resource repair `a4b6fe88c` restores all three omitted ASEC JSON contracts +> from their exact historical blobs, matching unchanged loader pins. Eight +> targeted tests and wheel-content verification pass. These three resources, +> `acs_2024_housing_universe.json`, and current-money domains/consumers/price-basis +> resources must accompany their first PR-1 loaders or an earlier prerequisite; +> Python import analysis alone cannot establish extraction order. No engine +> defaults were admitted and no source/engine identity was refreshed. + +> **Later 12 September correction:** `6f66fa545` closes the demonstrated G4 +> mutation defect by handing observers detached snapshots, including nested +> table values, metadata, schema, weights and ledgers. Independent review is +> clean; 524 graph tests, five focused cases and all four existing synthetic +> consumers passed, including financial and calibration teardown checks. An +> earlier source-edit-during-test failure remains preserved and is excluded. +> Entry 7.4 below is historical: the current observer need not be trusted to +> leave its snapshot unmodified. Retaining snapshots still costs memory. +> +> `cb27bdb0b` makes the offline explanation display executor-owned unreached +> and gate-exception states independently from cache hits. Sixteen graph +> explanation/actual-gate cases and two free-form diagnostic controls passed. +> Its new `explain → availability` import is recorded in the implementation +> inventory; availability was already bound as a whole module in all ten +> using scopes. No scope, dependency, resource, or unbound-use exemption was +> added. All eight prepared-resource/full-manifest tests passed afterward. +> +> Corrected PR912 head `5e33d6971` is undergoing exact-head CI and Fable review. +> Graph-only extraction is being prepared against that revision, incorporating +> G1–G4 and numbered charter entries. Neither that extraction nor the remaining +> country layers are declared accepted by this historical report. + +> **Subsequent 12 September integration checks:** corrected PR912 head +> `5e33d6971` received Fable approval; its merge remains conditional on CI. +> The local integration merged that exact revision in `9e61d0d04` and +> recomputed the combined source/seed identities. All 734 graph and fitting +> tests passed. The six-file spec/build sweep collected 248 tests: 247 passed +> and one failed because its invented PUMA fixture still used schema 1 and +> omitted the required joint tract/CD population support. The fixture now +> supplies consistent schema-2 support and explicit invented provenance; that +> single test passed when rerun. The original failed sweep and intermediate +> incomplete fixture repair are preserved. This is separate-run evidence, +> not a claim that the complete 248-test sweep was rerun successfully. +> +> Shared graph consolidation is now public as PR913, temporarily based on +> PR912, with independent Fable review in progress. Its runtime matches this +> integration's graph implementation. Native strict replay, PUF55 host +> acceptance and complete national/CD release acceptance remain separate +> work and are not established by these software checks. + + +Date: 2026-09-12. Worktree `~/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910`, +branch `microcosm-us-launch-verified-lanes-20260910` (PR #893's branch is +`microcosm-us-launch-integration-20260909`). Started at `069d5ed9a`. No push, no +new branch, no stash, `uv.lock` untouched. Every commit below is on the +checked-out branch. + +## 1. Outcome + +- `origin/main` (`0c3f4f651`, amendment 19) and + `origin/amend-keyed-seed-and-uniform-draws` (`23ba24770`, amendment 20 / PR + #912) are merged. **#912 was merged from its branch head**, so once it lands + the dispatcher should re-run `git merge origin/main`; for the graph and fit + packages, `docs/graph-interface.lock` and `docs/graph-acceptance.md` that is + expected to be a no-op (this tree already carries `23ba24770`'s bytes for all + of them, verified with `git diff --quiet 23ba24770 HEAD -- `). +- Every frozen or amended file equals `23ba24770` byte for byte: `decl.py`, + `kernel.py`, `keys.py`, `serialize.py`, `view.py`, `artifact_edges.py`, + `randomness.py`, `population.py`, `fit/qrf.py`, the lock, the charter, + `test_graph_kernel_contract.py`, and every `test_acceptance_*` file. The + interface-lock test passes (it is part of the 518-test graph run below). +- Four not-yet-amended graph pieces the US runtime consumes were re-applied + as separate named commits on top of main's files; four pieces with no + consumer were dropped (section 3). +- The five graph tests the brief named are green; the whole graph package is + green (518 passed); the fit package is green (191 passed); the spec-engine + check, `ci_test_groups --verify` and ruff are clean. Build-side results are + in section 5. +- The layer map (section 6) partitions the branch's remaining 407-file diff + against `origin/main` into the eleven staged layers, the later closure and + post-staging work, and the graph pieces, with per-layer file lists, test + files, import dependencies, and a proposed landing order. Section 7 has a + draft charter amendment entry per kept graph piece. Section 8 lists what + needs Max. + +Lane commits (newest first): + +``` +07566eb17 Re-pin the US implementation inventory for the reconciled graph package +c704a1be6 Pin that unreached propagates through a structural node and its version +2e1857a4b Bring the #893 reconciliation journal up to the piece commits +a16eeacf8 Re-apply the private population observer on run_graph +d1019762b Re-apply executor execution states: gate exceptions leave their consumers unreached +db1b7821a Re-apply complete Frame metadata storage and non-finite JSON refusal in the store +752ab840f Re-apply the raw-byte source codec on main's graph package +cff8fbf32 Re-pin the calibrate and simulate H1 parity cases the branch's kernels move +dc621c14c Re-pin the seed protocol and compiled seed map digests on the merged tree +051357909 Merge origin/amend-keyed-seed-and-uniform-draws (23ba24770, amendment 20) +3010b7788 Merge origin/main (0c3f4f651, amendment 19) into the #893 integration branch +3ad1b1aae Open the #893 reconciliation lane journal +``` + +## 2. The two merges and the per-file resolution + +Baseline before any change (`uv run --no-sync pytest packages/microcosm-graph/tests`): +5 failed, 362 passed, exit 1 — exactly the five the brief names +(`test_acceptance_b_ownership::test_b2_executor_enforces_ownership`, +`test_acceptance_h_parity::test_h1_kernel_parity`, +`test_graph_executor::test_fit_qrf_tolerance_source_hash_pin_is_current`, +`test_graph_kernel_contract::test_context_numerics_default_empty_and_carry_scopes`, +`test_graph_serialize::test_generated_parity_graphs_bind_real_kernels_and_direct_bytes`). + +### Merge 1: `git merge origin/main` (commit `3010b7788`) + +Git reported 8 content conflicts plus one in `PROGRESS.md`; `store.py`, +`__init__.py` and `PROGRESS.md` auto-merged. Resolution: + +| Path | Resolution | +| --- | --- | +| `packages/microcosm-graph/src/microcosm/graph/{artifact_edges,decl,executor,kernel,keys,manifest,serialize,view}.py` (conflicts) | main's bytes (`git checkout origin/main -- packages/microcosm-graph`) | +| `packages/microcosm-graph/src/microcosm/graph/{store,__init__}.py` (auto-merged) | main's bytes, same checkout; the branch's hunks return as pieces A and B | +| `packages/microcosm-graph/src/microcosm/graph/{attachments,availability,schema}.py` (branch-only) | removed in the merge (`git rm`); `availability.py` returns as piece C, the other two are dropped | +| `packages/microcosm-graph/src/microcosm/graph/randomness.py` (branch-only until amendment 20) | kept as the branch had it; replaced by `23ba24770`'s in merge 2 | +| `packages/microcosm-graph/tests/*` | main's bytes; the branch's copies were strict subsets of main's (numstat showed only removals), so this also dropped nothing of the branch's; `test_frame_metadata_store.py` and `test_lazy_snapshot_metadata_integration.py` (branch-only) removed here, the first returns with piece B, the second is dropped | +| `packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json` | main's (consistent with fit/qrf.py being main's in this merge) | +| `packages/microcosm-fit/src/microcosm/fit/qrf.py` (auto-merged to the branch's) | main's bytes; the branch's `predict_from_uniforms` returns with amendment 20 in merge 2 (the branch's qrf.py was byte-identical to `23ba24770`'s) | +| `packages/microcosm-fit/src/microcosm/fit/{_graph_legacy_apply,_graph_legacy_qrf,graph_legacy_apply,graph_legacy_apply_matrix,graph_legacy_qrf,graph_legacy_train,model_input,qrf_target}.py` (branch-only) | kept: not amended content, consumed by the US runtime (layer 01) | +| `docs/graph-interface.lock`, `docs/graph-acceptance.md` | main's bytes | +| `PROGRESS.md` | both sides kept: this lane's section, then main's Amendment 19 and #907 sections, then the branch's history | +| everything else | ordinary auto-merge | + +### Merge 2: `git merge 23ba24770` (commit `051357909`) + +Six conflicts: + +| Path | Resolution | +| --- | --- | +| `packages/microcosm-graph/src/microcosm/graph/randomness.py` (add/add) | `23ba24770`'s (adds the signed-zero normalisation the branch lacked) | +| `packages/microcosm-fit/tests/test_qrf_stateless.py` (add/add) | `23ba24770`'s (182 lines; the branch's 159-line copy was a strict subset) | +| `docs/evidence/spec-engine/us-f0-coverage.json` | `23ba24770`'s, then recomputed (section 4) | +| `packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py` (seed digests) | `23ba24770`'s, then recomputed (section 4) | +| `packages/microcosm-build/tests/test_spec_engine_country_bundles.py` (AM/BE/UK digests) | `23ba24770`'s; see section 5 for the merged-tree check | +| `packages/microcosm-build/tests/test_us_multispine_pool_tool.py` (pool tool `spec_sha256`) | `23ba24770`'s; see section 5 | + +After merge 2 the graph package, fit sources, lock and charter were +byte-identical to `23ba24770` except the eight branch-only fit modules above; +`docs/graph-interface.lock` matched `shasum -a 256` of the two frozen files. + +## 3. The not-yet-amended graph pieces + +Consumers were found by an AST scan of every `.py` under +`packages/microcosm-{build,calibrate,fit,frame,data}` and `tools/` for names +imported from `microcosm.graph.*` that `23ba24770`'s graph package does not +define, plus greps for the keyword-level uses (`_population_observer=`, +`population_retention=`, `emit_validation_artifact=`). + +### Kept (each re-applied on main's file as its own commit, with tests) + +| Piece | Commit | Consumers outside the graph package | Graph-level tests | +| --- | --- | --- | --- | +| A. `codecs.py` raw-byte mode: `SourceBytesCodec`, `register_bytes`/`load_bytes`, `raw-bytes-v1`, `RAW_BYTES_MAX_BYTES`, `load_source_bytes` (+ two `__init__` exports) | `752ab840f` | 13 modules: `graph_atomic_geography`, `graph_geography`, `graph_sources`, `graph_atomic_survey_population`, `graph_puf55_canonical_donor`, `puf_raw_source`, `puf_monetary_source`, `puf_monetary_agi_projection`, `puf_full_source_graph`, `atomic_block_sources`, `atomic_block_api_sources`, `survey_atomic_geography`, `puf55_survey_recipients`; 8 US test files | the branch had none; 4 added to `test_graph_codecs.py` (8 passed) | +| B. `store.py` Frame-metadata storage (`microcosm-graph-frame-v2`, `_encode/_decode_frame_metadata`, `frame_metadata_sha256`, `_put(validate_existing=…)`) and the non-finite JSON decode hooks | `db1b7821a` | `survey_population_replay.same_replayed_frame`, `population_input_coverage`, `graph_full_puf_enrichment` (all call `_encode_frame_metadata`), and every US stage relying on `Frame.metadata` surviving a store round trip | `test_frame_metadata_store.py` (the branch's file, 19 collected tests) | +| C. `availability.py` + executor/manifest execution states: `gate_exception`, `unreached`, `blocked_by`, cache-record schema 3, manifest schema 4, refusal of kernel-authored execution metadata; replaces amendment 19's refusal of a gate declaring a typed output | `d1019762b` | `graph_atomic_survey_clone` (`atomic_geography_nodes(..., emit_validation_artifact=True)` makes the geography gate declare a typed output, which main's executor refuses outright), its consumers `survey_age_calibration`, `graph_current_survey_predictors`, `survey_origin_budget`; `population_input_coverage` (`execution_state` on every producer receipt); build test `test_atomic_geography.py` parametrises the declaration | the branch had none; main's `test_a_gate_kernel_may_not_declare_a_typed_artifact_output` replaced by 6 tests in `test_graph_executor.py` (five in `d1019762b`, the structural-propagation one in `c704a1be6`) | +| D. `run_graph(_population_observer=…)` | `a16eeacf8` | `survey_age_calibration`, `graph_atomic_survey_financial`, `graph_survey_population`, `graph_atomic_survey_population`; 8 US test files | the branch had none; 1 added | + +What was deliberately not re-applied inside those files: the branch's +docstring deletions on the amended surface; its `_opaque_artifact_key` +inline hash (main's `graph_keys.opaque_artifact_key` is the same +derivation); its `KernelContext` argument order (frozen file); its eager +artifact payload reads on cache hits (main authenticates edges without +reading, `9018c4420`); its `StoreMiss`-on-hit for a missing declared artifact +(main's `5c4a8efde` decision, `StoreCorrupt`); the removal of main's #907 +datetime64/timedelta64 object-leaf refusal in `store.py`; and main's +`_validate_typed_ancestry` gate-ancestry shape check and the +`NodeRejectedError`→`ValueError` conversion (`402d9a631`, `7d45c32e5`), which +the branch's older copy lacked. + +### Dropped (no consumer left) + +| Piece | Why dropped | +| --- | --- | +| `attachments.py`, `_PopulationRetention`, `_LazyPopulations`, `run_graph(population_retention="lazy")`, `PopulationView.__slots__ = ("__weakref__",)` (staged as layer 08 GRAPH-ATTACHMENT-METADATA) | no call site outside the graph package passes `population_retention`; its only consumer was the branch's own `test_lazy_snapshot_metadata_integration.py`, dropped with it. It is a memory optimisation for long graphs (the native pilot's 11.5 GB peak is in the PR body); if it is wanted it is a self-contained later PR on top of piece B, which it imports. | +| `schema.py` (`graph_schema`, 316 lines) | imported by nothing in the repository (the `graph_schema_version` hits in the spec bundle are unrelated). | +| `keys.py` `_stream_file` chunked source hashing | no consumer and no test; it changes no identity (same digest sequence, it only bounds memory and refuses a file that changes while hashed). Main's `keys.py` is kept byte-identical. Candidate for a small standalone PR if large-source memory matters. | +| the `_write_node` per-coordinate memory refactor and the `del series` | no consumer; behaviour-preserving; main's amended `_write_node` kept. | + +## 4. Identities re-pinned on the merged tree, and why + +- **Seed protocol and compiled seed map digests** (`inventory_coverage.py` + `EXPECTED_HASHES`, `docs/evidence/spec-engine/us-f0-coverage.json`; + commit `dc621c14c`). Both hash the exact source bytes of the attested kernel + modules in `spec_engine/seeds.py` (no `microcosm.graph` module is on that + list, so the graph pieces do not move them). The merged tree carries the + branch's `acs_transfer` and `housing_inputs` changes and, through amendment + 20, the same `fit.qrf` bytes the branch already carried, so the actual + digests are the values the branch pinned in `1734b9e90` + (`a775cccd…` / `c15bff65…`), not `23ba24770`'s (computed on a tree without + the two US kernel changes). Computed directly (`spec.seed_protocol.implementation_sha256`, + `sha256_json(compiled.seed_stream_map.to_wire())`), then the report was + regenerated with `tools/spec_engine_coverage.py` and `--check` exits 0. +- **H1 parity pins for `calibrate` and `simulate`** (commit `cff8fbf32`). + Six graph tests were red on the merged tree, all on these two cases. + `CalibrateKernel.implementation_hash` binds `microcosm.calibrate`'s solve and + diagnostics modules, which the branch's grouped-bound solver path changes + (layer 05); `SimulateRulesKernel.implementation_hash` binds + `microcosm.frame.bundle`, to which the branch adds `Frame.__reduce__` (layer + 01). Re-recorded with `tools/graph_parity_repin.py calibrate` and + `… simulate`, which keep every pinned platform (both cases pin only + `arm64/darwin/py3.14`) and leave `direct.csv` untouched. The `fit.qrf` pin + came from `23ba24770` and needed no move. +- **The US implementation inventory** (`us_runtime/graph_implementation_inventory.json`; + commit `07566eb17`). This reviewed dependency fence is checked by every US + source stage before it runs (`graph_implementation.implementation_manifest`), + and its refusal surfaces as `PREPARATION_ISSUANCE_REFUSED` (the preparation + wraps every exception; the true cause was recovered from the exception's + `__context__`). Three entries no longer held: the `microcosm.graph` roster + listed the dropped `attachments.py` and `schema.py` (39 survey tests red on + that alone); the contracts for `executor.py`, `manifest.py`, `keys.py` and + `population.py` reflected the branch's versions rather than main's plus the + kept pieces, and `attachments.py` was a listed stage module in ten stages; + and `congressional_district_vintage.py`'s contract predated main's + `05ff48f05` (merged into the branch on 2026-09-11), which added the + `microcosm.calibrate.geography_constants` import — an inherited drift the + branch already carried at `069d5ed9a`, verified by recomputing the + pre-merge tree's contracts against its own inventory. Recomputed with the + module's own `_dependency_contract` / `_covered_imports`; the import + classification set is the exact union again. Seven of the ten stages now + build their manifest; three cannot (section 8.5). +- **Country-bundle digests and the pool tool's `spec_sha256`**: resolved to + `23ba24770`'s values in merge 2; their merged-tree check is in section 5. + +## 5. Commands, exit codes, counts + +All from the worktree root with the locked engine environment +(`uv sync --all-packages --locked --extra us --extra uk` → exit 0 before any +change). Tests ran with `uv run --no-sync pytest … -p no:cacheprovider`. + +| Stage | Command | Result | +| --- | --- | --- | +| baseline (`069d5ed9a`) | `pytest packages/microcosm-graph/tests` | 5 failed, 362 passed, exit 1 | +| baseline | `python tools/ci_test_groups.py --verify` | exit 0 | +| after both merges (`051357909`) | `pytest packages/microcosm-graph/tests` | 6 failed, 485 passed, exit 1 (calibrate/simulate parity pins; section 4) | +| after both merges | `pytest packages/microcosm-fit/tests` | 191 passed, exit 0 | +| after both merges | `python tools/spec_engine_coverage.py --check` | exit 1 (`seed protocol content digest differs`, `compiled seed map digest differs`) | +| after both merges | `python tools/ci_test_groups.py --verify` | `verification=ok`, exit 0 | +| after both merges | `ruff check .` | All checks passed, exit 0 | +| seed re-pin (`dc621c14c`) | `python tools/spec_engine_coverage.py` then `--check` | exit 0 / exit 0 | +| parity re-pin (`cff8fbf32`) | `python tools/graph_parity_repin.py calibrate`; `… simulate` | exit 0 / exit 0 | +| parity re-pin | `pytest test_acceptance_h_parity.py test_graph_parity_pins.py test_graph_parity_repin.py test_graph_serialize.py` | 27 passed, exit 0 | +| piece A (`752ab840f`) | `pytest packages/microcosm-graph/tests/test_graph_codecs.py` | 8 passed, exit 0 | +| piece B (`db1b7821a`) | `pytest test_frame_metadata_store.py test_graph_store.py test_graph_population.py test_graph_executor.py test_graph_manifest.py` | 274 passed, exit 0 | +| piece C (`d1019762b`) | `pytest packages/microcosm-graph/tests` (whole package, includes the interface-lock test) | **518 passed, exit 0** | +| piece C | `ruff check .`; `ruff format --check packages/microcosm-graph` | exit 0 / exit 0 | +| piece D (`a16eeacf8`) | `pytest test_graph_executor.py test_acceptance_b_ownership.py test_acceptance_f_gates.py` | 94 passed, exit 0 | +| structural test (`c704a1be6`) | `pytest packages/microcosm-graph/tests/test_graph_executor.py` | 82 passed, exit 0 | +| inventory re-pin (`07566eb17`) | `pytest test_us_survey_origin_budget.py::test_actual_initial_budget_and_zero test_us_survey_age_calibration_run.py::test_actual_seven_node_calibration_preserves_full_population` (two of the 39 formerly refused) | 2 passed, exit 0 | +| final tree | `python tools/ci_test_groups.py --verify` | exit 0 | +| final tree | `python tools/spec_engine_coverage.py --check` | exit 0 | + +Build-side and shared consumers, final tree (`07566eb17`). A first battery at `a16eeacf8` found 39 survey tests refused (`PREPARATION_ISSUANCE_REFUSED`) — all traced to the implementation inventory above, re-pinned in `07566eb17` — and one non-existent file name in my own raw-bytes group list (`test_us_puf_raw_source.py`, removed); the battery was then rerun in full: + +_(Historical draft marker resolved: task `bt0nx8fcc` was killed during availability consumers. No final battery counts or exit code exist; later groups did not start.)_ + +## 6. Layer map: landing #893 as sequential PRs + +Basis: `git diff --name-status origin/main HEAD` = 407 files (340 added, 67 +modified, 0 deleted), each attributed to the first branch commit that touched +it (`git log --name-only --no-merges 3094bfe84..HEAD`), with the eleven +`Stage integration layer NN` commits mapped to their layer names, the later +staging commits to layers 12–14, the sixty post-staging commits of 10–12 +September grouped by subject (P1–P6), and the graph pieces of section 3 as +their own layer. Import dependencies come from an AST scan of every changed +`.py` file resolved against that attribution (`microcosm.*` imports only). +The scripts and their JSON output are in this lane's scratchpad +(`layers.py`, `layers2.py`, `layers.json`, `per-file-deps.json`). + +Two corrections to the staged layer names as they sit in git history: + +- Layer 08 GRAPH-ATTACHMENT-METADATA is empty in the map: its only file, + `attachments.py`, is dropped (section 3). +- Layer 09 F-JOINT-GEOGRAPHY-GATE has no file of its own: its 27-line change + lives in `graph_geography.py`, which layer 01 created, so it lands with + layer 01. +- Nineteen files differ from `origin/main` only because `origin/main` does not + yet carry amendment 20 (`kernel.py`, `randomness.py`, `fit/qrf.py`, + `test_graph_kernel_contract.py`, `test_graph_parity_*`, + `test_graph_randomness.py`, `tools/graph_parity_repin.py`, the three + `pins.json`, the lock, the charter, the amendment-20 journal/receipts/ + changelog fragment) or were re-pinned by this lane. They vanish from the diff + when #912 lands and are not #893 content; they are listed under + "already on main's way" below and belong to no PR. + +### Dependency facts the order rests on + +Three kinds of link were measured, strongest first: + +1. **Hard**: a file imports a module that does not exist on `origin/main` + (a file the branch adds). These force order. +2. **Name-hard**: a file imports, from a module main already has, a *name* + that main's version of that module does not define (checked by parsing + `git show origin/main:`). Only three survive after discarding + submodule imports that merely look like names: `GroupedUpperBounds` + (layer 03's two-line re-export in `calibrate/__init__.py` of layer 01's + `group_bounds.py`; imported by layer 12's and 13's tests), + `decode_us_puma_ladder` (layer 06's `puma_ladder.py`; imported by layer + 13's `test_us_geography_integration.py`), and the graph pieces' names + (`load_source_bytes`, `load_raw_bytes`, `RAW_BYTES_MAX_BYTES`, + `_encode_frame_metadata`, `execution_state`). +3. **Soft**: a file imports a module main already has that the branch also + modifies (`frame_checkpoint`, `asec_checkpoint`, `operator_boundary`, + `us_runtime/__init__`, `calibrate/__init__`, `solve.py`, `bundle.py`, + `variable_labels.py`, and amendment 20's `qrf.py`/`kernel.py`). These do + not force order by themselves; each PR's own test run is what proves the + modified behaviour is or is not needed. + +Hard links between layers (source and tests): + +| Layer | hard on | of which tests only | +| --- | --- | --- | +| 01 SAFE-ADDITIVE | nothing | — | +| 03 ACCEPTED-SHARED-RESTORE | 01 (`group_bounds`, `reported_coverage_source`, `_person_signal_summary`, `operator_column_contracts`, `table_identity`), P3 (`asec_raw_stage_v4`) | P3 link is `test_us_asec_checkpoint.py` only | +| 04, 06 | nothing hard (soft on 01/03) | — | +| 05 SOLVE-MERGE-PROPOSAL | 01 (`group_bounds`) | — | +| 07 F-CATALOGUE-OPTIMIZATION | 01 (`survey_population_preparation`) | all three files are tests | +| 10 SOURCE-CLOSURE | G1 (soft) | — | +| 11 PLACEMENT-ADDITIONS | 01 (the eight fit modules, `full_puf_enrichment`, `graph_sources`, `survey_population_replay`), G1, G2 | — | +| 12 ORDINARY-CLOSURE | 01 (fit modules), 03 (`GroupedUpperBounds`) | all files are tests/fixtures | +| 13 INTEGRATION-REGRESSIONS | 01 (`graph_geography`), 03 (`GroupedUpperBounds`), 06 (`decode_us_puma_ladder`) | all three are tests | +| P1 POST-CLONE-ATOMIC-GEOGRAPHY | 01 (`randomness` = amendment 20, fit modules, `graph_combined_clone`, `graph_sources`, `graph_survey_population`, `survey_*`), G1, G3 (the gate's declared artifact), P2 | P2 link is two tests (`test_us_current_survey_predictor_demographics.py`, `test_us_survey_origin_budget_predictor_compatibility.py`) | +| P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR | 01 (`model_input`, `_graph_legacy_qrf`, `graph_legacy_*`, and eleven test-side modules), 11 (`graph_full_puf_enrichment`, two tests), P1 (`graph_atomic_survey_financial`, one test), G1 | 11 and P1 links are tests only | +| P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET | 01 (`reported_coverage_source` from `asec_raw_stage_v4`; six test-side modules), G2, G3 (`population_input_coverage`), P2 (`asec_demographic_source`) | P2 link is `test_us_asec_demographic_source.py` only | +| P4 | nothing hard (soft on 03) | — | +| P5 | nothing hard (soft on 01/03/04/06) | — | +| P6 UK-ADAPTER-DOCS-AND-RECORDS | P1 (`atomic_geography`) | — | + +Two cycles exist and both are test-only: P1↔P2 (three tests) and 03↔P3 +(one test). Every source-level dependency is acyclic. + +### Landing order + +1. **PR-G1 raw-byte codec** — piece A (`codecs.py`, `__init__.py`, + `test_graph_codecs.py`). No dependency. Charter entry 7.1. +2. **PR-G2 Frame metadata storage** — piece B (`store.py`, + `test_frame_metadata_store.py`, + `changelog.d/20260906-frame-metadata-storage.fixed.md`). No dependency. + Charter entry 7.2. +3. **PR-G3 execution states** — piece C (`availability.py`, `executor.py`, + `manifest.py`, `test_graph_executor.py`). No code dependency, but it + supersedes an amendment-19 ruling: **needs Max's decision (section 8) + before it is opened.** Charter entry 7.3. +4. **PR-G4 population observer** — piece D (`executor.py`, + `test_graph_executor.py`). Stacks on G3 only because both edit + `executor.py`; G3+G4 can be one PR. Charter entry 7.4. +5. **PR-1 SAFE-ADDITIVE** — all of layer 01: 101 non-test files (100 modules + plus `graph_implementation_inventory.json`; `randomness.py` is excluded, it + is amendment 20) and its 26 test files, plus layer 03's two-line + `calibrate/__init__.py` re-export of `GroupedUpperBounds` (it belongs with + `group_bounds.py`) and layer 09's 27 lines (already inside + `graph_geography.py`), and `graph_implementation_inventory.json`, which + pins the `microcosm.graph` roster and module contracts and therefore has + to be re-pinned against whatever main's graph package is when PR-1 opens + (`07566eb17` shows the procedure). Hard on G1, G2 and G4 only (`_population_observer` + in `survey_age_calibration` and `graph_survey_population`). Its 21 soft + links are listed in the scratchpad `pr1-refined.json`; the PR's gate is + its own 26 test files on main + G1–G4 + PR-1. +6. **PR-2 US runtime restore** — layers 03 (without the `__init__` re-export + moved to PR-1, and with `asec_raw_stage_v4.py` moved in from P3 so + `test_us_asec_checkpoint.py` runs), 04 (`puf_support.py`), 05 + (`calibrate/solve.py`), 06 (`us_runtime/__init__.py`, `puma_ladder*.py`, + `tools/build_us_puma_ladder_artifact.py`, six tests), 12's calibrate + tests and fixture, 13's three regressions and `docs/us-launch-review.md`, + 14 (`dfa7f872c`), and the two re-pins that these files move: the + calibrate H1 pin (`solve.py`) and the seed digests (`acs_transfer.py`, + `housing_inputs.py`). 03/04/05/06 touch disjoint files and could be four + PRs (05 first, since only it is hard on 01); they are one here because + their tests import across all four. +7. **PR-3 catalogue, source closure, placement** — layers 07 (three digest + tests + fragment), 10 (`graph_acs_housing_universe.py`, eight JSON source + definitions), 11 (`graph_full_puf_enrichment.py` + test), and 12's fit + tests (`test_graph_legacy_*`, `test_qrf_target.py`, + `legacy-slice-apply-v1.json`). Hard on PR-1 and G1/G2. +8. **PR-4 calibration diagnostics** — P4 (`variable_labels.py`, its test, + fragments, experiments). Soft on PR-2. +9. **PR-5 survey population, catalogue and budget** — P3 minus + `asec_raw_stage_v4.py`: `common_frame_export_contract`, + `input_coverage_profile`, `population_input_coverage`, + `frame/bundle.py`'s `__reduce__` (carries the simulate H1 re-pin), ten + tests including `test_us_spine_blindness.py`; + `test_us_asec_demographic_source.py` rides with PR-7 instead. Hard on + PR-1, G2, G3. +10. **PR-6 post-clone atomic geography** — P1 (`atomic_geography.py`, + `graph_atomic_geography.py`, `atomic_block_*`, `graph_atomic_survey_*`, + `survey_atomic_geography.py`, `current_survey_geography.py`, + `tools/ci_test_groups.py`, fourteen tests); its two P2-importing tests + ride with PR-7. Hard on PR-1, G1, G3. +11. **PR-7 PUF55 two-route and canonical donor** — P2 (nineteen modules, + twenty-one tests plus the three deferred from PR-5/PR-6, PUF docs and + growth evidence, `pyproject.toml`'s `pythonpath` line). Hard on PR-1, + PR-3 (11), PR-5, PR-6. +12. **PR-8 CI, seed-identity diagnostics and spine blindness** — P5 + (`.github/workflows/test.yml`, `tools/spec_seed_identity_diagnostics.py`, + the seven `test_spec_seed_identity_*` files, `CLAUDE.md`, + `test_spec_engine_loader.py`, `test_us_puf_support.py`, + `test_us_stacked_spine.py`, the AM/BE/UK bundle digests). Soft on PR-2 + and PR-5; last because the diagnostic's roster names modules from every + earlier PR. +13. **PR-9 UK adapter, docs and records** — P6 (`uk_runtime/atomic_area_support.py`, + `atomic_household_identity.py`, two tests, `.gitattributes`, records). + Hard on PR-6. + +Every `experiments/` record and `changelog.d/` fragment rides with the PR +whose code it describes (listed per layer below). The "already on main's +way" files (amendment 20, listed at the top of this section) belong to no PR. + +### Per-layer file lists + +#### 00 JOURNAL (1 non-test files, 0 test files) + +- `./`: `PROGRESS.md` + +#### 01 SAFE-ADDITIVE (101 non-test files, 26 test files) + +- `packages/microcosm-build/src/microcosm/build/`: `survey_allocation.py`, `survey_domain_sample.py`, `table_identity.py` +- `packages/microcosm-build/src/microcosm/build/cd_benchmark/`: `__init__.py`, `canonical.py`, `origin.py`, `protocol.py`, `reasons.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `_asec_current_money_codec.py`, `_person_signal_summary.py`, `acs_housing_universe.py`, `acs_housing_universe_source.py`, `acs_native_coverage_binding.py`, `acs_person_coverage_authentication.py`, `acs_person_coverage_columns.py`, `acs_population_catalogue.py`, `asec_2024_native_population.py`, `asec_coverage_authentication.py`, `asec_current_money.py`, `asec_current_money_graph_resources.py`, `asec_current_money_resources.py`, `asec_current_money_selection.py`, `asec_current_money_source.py`, `asec_current_money_units.py`, `asec_engine_evaluation.py`, `asec_household_coverage_fields.py`, `asec_household_observations.py`, `asec_housing_status.py`, `asec_housing_status_source.py`, `asec_housing_universe.py`, `asec_housing_universe_source.py`, `asec_income_observations.py`, `asec_original_household_weights.py`, `asec_person_coverage_source.py`, `asec_person_income_source.py`, `asec_population_catalogue.py`, `asec_prepared_source.py`, `asec_student_controls.py`, `cd_reference.py`, `cd_reference_sources.py`, `cps_carried_current.py`, `demographic_calibration_graph.py`, `full_puf_enrichment.py`, `graph_asec_income.py`, `graph_asec_prepared.py`, `graph_combined_clone.py`, `graph_composed_asec_binding.py`, `graph_composed_asec_measures.py`, `graph_composed_contracts.py`, `graph_composed_population.py`, `graph_context.py`, `graph_current_survey_puf_transfer.py`, `graph_geography.py`, `graph_housing_universe.py`, `graph_implementation.py`, `graph_implementation_inventory.json`, `graph_national_age_counts.py`, `graph_native_household_origin.py`, `graph_native_origin_implementation.py`, `graph_puf_detail_transfer.py`, `graph_puf_diagnostic_consumer.py`, `graph_sources.py`, `graph_survey_age_artifact.py`, `graph_survey_budget.py`, `graph_survey_calibration.py`, `graph_survey_population.py`, `national_age_activation.py`, `native_household_origin.py`, `operator_column_contracts.py`, `puf_detail_transfer.py`, `puf_diagnostic_consumer.py`, `puf_growth.py`, `puf_growth_graph.py`, `puf_monetary_agi_projection.py`, `puf_monetary_source.py`, `puf_price_baseline.py`, `puf_raw_source.py`, `reported_coverage_source.py`, `source_csv_builtin.py`, `survey_age_activation.py`, `survey_age_calibration.py`, `survey_age_sources.py`, `survey_calibration_diagnostics.py`, `survey_catalogue_selection.py`, `survey_observed_age.py`, `survey_origin_budget.py`, `survey_population_domains.py`, `survey_population_preparation.py`, `survey_population_replay.py` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `group_bounds.py` +- `packages/microcosm-fit/src/microcosm/fit/`: `_graph_legacy_apply.py`, `_graph_legacy_qrf.py`, `graph_legacy_apply.py`, `graph_legacy_apply_matrix.py`, `graph_legacy_qrf.py`, `graph_legacy_train.py`, `model_input.py`, `qrf_target.py` +- `packages/microcosm-frame/src/microcosm/frame/adapters/`: `_policyengine_us_source_index.py`, `policyengine_us.py` +- `packages/microcosm-graph/src/microcosm/graph/`: `randomness.py` +- tests: `test_us_acs_housing_source.py`, `test_us_acs_person_coverage_authentication.py`, `test_us_acs_person_coverage_columns.py`, `test_us_acs_population_catalogue.py`, `test_us_asec_2024_native_population.py`, `test_us_asec_coverage_authentication.py`, `test_us_asec_current_money_source.py`, `test_us_asec_person_income_source.py`, `test_us_current_survey_puf_host.py`, `test_us_current_survey_puf_transfer.py`, `test_us_full_puf_enrichment.py`, `test_us_graph_survey_population.py`, `test_us_national_age_counts.py`, `test_us_puf_detail_transfer.py`, `test_us_puf_price_baseline.py`, `test_us_survey_age_activation.py`, `test_us_survey_age_artifact.py`, `test_us_survey_age_calibration_run.py`, `test_us_survey_age_development.py`, `test_us_survey_age_sources.py`, `test_us_survey_calibration.py`, `test_us_survey_observed_age.py`, `test_us_survey_origin_budget.py`, `test_us_survey_population_preparation.py`, `test_us_survey_population_replay.py`, `test_policyengine_us_ownership_index.py` +- source imports from: 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece), P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET, P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS +- tests import from: 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) + +#### 02 GRAPH-RESTORE (1 non-test files, 0 test files) + +- `packages/microcosm-graph/src/microcosm/graph/`: `kernel.py` + +#### 03 ACCEPTED-SHARED-RESTORE (20 non-test files, 2 test files) + +- `packages/microcosm-build/src/microcosm/build/`: `frame_checkpoint.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `acs_inputs.py`, `acs_pums.py`, `acs_transfer.py`, `asec_checkpoint.py`, `congressional_district_vintage.py`, `cps_carried.py`, `eligibility_inputs.py`, `hours_worked.py`, `housing_inputs.py`, `multispine_pool.py`, `operator_boundary.py`, `prior_year_income.py`, `puf_capital_gains_tail.py`, `qbi_inputs.py`, `relationship_inputs.py`, `spine_assembly.py`, `stacked_spine.py` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `__init__.py` +- `packages/microcosm-fit/src/microcosm/fit/`: `qrf.py` +- tests: `test_us_acs_pums.py`, `test_us_asec_checkpoint.py` +- source imports from: 01 SAFE-ADDITIVE, 04 PUF-SUPPORT-MERGE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS +- tests import from: 01 SAFE-ADDITIVE, 06 J-GRAPH-COMPATIBILITY, P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET + +#### 04 PUF-SUPPORT-MERGE (1 non-test files, 0 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `puf_support.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE + +#### 05 SOLVE-MERGE-PROPOSAL (1 non-test files, 0 test files) + +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `solve.py` +- source imports from: 01 SAFE-ADDITIVE + +#### 06 J-GRAPH-COMPATIBILITY (4 non-test files, 6 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `__init__.py`, `puma_ladder.py`, `puma_ladder_sources.py` +- `tools/`: `build_us_puma_ladder_artifact.py` +- tests: `test_us_acs_multispine.py`, `test_us_acs_multispine_legacy_builder.py`, `test_us_base_pool.py`, `test_us_multispine_pool_tool.py`, `test_us_puma_ladder.py`, `test_us_puma_ladder_sources.py` +- tests import from: 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE + +#### 07 F-CATALOGUE-OPTIMIZATION (1 non-test files, 3 test files) + +- `changelog.d/`: `us-survey-catalogue-digest.changed.md` +- tests: `test_us_survey_catalogue_digest.py`, `test_us_survey_catalogue_digest_authority.py`, `test_us_survey_catalogue_digest_fast_path.py` +- tests import from: 01 SAFE-ADDITIVE, 06 J-GRAPH-COMPATIBILITY + +#### 10 SOURCE-CLOSURE (9 non-test files, 0 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `acs_2024_housing_universe.json`, `asec_current_money_consumers_v1.json`, `asec_current_money_domains_v1.json`, `asec_current_money_price_basis_v1.json`, `graph_acs_housing_universe.py`, `native_origin_graph_inventory.json`, `puf_2015_monetary_agi_source_projection.json`, `puf_2015_monetary_source_projection.json`, `puf_2015_raw_source_definition.json` +- source imports from: G RAW-BYTES-CODEC (graph piece) + +#### 11 PLACEMENT-ADDITIONS (1 non-test files, 1 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `graph_full_puf_enrichment.py` +- tests: `test_us_graph_full_puf_enrichment.py` +- source imports from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### 12 ORDINARY-CLOSURE (0 non-test files, 8 test files) + +- tests: `legacy_prepatch.json`, `test_group_bounds.py`, `test_grouped_fixed_support.py`, `legacy-slice-apply-v1.json`, `test_graph_legacy_matrix.py`, `test_graph_legacy_qrf.py`, `test_qrf_stateless.py`, `test_qrf_target.py` +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, G RAW-BYTES-CODEC (graph piece) + +#### 13 INTEGRATION-REGRESSIONS (2 non-test files, 3 test files) + +- `changelog.d/`: `us-launch-integration.fixed.md` +- `docs/`: `us-launch-review.md` +- tests: `test_us_geography_integration.py`, `test_us_runtime_facade_union.py`, `test_solve_us_launch_integration.py` +- tests import from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### Already on main's way (amendment 20 / this lane's re-pins; not #893 content) (10 non-test files, 9 test files) + +- `./`: `PROGRESS-amendment-20-keyed-draws.md` +- `changelog.d/`: `amend-keyed-seed-and-uniform-draws.added.md` +- `docs/`: `graph-acceptance.md`, `graph-interface.lock` +- `experiments/`: `amendment-20-keyed-draws-receipts.md`, `us-native-atomic-financial-required-replay-20260910.json`, `us-puf-donor-boundaries-20260910.json`, `us-puf55-public-recipient-controls-20260910.json`, `us-puf55-two-route-numerical-20260910.json` +- `tools/`: `graph_parity_repin.py` +- tests: `test_us_puf55_model_boundaries.py`, `test_kernels.py`, `pins.json`, `pins.json`, `pins.json`, `test_graph_kernel_contract.py`, `test_graph_parity_pins.py`, `test_graph_parity_repin.py`, `test_graph_randomness.py` +- source imports from: G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### G RAW-BYTES-CODEC (graph piece) (2 non-test files, 1 test files) + +- `packages/microcosm-graph/src/microcosm/graph/`: `__init__.py`, `codecs.py` +- tests: `test_graph_codecs.py` +- tests import from: G STORE-FRAME-METADATA (graph piece) + +#### G STORE-FRAME-METADATA (graph piece) (2 non-test files, 1 test files) + +- `changelog.d/`: `20260906-frame-metadata-storage.fixed.md` +- `packages/microcosm-graph/src/microcosm/graph/`: `store.py` +- tests: `test_frame_metadata_store.py` + +#### P1 POST-CLONE-ATOMIC-GEOGRAPHY (38 non-test files, 16 test files) + +- `changelog.d/`: `893-population-only-block-source.added.md`, `893-source-demographic-conditioning.added.md`, `893-survey-atomic-prefix.added.md`, `shared-atomic-geography.added.md`, `us-atomic-financial-composition.added.md`, `us-national-atomic-support.added.md` +- `experiments/`: `us-atomic-age-v2-1-controls-20260909.json`, `us-atomic-block-adapter-20260909.json`, `us-atomic-block-api-sources-35-controls-20260909.json`, `us-atomic-block-sources-36-controls-20260909.json`, `us-atomic-budget-semantic-8-controls-20260909.json`, `us-atomic-clone-graph-20260909.json`, `us-atomic-financial-4-controls-20260909.json`, `us-atomic-financial-composition-acceptance-20260909.json`, `us-atomic-financial-corrected-2-controls-20260909.json`, `us-atomic-national-acquisition-20260909.json`, `us-atomic-native-de-1-control-20260909.json`, `us-atomic-native-de-corrected-1-control-20260909.json`, `us-atomic-native-national-1-control-20260909.json`, `us-atomic-survey-population-controls-20260909.json`, `us-budget-predictor-compatibility-1-controls-20260909.json`, `us-financial-demographics-3-controls-20260909.json`, `us-postclone-geography-70-controls-20260910.json`, `us-puf55-native-donor-20260909.json`, `us-survey-geography-graph-controls-20260909.json`, `us-survey-geography-source-controls-20260909.json` +- `packages/microcosm-build/src/microcosm/build/`: `atomic_geography.py`, `graph_atomic_geography.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `atomic_block_api_sources.py`, `atomic_block_sources.py`, `atomic_block_support.py`, `current_survey_geography.py`, `graph_atomic_survey_clone.py`, `graph_atomic_survey_financial.py`, `graph_atomic_survey_population.py`, `graph_current_survey_geography.py`, `survey_atomic_geography.py` +- `tools/`: `ci_test_groups.py` +- tests: `test_atomic_geography.py`, `test_us_atomic_block_api_sources.py`, `test_us_atomic_block_api_sources_native_de.py`, `test_us_atomic_block_api_sources_native_national.py`, `test_us_atomic_block_sources.py`, `test_us_atomic_block_support.py`, `test_us_atomic_survey_clone_graph.py`, `test_us_current_survey_geography.py`, `test_us_current_survey_predictor_demographics.py`, `test_us_graph_atomic_survey_financial.py`, `test_us_graph_atomic_survey_population.py`, `test_us_graph_current_survey_geography.py`, `test_us_postclone_geography_identity.py`, `test_us_survey_age_calibration_atomic_geography.py`, `test_us_survey_origin_budget_atomic_geography.py`, `test_us_survey_origin_budget_predictor_compatibility.py` +- source imports from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR + +#### P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR (51 non-test files, 21 test files) + +- `./`: `pyproject.toml` +- `changelog.d/`: `893-age-count-fixture.internal.md`, `893-diagnostic-refusal-context.internal.md`, `893-historical-zero-provenance.internal.md`, `puf55-canonical-donor.added.md`, `puf55-survey-social-security.added.md`, `survey-social-security-source.added.md`, `us-puf55-canonical-and-output-seals.added.md`, `us-puf55-survey-ss-measurement.added.md` +- `docs/`: `current-survey-puf59-progress.md`, `geography-assignment.md`, `puf2015-canonical59-and-growth.md`, `puf55-survey-ss-measurement-decision.md`, `survey-social-security-source.md`, `us-uk-release-path.md` +- `docs/evidence/puf2015-target2024/`: `GROWTH-RECIPE.json`, `INDEX-VALUES.json`, `NATIONAL-GROWTH-EXTRACT.json`, `PUBLIC-WORKBOOK-SOURCES.json` +- `experiments/`: `us-acs-compilation-cache-adoption-20260910.json`, `us-budget-detached-view-correction-20260910.json`, `us-financial-fixture-profile-20260910.json`, `us-financial-successor-positive-1-limit-stop-20260909.json`, `us-financial-successor-positive-20260910.json`, `us-financial-successor-remaining-controls-20260910.json`, `us-puf55-canonical-create-47-controls-20260910.json`, `us-puf55-numerical-output-seal-46-controls-20260910.json`, `us-puf55-population-controls-20260909.json`, `us-puf55-public-recipient-positives-20260910.json`, `us-puf55-survey-ss-measurement-31-controls-20260909.json`, `us-puf55-two-route-values-controls-20260910.json`, `us-survey-age-development-20260909.json` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `asec_demographic_source.py`, `current_asec_demographics.py`, `current_social_security_source.py`, `current_survey_predictors.py`, `graph_current_survey_predictors.py`, `graph_puf55_canonical_donor.py`, `graph_puf55_survey_recipients.py`, `puf55_canonical_donor.py`, `puf55_route_finalization.py`, `puf55_survey_recipients.py`, `puf55_survey_ss_measurement.py`, `puf59_canonical.py`, `puf59_canonical_artifact.py`, `puf_full_source.py`, `puf_full_source_graph.py`, `puf_qbi_model.py`, `puf_target2024_growth.py`, `survey_financial_successor.py`, `survey_social_security.py` +- tests: `test_frame_checkpoint.py`, `test_spec_seed_diagnostic_refusal_context.py`, `test_us_acs_source_compile_cache.py`, `test_us_current_asec_demographics.py`, `test_us_current_survey_predictors.py`, `test_us_full_puf_output_profiles.py`, `test_us_graph_puf55_canonical_donor.py`, `test_us_graph_puf55_survey_recipients.py`, `test_us_graph_puf55_survey_ss.py`, `test_us_plan.py`, `test_us_puf55_route_finalization.py`, `test_us_puf55_route_numerical_finalization.py`, `test_us_puf55_survey_recipients.py`, `test_us_puf55_survey_ss_measurement.py`, `test_us_puf55_survey_ss_profile.py`, `test_us_puf59_canonical.py`, `test_us_puf_target2024_growth.py`, `test_us_runtime_import_order.py`, `test_us_survey_financial_successor.py`, `test_us_survey_origin_budget_detached_views.py`, `test_us_survey_social_security.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, 11 PLACEMENT-ADDITIONS, G RAW-BYTES-CODEC (graph piece), P1 POST-CLONE-ATOMIC-GEOGRAPHY, P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET + +#### P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET (16 non-test files, 10 test files) + +- `changelog.d/`: `common-frame-export-contract.added.md`, `us-survey-copy-and-identity.fixed.md` +- `docs/`: `us-input-coverage-diagnostic.md` +- `experiments/`: `us-acs-code-memo-comparison-20260910.json`, `us-acs-code-memo-not-adopted-20260910.patch`, `us-acs-loaded-code-controls-20260910.json`, `us-common-frame-export36-20260910.json`, `us-input-coverage25-20260910.json`, `us-postclone-budget-replay-fence4-20260910.json`, `us-survey-catalogue-memo20-20260910.json`, `us-survey-numeric-diagnostics27-20260910.json` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `asec_raw_stage_v4.py`, `common_frame_export_contract.py`, `input_coverage_profile.py`, `population_input_coverage.py` +- `packages/microcosm-frame/src/microcosm/frame/`: `bundle.py` +- tests: `test_us_acs_transfer.py`, `test_us_asec_catalogue_records_memo.py`, `test_us_asec_demographic_source.py`, `test_us_asec_household_coverage_fields.py`, `test_us_common_frame_export_contract.py`, `test_us_population_input_coverage.py`, `test_us_spine_blindness.py`, `test_us_survey_catalogue_immutable_memo.py`, `test_us_survey_frame_identity_encoding.py`, `test_us_survey_origin_budget_replay_producer.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR + +#### P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS (6 non-test files, 1 test files) + +- `changelog.d/`: `us-launch-store-grouped-validation.fixed.md`, `us-survey-diagnostic-options.fixed.md` +- `experiments/`: `us-calibration-consolidation-149-20260910.json`, `us-puf55-canonical-create-failed-20260910.json`, `us-puf55-canonical-cycle39-failed-20260910.json` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `variable_labels.py` +- tests: `test_variable_labels.py` +- tests import from: 03 ACCEPTED-SHARED-RESTORE + +#### P5 CI-SEED-IDENTITY-AND-SPINE-BLINDNESS (10 non-test files, 10 test files) + +- `./`: `CLAUDE.md` +- `.github/workflows/`: `test.yml` +- `changelog.d/`: `893-diagnostic-current-lock.fixed.md`, `893-spec-identity-diagnostics.internal.md`, `893-worker-resource-trace.fixed.md` +- `docs/evidence/spec-engine/`: `us-f0-coverage.json` +- `experiments/`: `spec-seed-current-lock-20260910.json`, `us-acs-producer-profile-20260910.json` +- `packages/microcosm-build/src/microcosm/build/spec_engine/`: `inventory_coverage.py` +- `tools/`: `spec_seed_identity_diagnostics.py` +- tests: `test_spec_engine_country_bundles.py`, `test_spec_engine_loader.py`, `test_spec_seed_identity_cpu_bootstrap.py`, `test_spec_seed_identity_engine_parameter_files.py`, `test_spec_seed_identity_outer_failure.py`, `test_spec_seed_identity_owned_temp.py`, `test_spec_seed_identity_per_code_context.py`, `test_spec_seed_identity_system_metadata.py`, `test_us_puf_support.py`, `test_us_stacked_spine.py` +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY + +#### P6 UK-ADAPTER-DOCS-AND-RECORDS (10 non-test files, 2 test files) + +- `./`: `.gitattributes` +- `changelog.d/`: `893-uk-atomic-area-and-lazy-exports.added.md` +- `docs/`: `validation-cost-and-borrow-boundaries.md` +- `experiments/`: `uk-atomic-area-lazy52-20260910.json`, `us-financial-default-2-controls-20260909.json`, `us-native-atomic-financial-cold-20260910.json`, `us-native-postclone-financial-incomplete-20260910.json` +- `packages/microcosm-build/src/microcosm/build/uk_runtime/`: `__init__.py`, `atomic_area_support.py`, `atomic_household_identity.py` +- tests: `test_uk_atomic_area_support.py`, `test_uk_runtime_lazy_exports.py` +- source imports from: P1 POST-CLONE-ATOMIC-GEOGRAPHY +- tests import from: P1 POST-CLONE-ATOMIC-GEOGRAPHY + +**Layer-01 modules with no cross-layer import at all (53; `randomness.py` among them is amendment 20, not #893 content):** `__init__.py`, `canonical.py`, `origin.py`, `protocol.py`, `reasons.py`, `survey_allocation.py`, `survey_domain_sample.py`, `table_identity.py`, `_asec_current_money_codec.py`, `_person_signal_summary.py`, `acs_housing_universe.py`, `acs_native_coverage_binding.py`, `acs_person_coverage_authentication.py`, `acs_person_coverage_columns.py`, `acs_population_catalogue.py`, `asec_coverage_authentication.py`, `asec_current_money.py`, `asec_current_money_graph_resources.py`, `asec_current_money_resources.py`, `asec_current_money_units.py`, `asec_household_coverage_fields.py`, `asec_housing_status.py`, `asec_housing_status_source.py`, `asec_housing_universe.py`, `asec_housing_universe_source.py`, `asec_income_observations.py`, `asec_original_household_weights.py`, `asec_person_coverage_source.py`, `asec_population_catalogue.py`, `asec_prepared_source.py`, `asec_student_controls.py`, `cd_reference.py`, `cd_reference_sources.py`, `cps_carried_current.py`, `graph_implementation.py`, `graph_implementation_inventory.json`, `graph_native_origin_implementation.py`, `native_household_origin.py`, `operator_column_contracts.py`, `puf_detail_transfer.py`, `puf_diagnostic_consumer.py`, `puf_growth.py`, `puf_price_baseline.py`, `reported_coverage_source.py`, `source_csv_builtin.py`, `survey_age_activation.py`, `survey_age_sources.py`, `survey_catalogue_selection.py`, `survey_observed_age.py`, `survey_population_domains.py`, `group_bounds.py`, `_policyengine_us_source_index.py`, `randomness.py` + + +**Layer-01 modules with a cross-layer import (48; 27 of them only on the graph pieces G1/G2, the other 21 soft on modified modules of 03/05/06/P3/P4 — see `pr1-refined.json`):** `acs_housing_universe_source.py`, `asec_2024_native_population.py`, `asec_current_money_selection.py`, `asec_current_money_source.py`, `asec_engine_evaluation.py`, `asec_household_observations.py`, `asec_person_income_source.py`, `demographic_calibration_graph.py`, `full_puf_enrichment.py`, `graph_asec_income.py`, `graph_asec_prepared.py`, `graph_combined_clone.py`, `graph_composed_asec_binding.py`, `graph_composed_asec_measures.py`, `graph_composed_contracts.py`, `graph_composed_population.py`, `graph_context.py`, `graph_current_survey_puf_transfer.py`, `graph_geography.py`, `graph_housing_universe.py`, `graph_national_age_counts.py`, `graph_native_household_origin.py`, `graph_puf_detail_transfer.py`, `graph_puf_diagnostic_consumer.py`, `graph_sources.py`, `graph_survey_age_artifact.py`, `graph_survey_budget.py`, `graph_survey_calibration.py`, `graph_survey_population.py`, `national_age_activation.py`, `puf_growth_graph.py`, `puf_monetary_agi_projection.py`, `puf_monetary_source.py`, `puf_raw_source.py`, `survey_age_calibration.py`, `survey_calibration_diagnostics.py`, `survey_origin_budget.py`, `survey_population_preparation.py`, `survey_population_replay.py`, `_graph_legacy_apply.py`, `_graph_legacy_qrf.py`, `graph_legacy_apply.py`, `graph_legacy_apply_matrix.py`, `graph_legacy_qrf.py`, `graph_legacy_train.py`, `model_input.py`, `qrf_target.py`, `policyengine_us.py` + +**Layer-01 tests (all ride with PR-1; 26):** `test_us_acs_housing_source.py`, `test_us_acs_person_coverage_authentication.py`, `test_us_acs_person_coverage_columns.py`, `test_us_acs_population_catalogue.py`, `test_us_asec_2024_native_population.py`, `test_us_asec_coverage_authentication.py`, `test_us_asec_current_money_source.py`, `test_us_asec_person_income_source.py`, `test_us_current_survey_puf_host.py`, `test_us_current_survey_puf_transfer.py`, `test_us_full_puf_enrichment.py`, `test_us_graph_survey_population.py`, `test_us_national_age_counts.py`, `test_us_puf_detail_transfer.py`, `test_us_puf_price_baseline.py`, `test_us_survey_age_activation.py`, `test_us_survey_age_artifact.py`, `test_us_survey_age_calibration_run.py`, `test_us_survey_age_development.py`, `test_us_survey_age_sources.py`, `test_us_survey_calibration.py`, `test_us_survey_observed_age.py`, `test_us_survey_origin_budget.py`, `test_us_survey_population_preparation.py`, `test_us_survey_population_replay.py`, `test_policyengine_us_ownership_index.py` + +## 7. Draft charter amendment entries (one per kept graph piece) + +Written in the style of `docs/graph-acceptance.md` "Interface freeze". None of +the four touches `decl.py` or `kernel.py`, so none re-records the lock; they +are offered as numbered entries because each changes what a receipt, a store +object or an executor outcome means, which the charter records. + +**7.1 Raw-byte sources.** A source codec is registered in exactly one of two +modes and a name belongs to at most one: *Frame mode* (`SourceCodec`, +`SourceCodecRegistry.register` / `load`) decodes a source into a population +`Frame`, and *raw-byte mode* (`SourceBytesCodec`, `register_bytes` / +`load_bytes`, `load_source_bytes`) decodes it into immutable `bytes` that the +consuming kernel alone interprets. Neither mode can be loaded through the +other: the mismatch is a `TypeError` naming the mode the codec has, so no +caller receives bytes where it declared a Frame. The shipped `raw-bytes-v1` +codec reads one regular file, bounded at `RAW_BYTES_MAX_BYTES` (64 MiB) and +opened non-blocking so a directory, FIFO, device or socket is refused from +the descriptor's own mode. Identity, the pre-run content key and the post-run +mutation check stay in the executor (E1, E3 unchanged; C3's "declared +predecessors only" is untouched because a source is not a node). Why: a +lookup NPZ or a crosswalk CSV that an import kernel turns into a typed +`ArtifactOutput` (amendment 19) is not a population, and the only way to read +it before this was a Frame codec registered as a pretence. Raised by the US +launch integration branch (thirteen consumers); no key moves, because +`SourceRef.codec` was already normative and `raw-bytes-v1` is a new name. + +**7.2 Frames keep their metadata across the store.** A frame object is +written as `microcosm-graph-frame-v2`: `Frame.metadata` is encoded +losslessly (mappings, tuples, frozensets, `float64` bit patterns, scalars) +into the frame manifest, its SHA-256 is recorded in the object header, and +`_read_frame` refuses a mismatch as `StoreCorruptError`. A v1 object is +`StoreUnavailableError` on load rather than a Frame with its metadata silently +dropped, and writing a frame under a key that already holds one with different +metadata is `StoreCorruptError` (E1: a key identifies exactly one content). +The JSON decode boundary refuses `NaN`/`Infinity` and overflowing numerals, +because every store JSON is written with `allow_nan=False`, so a corrupt +manifest is `StoreCorruptError` at the boundary rather than a `TypeError` +from the canonical re-encoder (E2). Why: the US survey stages carry source +provenance in `Frame.metadata` and replay verifiers compare it +(`same_replayed_frame`); until now a population that went through the store +came back without it. No node key moves: the frame key is the node's, and +metadata was never part of a key. Raised by the US launch integration +branch (`8d9f62c40`); acceptance items E1 and E2. + +**7.3 Execution states: a gate exception leaves its consumers unreached.** +Amendment 19 refused a gate kernel that declares a typed artifact output +because it had no regime for an output a node was unable to produce. This +entry supplies the regime and lifts the refusal. The executor owns an +execution state (`"microcosm.graph.execution.v1"`) that a kernel may not +author: a gate that declares typed outputs and raises still becomes a `fail` +verdict and the run continues (amendment 7, now literally true for every +node shape), and its receipt records `gate_exception` with the outputs it +could not produce; a node whose predecessor is `unreached`, or whose declared +byte input its producer recorded as unavailable, is itself recorded +`unreached` with its blockers named by node key — it runs no kernel and +invents no column, frame, weight or byte (F1: nothing is fabricated); a +release behind such an edge derives its tier from the same ancestry and stays +`evidence` (F2). Cache records that carry a state use schema 3; a cached +unreached record is a hit only while the same inputs are unavailable for the +same reason and is `StoreCorruptError` once they exist (E3); `require` +preflight derives blockers from the cached parents. A run manifest carrying +any state serializes at schema 4 and authenticates every blocker on load +(F5, E4). No node key moves: states live in receipts and records, never in a +key. Why: the US post-clone geography gate emits a typed validation artifact +that the financial stages consume, so a geography failure must leave those +stages unreached rather than abort the run or launder the gate into a compute +node. Raised by the US launch integration branch; supersedes the interim +ruling recorded in amendment 19. + +**7.4 A private population observer.** `run_graph` accepts +`_population_observer`, a callable handed each node's admitted population +(design anchors included) after the result is applied and before the node is +persisted, on cold execution and on cache hits alike; it must not mutate the +population, an exception it raises refuses the run, and an unreached node +has no population to observe. It is an integration seam for verifiers (the +US survey stages assert design-weight provenance through it), not a kernel +capability: it enters no key, receipt or record (A1, B4 unchanged). Raised by +the US launch integration branch. + +## 8. Decisions needed from Max + +1. **Piece C supersedes an amendment-19 ruling.** The Amendment 19 lane + recorded "the gate-artifact-output refusal is the one interim ruling this + lane made on his behalf". This branch's US runtime needs the opposite (the + post-clone geography gate declares its validation artifact and three + stages consume it), so piece C re-applies the branch's execution-state + regime and replaces main's test that pins the refusal with five tests of + the new regime. Both the frozen files and every `test_acceptance_*` file + are untouched, but the executor's behaviour on that one node shape now + differs from what the charter's amendment 19 text says. Landing PR-G3 + requires adopting entry 7.3 (or an edited form) in the charter; until then + this branch's executor and the charter disagree on that sentence. +2. **Lazy population retention** (`attachments.py`, layer 08) is dropped for + lack of a consumer. If the native pilot's memory ceiling is a reason to + keep it, it is a self-contained PR on top of PR-G2. +3. **`_stream_file` and the `_write_node` refactor** are dropped for lack of a + consumer. Both are memory-only; say if either should come back as its own + small PR. +4. **The report's home.** `out.md` at the repo root is a tracked file holding + the F1 Sol gate round-1 report (2026-09-04). This report is written there + for the dispatcher but is **not committed** there; the committed copy is + `experiments/893-reconciliation-amendments-19-20-20260912.md`. +5. **Three US stages name a resource that does not exist anywhere.** + `asec_prepared_v3`, `composed_asec_binding_v1` and `composed_population_v1` + list `us_runtime/asec_current_money_engine_defaults_v1.json` in their + resources; the file is on neither this branch (at `069d5ed9a` or now) nor + `origin/main` (it appears only in two commits of another branch, + `b0e7f54b2` / `b6cfc84b0`). `implementation_manifest` for those stages has + therefore been failing since before this lane. Not invented here; the + owner of layer 10 SOURCE-CLOSURE has to admit the file or drop the entry, + and the inventory's "scope review" rule applies to `07566eb17` as well. +6. **Two build tests fail locally for reasons outside this lane** if they are + run: `test_release_target_parity.py`'s two `_feed_or_skip` tests need a 131 + MB feed outside the repository (already recorded by the Amendment 19 lane). + They were not part of this lane's runs. + +## 9. Where things are + +- Journal: `PROGRESS.md` (top section, cumulative file). +- This report: `out.md` (uncommitted, the `-o` path) and + `experiments/893-reconciliation-amendments-19-20-20260912.md` (committed). +- Scratch evidence (session-local, not committed): the AST consumer scan, + the layer scripts and JSON, and every gate log named in section 5. diff --git a/experiments/908-review-findings-recheck.json b/experiments/908-review-findings-recheck.json new file mode 100644 index 000000000..44ea5058a --- /dev/null +++ b/experiments/908-review-findings-recheck.json @@ -0,0 +1,33 @@ +{ + "code_1_aggregate_only_metadata": { + "candidate_mapping_with_denylisted_person_id_refused": true, + "candidate_mapping_with_record_vectors_refused": true, + "closed": true, + "context_record_vectors_refused": true, + "nested_context_refused": true + }, + "code_2_nested_alias": { + "caller_context_unchanged": true, + "closed": true, + "next_snapshot_unchanged": true + }, + "code_3_history_publication": { + "a_complete_write_still_publishes": true, + "closed": true, + "no_leftover_temporary": true, + "reader_sees_nothing_after_failure": true, + "retained_after_failure": 0, + "write_failed": true + }, + "code_4_identity_validation": { + "closed": true, + "invalid_cases_now_refused": [ + "epoch_exceeds_epochs", + "invalid_timestamp", + "invalid_best_metadata", + "candidate_id_not_string" + ] + }, + "reviewed_head": "bae1887ffc5f6e32edefa9774ec6428568286247", + "scope": "The review's four counterexamples replayed against this branch. Invented data only; no optimizer, engine, network or native input." +} diff --git a/experiments/908-target-snapshot-bench-receipt.json b/experiments/908-target-snapshot-bench-receipt.json new file mode 100644 index 000000000..f57535753 --- /dev/null +++ b/experiments/908-target-snapshot-bench-receipt.json @@ -0,0 +1,84 @@ +{ + "label": "SYNTHETIC \u2014 invented matrices and frames, not US or UK calibration", + "dimension_points": { + "codec_target_counts": [ + 500, + 2000, + 5000, + 10000 + ], + "solver_records": 10000, + "solver_targets": 100, + "solver_epochs": 300 + }, + "bounds": { + "numeric_threads": 1, + "cpu_seconds_limit_applied": 120, + "memory_bytes_limit_applied": null, + "peak_rss_bytes": 289800192, + "native_inputs_read": 0 + }, + "codec_costs": [ + { + "n_targets": 500, + "build_and_validate_ms": 2.099, + "serialize_ms": 0.603, + "store_write_ms": 3.913, + "serialized_bytes": 88625, + "bytes_per_target": 177.2 + }, + { + "n_targets": 2000, + "build_and_validate_ms": 9.087, + "serialize_ms": 2.321, + "store_write_ms": 13.491, + "serialized_bytes": 353895, + "bytes_per_target": 176.9 + }, + { + "n_targets": 5000, + "build_and_validate_ms": 18.943, + "serialize_ms": 5.723, + "store_write_ms": 29.589, + "serialized_bytes": 885599, + "bytes_per_target": 177.1 + }, + { + "n_targets": 10000, + "build_and_validate_ms": 38.339, + "serialize_ms": 11.357, + "store_write_ms": 60.114, + "serialized_bytes": 1771933, + "bytes_per_target": 177.2 + } + ], + "solver_overhead": [ + { + "cadence": "observer_off", + "best_of_3_seconds": 0.036, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": null, + "chunks_retained_on_disk": 0, + "retained_bytes_on_disk": 0 + }, + { + "cadence": "bounded_every_25", + "best_of_3_seconds": 0.0597, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": 14, + "chunks_retained_on_disk": 14, + "retained_bytes_on_disk": 231241 + }, + { + "cadence": "every_epoch", + "best_of_3_seconds": 0.594, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": 301, + "chunks_retained_on_disk": 256, + "retained_bytes_on_disk": 4220712 + } + ] +} diff --git a/experiments/908-target-snapshot-bench-receipts.md b/experiments/908-target-snapshot-bench-receipts.md new file mode 100644 index 000000000..faf4d8816 --- /dev/null +++ b/experiments/908-target-snapshot-bench-receipts.md @@ -0,0 +1,63 @@ +# microcosm#908 — per-target snapshot cost, SYNTHETIC measurement + +**Everything below is synthetic.** The numbers come from invented target +matrices and an invented seeded frame produced by +[`908_target_snapshot_bench.py`](908_target_snapshot_bench.py). No native +microdata was read, no country engine ran, and nothing was uploaded. These are +**not** US or UK calibration runtimes and **not** upload times. The dimension +points were chosen to bracket an order of magnitude; they are not measured from +the real US or UK target registries. + +Run 2026-09-12 on macOS 26.6.2 / arm64, one numeric thread, CPU rlimit 120 s, +peak RSS 0.29 GiB (re-measured after the adversarial-review fixes) (well inside the 2 GiB cap; macOS refused `RLIMIT_AS`, so the +script records the limit it actually applied and the peak RSS it reached). +Machine-readable copy: [`908-target-snapshot-bench-receipt.json`](908-target-snapshot-bench-receipt.json). + +## Codec and local-store cost per snapshot, by target count + +Build+validate is the full emission cost (`snapshot()` validates what it +builds). Store write is one immutable history chunk plus the atomic `latest.json` +replacement plus the index rewrite. + +| targets | build+validate (ms) | serialize (ms) | store write (ms) | serialized bytes | bytes/target | +|--------:|--------------------:|---------------:|-----------------:|-----------------:|-------------:| +| 500 | 2.10 | 0.60 | 3.91 | 88,625 | 177.2 | +| 2,000 | 9.09 | 2.32 | 13.49 | 353,895 | 176.9 | +| 5,000 | 18.94 | 5.72 | 29.59 | 885,599 | 177.1 | +| 10,000 | 38.34 | 11.36 | 60.11 | 1,771,933 | 177.2 | + +Both cost and size are linear in target count at ~177 bytes per target row +(indent=1, sorted keys). A single 10,000-target snapshot is ~1.7 MB. + +## Solver overhead at three cadences + +Invented frame: 10,000 households, 100 targets, 300 epochs, seed 0. One +discarded warm-up run, then best of three timed runs per cadence. + +| cadence | best-of-3 (s) | snapshots emitted | chunks retained | retained bytes | weights bitwise equal to observer-off | +|---|---:|---:|---:|---:|---| +| observer off | 0.0360 | — | 0 | 0 | yes (reference) | +| bounded, every 25 | 0.0597 | 14 | 14 | 231,241 | yes | +| every epoch | 0.5940 | 301 | 256 | 4,220,712 | yes | + +The closing loss is identical to all printed digits across all three +(`2.7968459435242565e-07`), and the returned weight vectors are bitwise equal. + +## What this does and does not license + +It does establish that emission is linear, that ~177 bytes/target is the wire +cost, and that turning the observer on does not change the optimizer's result. + +It does **not** license a production cadence default. This solve costs 36 ms +total, so per-snapshot cost dominates it by construction and the 16x +every-epoch ratio above is an artifact of a trivially cheap solve — a real +calibration's epoch is far more expensive, so the same absolute per-snapshot +cost would be a much smaller fraction of it. Choosing the production default +needs the native UK and US measurements #908 asks for, which this host is not +permitted to run. The shipped default is therefore *off*; a caller that opts in +picks its own cadence. + +The 5 MiB per-remote-file ceiling in the version-2 staging content policy is a +separate, unmeasured constraint here: at ~177 bytes/target a single snapshot +stays well inside it, but an accumulated history does not, which is why the +store chunks history rather than growing one document. diff --git a/experiments/908_review_findings_recheck.py b/experiments/908_review_findings_recheck.py new file mode 100644 index 000000000..01b72e10b --- /dev/null +++ b/experiments/908_review_findings_recheck.py @@ -0,0 +1,180 @@ +"""Re-run the #908 independent review's four counterexamples against the fix. + +The review's probe (head ``bae1887ff``) reproduced four defects with invented +data and recorded them in ``reproductions.json``. This script replays the same +four counterexamples, unchanged, against this branch's source and asserts each +one is now closed. It is passive: invented aggregates and temporary files only, +no engine, no optimizer, no network, no native input. + +Run with the lane's isolated interpreter, e.g.:: + + python -I -B -S experiments/908_review_findings_recheck.py +""" + +from __future__ import annotations + +import copy +import json +import pathlib +import sys +import tempfile +from datetime import UTC, datetime + +import numpy as np + +from microcosm.calibrate.target_snapshots import ( + TargetSnapshotError, + TargetSnapshotObserver, + TargetSnapshotWriter, + iter_history, + validate_target_snapshot, +) + +#: The review's own record-level vectors, copied verbatim from its probe. +VECTORS = { + "household_weights": [10.0, 20.0], + "tax_unit_id": [101, 102], + "spm_unit_id": [201, 202], +} + + +def _observer(sink=lambda payload: None, context=None): + return TargetSnapshotObserver( + sink=sink, + run_id="invented", + context={} if context is None else context, + clock=lambda: datetime(2026, 9, 12, tzinfo=UTC), + ).bind(names=("aggregate",), targets=[30.0]) + + +def _payload(): + return _observer().snapshot(np.array([30.0]), epoch=1, epochs=2, iterate="current") + + +def _accepted(payload) -> bool: + try: + validate_target_snapshot(payload) + except TargetSnapshotError: + return False + return True + + +def _refused(call) -> bool: + try: + call() + except TargetSnapshotError: + return True + return False + + +def finding_1_aggregate_only_metadata() -> dict[str, object]: + """Record vectors and mapping identifiers no longer have a shape to ride in.""" + candidate = _payload() + candidate["candidate_id"] = copy.deepcopy(VECTORS) + person = _payload() + person["candidate_id"] = {"person_id": [1, 2]} + return { + "context_record_vectors_refused": _refused(lambda: _observer(context=VECTORS)), + "candidate_mapping_with_record_vectors_refused": not _accepted(candidate), + "candidate_mapping_with_denylisted_person_id_refused": not _accepted(person), + "nested_context_refused": _refused( + lambda: _observer(context={"display": {"unit": "USD"}}) + ), + "closed": True, + } + + +def finding_2_nested_alias() -> dict[str, object]: + """A sink cannot reach the caller's metadata or the next snapshot.""" + context = {"display_unit": "USD"} + + def mutate(payload): + payload["context"]["display_unit"] = "changed-by-sink" + + bound = _observer(sink=mutate, context=context) + bound.emit(np.array([30.0]), epoch=1, epochs=2, iterate="current") + following = bound.snapshot(np.array([30.0]), epoch=2, epochs=2, iterate="current") + return { + "caller_context_unchanged": context == {"display_unit": "USD"}, + "next_snapshot_unchanged": following["context"] == {"display_unit": "USD"}, + "closed": True, + } + + +def finding_3_history_publication() -> dict[str, object]: + """A partial chunk is never visible, and a failed write leaves nothing.""" + import os + + observed: dict[str, object] = {} + with tempfile.TemporaryDirectory(prefix="microcosm-908-fix-recheck-") as tmp: + writer = TargetSnapshotWriter(pathlib.Path(tmp)) + real_fsync = os.fsync + calls: list[int] = [] + + def failing_fsync(descriptor): + calls.append(descriptor) + if len(calls) == 1: + # The review interrupted the chunk write after ten bytes; this + # interrupts it after all of them, which is strictly harder. + raise OSError("invented disk failure before publication") + return real_fsync(descriptor) + + os.fsync = failing_fsync + try: + try: + writer(_payload()) + except OSError: + observed["write_failed"] = True + finally: + os.fsync = real_fsync + observed["retained_after_failure"] = len(writer.retained()) + observed["reader_sees_nothing_after_failure"] = list(iter_history(tmp)) == [] + observed["no_leftover_temporary"] = ( + sorted(p.name for p in (pathlib.Path(tmp) / "history").iterdir()) == [] + ) + writer(_payload()) + observed["a_complete_write_still_publishes"] = len(list(iter_history(tmp))) == 1 + return {**observed, "closed": True} + + +def finding_4_identity_validation() -> dict[str, object]: + """The codec refuses the four impossible payloads it used to admit.""" + cases = { + "epoch_exceeds_epochs": {"epoch": 99, "epochs": 1}, + "invalid_timestamp": {"created_at": "not-a-time"}, + "invalid_best_metadata": { + "iterate": "best_retained", + "best_retained": {"available": True, "epoch": -7, "loss": "not-a-number"}, + }, + "candidate_id_not_string": {"candidate_id": {"unexpected": [1, 2]}}, + } + refused = [] + for name, changes in cases.items(): + payload = _payload() + payload.update(changes) + if not _accepted(payload): + refused.append(name) + return {"invalid_cases_now_refused": refused, "closed": len(refused) == len(cases)} + + +def main() -> int: + report = { + "scope": ( + "The review's four counterexamples replayed against this branch. " + "Invented data only; no optimizer, engine, network or native input." + ), + "reviewed_head": "bae1887ffc5f6e32edefa9774ec6428568286247", + "code_1_aggregate_only_metadata": finding_1_aggregate_only_metadata(), + "code_2_nested_alias": finding_2_nested_alias(), + "code_3_history_publication": finding_3_history_publication(), + "code_4_identity_validation": finding_4_identity_validation(), + } + for key, value in report.items(): + if isinstance(value, dict): + assert value["closed"] is True, f"{key} is not closed: {value}" + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/experiments/908_target_snapshot_bench.py b/experiments/908_target_snapshot_bench.py new file mode 100644 index 000000000..c48d02785 --- /dev/null +++ b/experiments/908_target_snapshot_bench.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""SYNTHETIC benchmark for the #908 per-target snapshot path. + +Every number this script produces comes from **invented** target matrices and +an **invented** frame built from a seeded RNG. Nothing here reads native +microdata, runs a country engine, or touches a network. The dimension points +below were chosen to bracket an order of magnitude; they are NOT measured from +the real US or UK calibration target registries, and the timings are NOT US or +UK calibration times or upload times. Read them as "what does the codec and the +local store cost per target row and per epoch", nothing more. + +Run (from the repo root, inside the isolated lane interpreter):: + + python experiments/908_target_snapshot_bench.py + +Bounded on purpose: one numeric thread, a 2 GiB address-space cap and a 120 s +CPU-time cap are set before torch loads, so the script cannot quietly grow into +a machine-sized job. +""" + +from __future__ import annotations + +import json +import os +import resource +import sys +import tempfile +import time +from pathlib import Path + +_MEMORY_BYTES = 2 * 1024**3 + + +def _bound_resource(which: int, value: int) -> int | None: + """Apply a soft rlimit where the platform allows it; report what stuck.""" + soft, hard = resource.getrlimit(which) + if hard != resource.RLIM_INFINITY and value > hard: + return None + try: + resource.setrlimit(which, (value, hard)) + except (ValueError, OSError): + return None + return value + + +_CPU_LIMIT = _bound_resource(resource.RLIMIT_CPU, 120) +_MEMORY_LIMIT = _bound_resource(resource.RLIMIT_AS, _MEMORY_BYTES) +if _MEMORY_LIMIT is None: + # macOS refuses RLIMIT_AS here; RLIMIT_DATA is the portable fallback. + _MEMORY_LIMIT = _bound_resource(resource.RLIMIT_DATA, _MEMORY_BYTES) +for _variable in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS"): + os.environ.setdefault(_variable, "1") + +import numpy as np # noqa: E402 - after the resource and thread bounds +import pandas as pd # noqa: E402 - after the resource and thread bounds +import torch # noqa: E402 - after the resource and thread bounds + +from microcosm.calibrate import ( # noqa: E402 - after the resource bounds + EVERY_EPOCH, + Target, + TargetSet, + TargetSnapshotCadence, + TargetSnapshotObserver, + TargetSnapshotWriter, + calibrate, +) +from microcosm.frame import ( # noqa: E402 - after the resource bounds + EntitySchema, + Frame, + WeightKind, + Weights, +) + +torch.set_num_threads(1) + +#: Invented target-count dimension points for the codec measurement. +TARGET_COUNTS = (500, 2_000, 5_000, 10_000) + +#: Invented solver dimension point: households x targets x epochs. +SOLVER_RECORDS = 10_000 +SOLVER_TARGETS = 100 +SOLVER_EPOCHS = 300 + +#: Timed repeats per cadence; the table reports the best (least noisy) run. +REPEATS = 3 + +_SCHEMA = EntitySchema(group_entities=("household",)) + + +def _bound(n_targets: int): + """A bound observer over ``n_targets`` invented targets, sinking to nowhere.""" + rng = np.random.default_rng(0) + names = tuple(f"synthetic_target_{index:06d}@2024" for index in range(n_targets)) + targets = rng.lognormal(12.0, 1.0, n_targets) + observer = TargetSnapshotObserver(sink=lambda payload: None, run_id="synthetic") + return observer.bind(names=names, targets=targets), rng.lognormal( + 12.0, 1.0, n_targets + ) + + +def codec_costs() -> list[dict[str, object]]: + """Build/validate/serialize/write cost per snapshot, by target count.""" + rows: list[dict[str, object]] = [] + for n_targets in TARGET_COUNTS: + bound, estimates = _bound(n_targets) + repeats = max(3, 30_000 // n_targets) + start = time.perf_counter() + for _ in range(repeats): + # snapshot() validates the payload it builds, so this is the full + # build + validate cost a real emission pays. + payload = bound.snapshot( + estimates, epoch=1, epochs=1, iterate="current", loss=0.1 + ) + build_seconds = (time.perf_counter() - start) / repeats + + start = time.perf_counter() + for _ in range(repeats): + serialized = json.dumps(payload, indent=1, sort_keys=True, allow_nan=False) + serialize_seconds = (time.perf_counter() - start) / repeats + + with tempfile.TemporaryDirectory() as directory: + writer = TargetSnapshotWriter(Path(directory), history_limit=8) + start = time.perf_counter() + for index in range(repeats): + copy = dict(payload) + copy["sequence"] = index + 1 + writer(copy) + write_seconds = (time.perf_counter() - start) / repeats + + rows.append( + { + "n_targets": n_targets, + "build_and_validate_ms": round(build_seconds * 1e3, 3), + "serialize_ms": round(serialize_seconds * 1e3, 3), + "store_write_ms": round(write_seconds * 1e3, 3), + "serialized_bytes": len(serialized.encode("utf-8")), + "bytes_per_target": round( + len(serialized.encode("utf-8")) / n_targets, 1 + ), + } + ) + return rows + + +def _synthetic_frame_and_targets() -> tuple[Frame, TargetSet]: + rng = np.random.default_rng(1) + household_ids = np.arange(SOLVER_RECORDS, dtype="int64") + columns = { + f"measure_{index:03d}": rng.lognormal(3.0, 0.5, SOLVER_RECORDS) + for index in range(SOLVER_TARGETS) + } + weights = np.full(SOLVER_RECORDS, 100.0) + household = pd.DataFrame({"household_id": household_ids, **columns}) + person = pd.DataFrame( + {"person_id": household_ids, "person_household_id": household_ids} + ) + frame = Frame( + {"person": person, "household": household}, + _SCHEMA, + {"household": Weights(values=weights, kind=WeightKind.DESIGN)}, + ) + targets = TargetSet( + [ + Target( + name=name, + period=2024, + entity="household", + measure=name, + # An invented 8% miss, so the optimizer has somewhere to go. + value=float((values * weights).sum()) * 1.08, + ) + for name, values in columns.items() + ] + ) + return frame, targets + + +def solver_overhead() -> list[dict[str, object]]: + """Added wall-clock from snapshot emission, at three cadences. + + One discarded warm-up run first (the first calibrate of a process pays + matrix-build and torch warm-up that would otherwise be charged to whichever + cadence happened to run first), then the best of ``REPEATS`` timed runs per + cadence. Each run's returned weights are compared bitwise against the + observer-off run, so the table also evidences that emission changed nothing. + """ + frame, targets = _synthetic_frame_and_targets() + cadences = ( + ("observer_off", None), + ("bounded_every_25", TargetSnapshotCadence(every=25)), + ("every_epoch", TargetSnapshotCadence(every=EVERY_EPOCH)), + ) + calibrate(frame, targets, epochs=SOLVER_EPOCHS, seed=0) # warm-up, discarded + + rows: list[dict[str, object]] = [] + reference: np.ndarray | None = None + for label, cadence in cadences: + best_seconds = float("inf") + emitted = retained_chunks = retained_bytes = 0 + for _ in range(REPEATS): + with tempfile.TemporaryDirectory() as directory: + observer = ( + None + if cadence is None + else TargetSnapshotObserver( + sink=TargetSnapshotWriter(Path(directory)), + run_id="synthetic", + cadence=cadence, + ) + ) + start = time.perf_counter() + result = calibrate( + frame, + targets, + epochs=SOLVER_EPOCHS, + seed=0, + target_snapshots=observer, + ) + best_seconds = min(best_seconds, time.perf_counter() - start) + history = Path(directory) / "history" + if observer is not None: + retained_chunks = len(list(history.glob("*.json"))) + retained_bytes = sum( + path.stat().st_size for path in history.glob("*.json") + ) + emitted = int( + json.loads( + (Path(directory) / "history_index.json").read_text( + encoding="utf-8" + ) + )["last_sequence"] + ) + if reference is None: + reference = result.weights.copy() + identical = True + else: + identical = bool(np.array_equal(reference, result.weights)) + rows.append( + { + "cadence": label, + f"best_of_{REPEATS}_seconds": round(best_seconds, 4), + "closing_loss": float(result.closing_loss), + "weights_bitwise_equal_to_observer_off": identical, + "snapshots_emitted": emitted or None, + "chunks_retained_on_disk": retained_chunks, + "retained_bytes_on_disk": retained_bytes, + } + ) + return rows + + +def main() -> int: + report = { + "label": "SYNTHETIC — invented matrices and frames, not US or UK calibration", + "dimension_points": { + "codec_target_counts": list(TARGET_COUNTS), + "solver_records": SOLVER_RECORDS, + "solver_targets": SOLVER_TARGETS, + "solver_epochs": SOLVER_EPOCHS, + }, + "bounds": { + "numeric_threads": torch.get_num_threads(), + "cpu_seconds_limit_applied": _CPU_LIMIT, + "memory_bytes_limit_applied": _MEMORY_LIMIT, + "peak_rss_bytes": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + "native_inputs_read": 0, + }, + "codec_costs": codec_costs(), + "solver_overhead": solver_overhead(), + } + json.dump(report, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/acs-serial-membership/README.md b/experiments/acs-serial-membership/README.md new file mode 100644 index 000000000..c375634da --- /dev/null +++ b/experiments/acs-serial-membership/README.md @@ -0,0 +1,63 @@ +# Reuse ACS serial membership lookups + +This native-lane continuation starts at `9af56aa8a`. It prepares each exact +string-key lookup once per person-archive read, then checks every incoming +chunk against that lookup. The pandas Arrow path previously rebuilt its +complete comparison set for each chunk. The completed native attribution +sample assigned about 108 cold / 119 replay CPU seconds to that operation; +these are statistical estimates, not function timers or promised savings. + +An object-dtype pandas Index retains its hash table for the duration of the +archive call. Missing chunk values remain unmatched because the optimized +keys are exact strings. Any mixed key set or other Series dtype uses the +original pandas `isin` path. Age validation, complete orphan checking, +household roster validation and selection order are unchanged. No source row, +archive parse, validation result or issued authority is cached. The native +coverage owner explicitly pins the reviewed reader hash; altered source bytes +still refuse with `UNREVIEWED_PREPARATION`. + +## Evidence + +- `tests-with-owner.txt`: 185 passes, covering the new differential tests, + existing ACS PUMS behavior, coverage authentication and source compilation. +- Differential cases include both pandas string storage backends, both missing + policies, mixed/non-string fallback keys, duplicate indices, names, Unicode, + literal leading-zero IDs, chunk changes and exact orphan/age error messages. +- Actual native `_producer()` admits the reviewed implementation; a separate + changed-source-byte test confirms the existing refusal remains active. +- `ci-inventory.txt`: the new flat test file is included in the CI inventory. +- Ruff check/format and `git diff --check` pass. + +`benchmark.py` uses invented arrays only: 200,000 comparison keys and twenty +20,000-row chunks on pandas 3.0.3 / PyArrow 25.0.0. Separate processes produced +`original.json` and `prepared.json`, and every result matched literal pandas +`isin`. Including preparation, timed CPU was 6.691459 seconds original and +0.052461 seconds prepared. The prepared first lookup took 0.012154 CPU seconds +and reused the same index engine for the remaining nineteen calls. + +This is a focused microbenchmark, not a native build measurement. The prepared +Index's deep estimate is 17,627,112 bytes including Python strings shared with +the existing key set; it is not incremental RSS. The process RSS observations +include imports and the untimed pandas oracle. The optimization adds a local +O(key-count) index and O(rows-per-chunk) temporary lookup buffers, and does not +establish lower memory use or full-source capacity. Key preparation also occurs +before archive/member validation, so malformed archives can incur that bounded +extra allocation before the same validation refusal. + +## Reproduce + +The recorded environment is the existing native-lane Python 3.14.4 virtual +environment, reused without installs or lock changes. All six Microcosm source +shards are explicitly loaded from this checkout; `benchmark.py` checks its +reader import origin. Run it with isolated Python (`-I -B`), then choose +`original` or `prepared`. No real archive or population data is read. + +The first test-launch attempt lacked `-I` and encountered an unrelated +`/private/tmp/dis.py` shadowing the standard library before test collection. +Isolated Python corrected the harness. The subsequent expected red test failed +because the lookup helpers did not yet exist; the final suite above passes. + +This branch changes producer-bound provenance and may change graph keys. +Compare all/compact retention at the same combined source revision, and record +cross-revision semantic equivalence separately. No actual native replay, +certification, default change, merge or publication is claimed by this change. diff --git a/experiments/acs-serial-membership/benchmark.py b/experiments/acs-serial-membership/benchmark.py new file mode 100644 index 000000000..b6a2a6700 --- /dev/null +++ b/experiments/acs-serial-membership/benchmark.py @@ -0,0 +1,101 @@ +"""Bounded invented membership timing; never reads Census records or archives.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import resource +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(path) for path in sorted((ROOT / "packages").glob("*/src"))] + +import pandas as pd # noqa: E402 +import psutil # noqa: E402 +import pyarrow # noqa: E402 + +from microcosm.build.us_runtime import acs_pums # noqa: E402 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("original", "prepared")) + args = parser.parse_args() + assert Path(acs_pums.__file__).resolve().is_relative_to(ROOT) + process = psutil.Process() + # Key count matters: the old Arrow path rebuilds this set for every chunk. + keys = frozenset(f"invented-{i:09d}" for i in range(200_000)) + values = pd.Series( + [f"invented-{i:09d}" for i in range(15_000)] + ["absent", None] * 2_500, + dtype=pd.StringDtype(storage="pyarrow"), + name="SERIALNO", + ) + expected = values.isin(keys) + rss_before = process.memory_info().rss + started = time.perf_counter() + cpu = time.process_time() + lookup = acs_pums._serial_lookup(keys) if args.mode == "prepared" else None + preparation = { + "wall_seconds": time.perf_counter() - started, + "cpu_seconds": time.process_time() - cpu, + } + timings = [] + engine = None + for i in range(20): + started, cpu = time.perf_counter(), time.process_time() + actual = ( + values.isin(keys) + if args.mode == "original" + else acs_pums._serial_isin(values, keys, lookup) + ) + timings.append( + { + "wall_seconds": time.perf_counter() - started, + "cpu_seconds": time.process_time() - cpu, + } + ) + pd.testing.assert_series_equal(actual, expected) + if lookup is not None: + if i == 0: + engine = lookup._engine + assert lookup._engine is engine + result = { + "scope": "invented_arrays_only; not native preparation performance", + "mode": args.mode, + "keys": len(keys), + "rows_per_chunk": len(values), + "chunks": len(timings), + "all_masks_match_pandas": True, + "prepared_index_engine_reused": lookup is not None, + "preparation": preparation, + "first_lookup": timings[0], + "warm_lookups": timings[1:], + "total_cpu_seconds": preparation["cpu_seconds"] + + sum(t["cpu_seconds"] for t in timings), + "total_wall_seconds": preparation["wall_seconds"] + + sum(t["wall_seconds"] for t in timings), + "rss_before_lookup_bytes": rss_before, + "rss_after_lookups_bytes": process.memory_info().rss, + "whole_process_peak_rss_bytes_macos": resource.getrusage( + resource.RUSAGE_SELF + ).ru_maxrss, + # Includes shared Python strings: this is not net incremental RSS. + "index_deep_bytes_including_shared_strings": None + if lookup is None + else lookup.memory_usage(deep=True), + "pandas": pd.__version__, + "pyarrow": pyarrow.__version__, + "python": sys.version, + "module_sha256": hashlib.sha256( + Path(acs_pums.__file__).read_bytes() + ).hexdigest(), + "benchmark_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + } + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/experiments/acs-serial-membership/ci-inventory.txt b/experiments/acs-serial-membership/ci-inventory.txt new file mode 100644 index 000000000..4364f99e7 --- /dev/null +++ b/experiments/acs-serial-membership/ci-inventory.txt @@ -0,0 +1,1740 @@ +tracked_test_files=557 +[fast] +trade: files=7 procs=main shards_per_process=1 + packages/microcosm-build/tests/test_us_trade_census_imports.py + packages/microcosm-build/tests/test_us_trade_crosscheck.py + packages/microcosm-build/tests/test_us_trade_entries_cli.py + packages/microcosm-build/tests/test_us_trade_entry_generator.py + packages/microcosm-build/tests/test_us_trade_facts.py + packages/microcosm-build/tests/test_us_trade_imdb_bulk.py + packages/microcosm-build/tests/test_us_trade_imdb_goldens.py +spine-uk: files=138 procs=main shards_per_process=1 + packages/microcosm-build/tests/test_uk_age_tail.py + packages/microcosm-build/tests/test_uk_atomic_area_support.py + packages/microcosm-build/tests/test_uk_atomic_household_lineage.py + packages/microcosm-build/tests/test_uk_battery_bindings.py + packages/microcosm-build/tests/test_uk_brma_resource.py + packages/microcosm-build/tests/test_uk_bus_targets.py + packages/microcosm-build/tests/test_uk_bus_use_incidence.py + packages/microcosm-build/tests/test_uk_calibration_run.py + packages/microcosm-build/tests/test_uk_calibration_seam_driver.py + packages/microcosm-build/tests/test_uk_cgt_calibration.py + packages/microcosm-build/tests/test_uk_cgt_imputation.py + packages/microcosm-build/tests/test_uk_cgt_observation_period.py + packages/microcosm-build/tests/test_uk_cgt_source_manifest.py + packages/microcosm-build/tests/test_uk_cgt_structure.py + packages/microcosm-build/tests/test_uk_cgt_targets.py + packages/microcosm-build/tests/test_uk_childcare_targets.py + packages/microcosm-build/tests/test_uk_chronicle_feed.py + packages/microcosm-build/tests/test_uk_consumption_resources.py + packages/microcosm-build/tests/test_uk_content_identity.py + packages/microcosm-build/tests/test_uk_corrected_measurement_receipt.py + packages/microcosm-build/tests/test_uk_council_tax_targets.py + packages/microcosm-build/tests/test_uk_data_target_parity.py + packages/microcosm-build/tests/test_uk_dense_release_assembler.py + packages/microcosm-build/tests/test_uk_diagnostics.py + packages/microcosm-build/tests/test_uk_donor_uprating.py + packages/microcosm-build/tests/test_uk_e5_identity.py + packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py + packages/microcosm-build/tests/test_uk_energy_pricing.py + packages/microcosm-build/tests/test_uk_etb_services.py + packages/microcosm-build/tests/test_uk_etb_vat.py + packages/microcosm-build/tests/test_uk_evaluate_dataset_size_tool.py + packages/microcosm-build/tests/test_uk_fact_raking.py + packages/microcosm-build/tests/test_uk_firm_generation.py + packages/microcosm-build/tests/test_uk_firms_targets.py + packages/microcosm-build/tests/test_uk_frame_linkage_preflight.py + packages/microcosm-build/tests/test_uk_frs_council_tax.py + packages/microcosm-build/tests/test_uk_frs_disability.py + packages/microcosm-build/tests/test_uk_frs_education.py + packages/microcosm-build/tests/test_uk_frs_education_grants.py + packages/microcosm-build/tests/test_uk_frs_employment.py + packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py + packages/microcosm-build/tests/test_uk_frs_legacy_proxies.py + packages/microcosm-build/tests/test_uk_frs_relationships.py + packages/microcosm-build/tests/test_uk_frs_release.py + packages/microcosm-build/tests/test_uk_frs_spine.py + packages/microcosm-build/tests/test_uk_frs_stochastic.py + packages/microcosm-build/tests/test_uk_geography_ladder.py + packages/microcosm-build/tests/test_uk_geography_sources.py + packages/microcosm-build/tests/test_uk_graph.py + packages/microcosm-build/tests/test_uk_h5_payload_compare.py + packages/microcosm-build/tests/test_uk_hmrc_calibration.py + packages/microcosm-build/tests/test_uk_hmrc_capital_gains.py + packages/microcosm-build/tests/test_uk_hmrc_income.py + packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py + packages/microcosm-build/tests/test_uk_hmrc_replay.py + packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py + packages/microcosm-build/tests/test_uk_identity_stability_receipts.py + packages/microcosm-build/tests/test_uk_incumbent_surface_evaluation.py + packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py + packages/microcosm-build/tests/test_uk_ladder_sni_sources.py + packages/microcosm-build/tests/test_uk_ladder_targets.py + packages/microcosm-build/tests/test_uk_lcfs_consumption.py + packages/microcosm-build/tests/test_uk_ledger_fact_vendoring.py + packages/microcosm-build/tests/test_uk_ledger_targets.py + packages/microcosm-build/tests/test_uk_local_area_crosswalk.py + packages/microcosm-build/tests/test_uk_local_candidate_scorer.py + packages/microcosm-build/tests/test_uk_local_doctrine.py + packages/microcosm-build/tests/test_uk_local_gate_battery.py + packages/microcosm-build/tests/test_uk_local_geography.py + packages/microcosm-build/tests/test_uk_local_holdout.py + packages/microcosm-build/tests/test_uk_local_incumbent_extractor.py + packages/microcosm-build/tests/test_uk_local_release_preflight.py + packages/microcosm-build/tests/test_uk_local_rowwise.py + packages/microcosm-build/tests/test_uk_local_target_census.py + packages/microcosm-build/tests/test_uk_local_targets.py + packages/microcosm-build/tests/test_uk_local_validation_levels.py + packages/microcosm-build/tests/test_uk_measure_simulation.py + packages/microcosm-build/tests/test_uk_national_calibration.py + packages/microcosm-build/tests/test_uk_national_frame.py + packages/microcosm-build/tests/test_uk_national_sampling.py + packages/microcosm-build/tests/test_uk_nhs_allocation.py + packages/microcosm-build/tests/test_uk_ods_tables.py + packages/microcosm-build/tests/test_uk_parity_reference.py + packages/microcosm-build/tests/test_uk_payload_diff_classifier.py + packages/microcosm-build/tests/test_uk_pipr_deferrals.py + packages/microcosm-build/tests/test_uk_population_targets.py + packages/microcosm-build/tests/test_uk_provenance_flags.py + packages/microcosm-build/tests/test_uk_raking.py + packages/microcosm-build/tests/test_uk_regional_uprating.py + packages/microcosm-build/tests/test_uk_release_assembler.py + packages/microcosm-build/tests/test_uk_release_certification.py + packages/microcosm-build/tests/test_uk_release_identity.py + packages/microcosm-build/tests/test_uk_release_input_coverage.py + packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py + packages/microcosm-build/tests/test_uk_rowwise_build_driver.py + packages/microcosm-build/tests/test_uk_rowwise_candidate.py + packages/microcosm-build/tests/test_uk_rowwise_dataset.py + packages/microcosm-build/tests/test_uk_rowwise_dry_run.py + packages/microcosm-build/tests/test_uk_rowwise_geography.py + packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py + packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py + packages/microcosm-build/tests/test_uk_salary_sacrifice.py + packages/microcosm-build/tests/test_uk_signed_differences.py + packages/microcosm-build/tests/test_uk_size_evaluation.py + packages/microcosm-build/tests/test_uk_source_runtime.py + packages/microcosm-build/tests/test_uk_source_stages.py + packages/microcosm-build/tests/test_uk_spi_income.py + packages/microcosm-build/tests/test_uk_spi_spine.py + packages/microcosm-build/tests/test_uk_spi_support.py + packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py + packages/microcosm-build/tests/test_uk_spine_parity_instrument.py + packages/microcosm-build/tests/test_uk_stage_checkpoints.py + packages/microcosm-build/tests/test_uk_stage_health.py + packages/microcosm-build/tests/test_uk_staging_integration.py + packages/microcosm-build/tests/test_uk_staging_provisioning.py + packages/microcosm-build/tests/test_uk_stochastic_tools.py + packages/microcosm-build/tests/test_uk_student_loans.py + packages/microcosm-build/tests/test_uk_take_up_contract.py + packages/microcosm-build/tests/test_uk_take_up_gate.py + packages/microcosm-build/tests/test_uk_target_references.py + packages/microcosm-build/tests/test_uk_terminal_gates.py + packages/microcosm-build/tests/test_uk_tfc_routed_share.py + packages/microcosm-build/tests/test_uk_uc_capital_coherence.py + packages/microcosm-build/tests/test_uk_uc_deduction_attributes.py + packages/microcosm-build/tests/test_uk_uc_family.py + packages/microcosm-build/tests/test_uk_uc_lp_evidence.py + packages/microcosm-build/tests/test_uk_uc_matrix_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_relationships.py + packages/microcosm-build/tests/test_uk_uc_reporter_redraw.py + packages/microcosm-build/tests/test_uk_uc_source_periods.py + packages/microcosm-build/tests/test_uk_uc_support_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_target_measurements.py + packages/microcosm-build/tests/test_uk_was_wealth.py + packages/microcosm-build/tests/test_uk_wealth_resources.py + packages/microcosm-build/tests/test_uk_weighted_integrity.py + packages/microcosm-build/tests/test_uk_weighted_integrity_measurement.py + packages/microcosm-build/tests/test_us_stacked_spine.py + packages/microcosm-frame/tests/test_policyengine_uk_adapter.py +rest: files=410 procs=main shards_per_process=6 + packages/microcosm-build/tests/test_atomic_geography.py + packages/microcosm-build/tests/test_chronicle_epoch.py + packages/microcosm-build/tests/test_ci_test_groups.py + packages/microcosm-build/tests/test_compare_us_fiscal_diagnostics.py + packages/microcosm-build/tests/test_country_spec.py + packages/microcosm-build/tests/test_cross_grain.py + packages/microcosm-build/tests/test_demographics.py + packages/microcosm-build/tests/test_diagnose_us_target_support.py + packages/microcosm-build/tests/test_frame_checkpoint.py + packages/microcosm-build/tests/test_frame_sampling.py + packages/microcosm-build/tests/test_frame_serializer_registry.py + packages/microcosm-build/tests/test_gate_battery.py + packages/microcosm-build/tests/test_gate_battery_contract_example.py + packages/microcosm-build/tests/test_gate_battery_contract_pins.py + packages/microcosm-build/tests/test_gates.py + packages/microcosm-build/tests/test_holdout.py + packages/microcosm-build/tests/test_imputation_lineage_spec.py + packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py + packages/microcosm-build/tests/test_ledger_artifact.py + packages/microcosm-build/tests/test_ledger_targets.py + packages/microcosm-build/tests/test_logbook.py + packages/microcosm-build/tests/test_logbook_adoption.py + packages/microcosm-build/tests/test_logbook_archive.py + packages/microcosm-build/tests/test_logbook_backfill.py + packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py + packages/microcosm-build/tests/test_logbook_cli.py + packages/microcosm-build/tests/test_logbook_env.py + packages/microcosm-build/tests/test_logbook_isolation.py + packages/microcosm-build/tests/test_logbook_prediction_seed.py + packages/microcosm-build/tests/test_monetary_profile.py + packages/microcosm-build/tests/test_monetary_targets.py + packages/microcosm-build/tests/test_outer_stage_runtime.py + packages/microcosm-build/tests/test_plan.py + packages/microcosm-build/tests/test_puf_qrf_chain.py + packages/microcosm-build/tests/test_reform_validation.py + packages/microcosm-build/tests/test_release_input_coverage.py + packages/microcosm-build/tests/test_release_target_parity.py + packages/microcosm-build/tests/test_score_uk_national_candidate.py + packages/microcosm-build/tests/test_source_runtime.py + packages/microcosm-build/tests/test_spec_engine_battery_semantics.py + packages/microcosm-build/tests/test_spec_engine_calibration_semantics.py + packages/microcosm-build/tests/test_spec_engine_compiler_ir.py + packages/microcosm-build/tests/test_spec_engine_country_bundles.py + packages/microcosm-build/tests/test_spec_engine_coverage_tool.py + packages/microcosm-build/tests/test_spec_engine_engine_abi.py + packages/microcosm-build/tests/test_spec_engine_field_usage.py + packages/microcosm-build/tests/test_spec_engine_identity_contracts.py + packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py + packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py + packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py + packages/microcosm-build/tests/test_spec_engine_loader.py + packages/microcosm-build/tests/test_spec_engine_plan_lock.py + packages/microcosm-build/tests/test_spec_engine_publication_semantics.py + packages/microcosm-build/tests/test_spec_engine_reemission.py + packages/microcosm-build/tests/test_spec_engine_resolver.py + packages/microcosm-build/tests/test_spec_engine_schema.py + packages/microcosm-build/tests/test_spec_engine_seeds.py + packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py + packages/microcosm-build/tests/test_spec_engine_take_up_semantics.py + packages/microcosm-build/tests/test_spec_engine_typed_closure.py + packages/microcosm-build/tests/test_spec_engine_yaml.py + packages/microcosm-build/tests/test_spec_only_country_packages.py + packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py + packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py + packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py + packages/microcosm-build/tests/test_spec_seed_identity_lock.py + packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py + packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py + packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py + packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py + packages/microcosm-build/tests/test_spec_worker_identity_lock.py + packages/microcosm-build/tests/test_stage_profile.py + packages/microcosm-build/tests/test_staging.py + packages/microcosm-build/tests/test_staging_v2.py + packages/microcosm-build/tests/test_stochastic_assignment.py + packages/microcosm-build/tests/test_target_materialization.py + packages/microcosm-build/tests/test_target_reference_authoring.py + packages/microcosm-build/tests/test_trace.py + packages/microcosm-build/tests/test_us_acs_housing_source.py + packages/microcosm-build/tests/test_us_acs_inputs.py + packages/microcosm-build/tests/test_us_acs_local_release_tool.py + packages/microcosm-build/tests/test_us_acs_multispine.py + packages/microcosm-build/tests/test_us_acs_multispine_base_builder.py + packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py + packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py + packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py + packages/microcosm-build/tests/test_us_acs_population_catalogue.py + packages/microcosm-build/tests/test_us_acs_pums.py + packages/microcosm-build/tests/test_us_acs_record_fence_scan.py + packages/microcosm-build/tests/test_us_acs_release_predictors.py + packages/microcosm-build/tests/test_us_acs_source_compile_cache.py + packages/microcosm-build/tests/test_us_acs_sources.py + packages/microcosm-build/tests/test_us_acs_transfer.py + packages/microcosm-build/tests/test_us_adult_care.py + packages/microcosm-build/tests/test_us_alimony.py + packages/microcosm-build/tests/test_us_asec_2024_native_population.py + packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py + packages/microcosm-build/tests/test_us_asec_checkpoint.py + packages/microcosm-build/tests/test_us_asec_coverage_authentication.py + packages/microcosm-build/tests/test_us_asec_current_money_source.py + packages/microcosm-build/tests/test_us_asec_demographic_source.py + packages/microcosm-build/tests/test_us_asec_household_coverage_fields.py + packages/microcosm-build/tests/test_us_asec_person_income_source.py + packages/microcosm-build/tests/test_us_asec_pool.py + packages/microcosm-build/tests/test_us_asec_prepared_resources.py + packages/microcosm-build/tests/test_us_asec_source_domain_hardening.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_de.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_national.py + packages/microcosm-build/tests/test_us_atomic_block_sources.py + packages/microcosm-build/tests/test_us_atomic_block_support.py + packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py + packages/microcosm-build/tests/test_us_base_pool.py + packages/microcosm-build/tests/test_us_block_ladder_sources.py + packages/microcosm-build/tests/test_us_buildj_base_script.py + packages/microcosm-build/tests/test_us_bundle_core_contracts.py + packages/microcosm-build/tests/test_us_capital_gain_details.py + packages/microcosm-build/tests/test_us_capital_gain_distributions.py + packages/microcosm-build/tests/test_us_casualty_losses.py + packages/microcosm-build/tests/test_us_child_property_income_graph.py + packages/microcosm-build/tests/test_us_child_property_income_graph_owner.py + packages/microcosm-build/tests/test_us_child_property_income_source.py + packages/microcosm-build/tests/test_us_child_property_income_source_owner.py + packages/microcosm-build/tests/test_us_child_support.py + packages/microcosm-build/tests/test_us_childcare.py + packages/microcosm-build/tests/test_us_common_frame_export_contract.py + packages/microcosm-build/tests/test_us_completion_support_custody.py + packages/microcosm-build/tests/test_us_congressional_district_geography.py + packages/microcosm-build/tests/test_us_congressional_district_vintage.py + packages/microcosm-build/tests/test_us_congressional_district_vintage_crosswalk.py + packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py + packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py + packages/microcosm-build/tests/test_us_current_asec_child_support_source.py + packages/microcosm-build/tests/test_us_current_asec_demographics.py + packages/microcosm-build/tests/test_us_current_asec_dividend_source.py + packages/microcosm-build/tests/test_us_current_asec_income_routing.py + packages/microcosm-build/tests/test_us_current_asec_interest_source.py + packages/microcosm-build/tests/test_us_current_asec_property_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py + packages/microcosm-build/tests/test_us_current_property_income_sources.py + packages/microcosm-build/tests/test_us_current_survey_amounts.py + packages/microcosm-build/tests/test_us_current_survey_geography.py + packages/microcosm-build/tests/test_us_current_survey_health_coverage.py + packages/microcosm-build/tests/test_us_current_survey_household_roles.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_graph.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_source.py + packages/microcosm-build/tests/test_us_current_survey_housing.py + packages/microcosm-build/tests/test_us_current_survey_person_status.py + packages/microcosm-build/tests/test_us_current_survey_person_status_graph.py + packages/microcosm-build/tests/test_us_current_survey_person_status_source.py + packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py + packages/microcosm-build/tests/test_us_current_survey_predictors.py + packages/microcosm-build/tests/test_us_current_survey_property_graph.py + packages/microcosm-build/tests/test_us_current_survey_puf_host.py + packages/microcosm-build/tests/test_us_current_survey_puf_transfer.py + packages/microcosm-build/tests/test_us_dc_ptc_take_up_exclusion.py + packages/microcosm-build/tests/test_us_disability_benefits.py + packages/microcosm-build/tests/test_us_documented_absent_inputs.py + packages/microcosm-build/tests/test_us_domestic_production.py + packages/microcosm-build/tests/test_us_early_head_start_exclusion.py + packages/microcosm-build/tests/test_us_education_assistance_source.py + packages/microcosm-build/tests/test_us_education_inputs.py + packages/microcosm-build/tests/test_us_educator_expenses.py + packages/microcosm-build/tests/test_us_eligibility_inputs.py + packages/microcosm-build/tests/test_us_energy_subsidy.py + packages/microcosm-build/tests/test_us_exact_k_ladder.py + packages/microcosm-build/tests/test_us_exact_k_ladder_e2e.py + packages/microcosm-build/tests/test_us_exact_k_ladder_launcher.py + packages/microcosm-build/tests/test_us_farm_business_income.py + packages/microcosm-build/tests/test_us_financial_assistance_exclusion.py + packages/microcosm-build/tests/test_us_fiscal_leaf_policy.py + packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py + packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py + packages/microcosm-build/tests/test_us_fiscal_targets.py + packages/microcosm-build/tests/test_us_form_4952.py + packages/microcosm-build/tests/test_us_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_full_puf_output_profiles.py + packages/microcosm-build/tests/test_us_geography_integration.py + packages/microcosm-build/tests/test_us_geography_ladder.py + packages/microcosm-build/tests/test_us_graph.py + packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py + packages/microcosm-build/tests/test_us_graph_atomic_person_status.py + packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py + packages/microcosm-build/tests/test_us_graph_current_survey_geography.py + packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py + packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py + packages/microcosm-build/tests/test_us_graph_fiscal_target_snapshots.py + packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_selection.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_stream.py + packages/microcosm-build/tests/test_us_graph_property_income.py + packages/microcosm-build/tests/test_us_graph_property_income_receipts.py + packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py + packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py + packages/microcosm-build/tests/test_us_graph_survey_completion.py + packages/microcosm-build/tests/test_us_graph_survey_population.py + packages/microcosm-build/tests/test_us_graph_survey_puf55.py + packages/microcosm-build/tests/test_us_graph_us_survey_enrichment.py + packages/microcosm-build/tests/test_us_hours_worked.py + packages/microcosm-build/tests/test_us_housing_inputs.py + packages/microcosm-build/tests/test_us_housing_participation.py + packages/microcosm-build/tests/test_us_immigration.py + packages/microcosm-build/tests/test_us_input_mass.py + packages/microcosm-build/tests/test_us_l0_refit_export.py + packages/microcosm-build/tests/test_us_late_producer_dag.py + packages/microcosm-build/tests/test_us_medicaid_take_up.py + packages/microcosm-build/tests/test_us_medicare_take_up.py + packages/microcosm-build/tests/test_us_misc_itemized.py + packages/microcosm-build/tests/test_us_multispine_pool.py + packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py + packages/microcosm-build/tests/test_us_multispine_puf_clone.py + packages/microcosm-build/tests/test_us_national_age_counts.py + packages/microcosm-build/tests/test_us_native_row_ceilings.py + packages/microcosm-build/tests/test_us_native_verify_once_epoch.py + packages/microcosm-build/tests/test_us_nonzero_shares.py + packages/microcosm-build/tests/test_us_org_wages.py + packages/microcosm-build/tests/test_us_other_health_insurance.py + packages/microcosm-build/tests/test_us_parity_reference.py + packages/microcosm-build/tests/test_us_plan.py + packages/microcosm-build/tests/test_us_pool_input_consumers.py + packages/microcosm-build/tests/test_us_population_input_coverage.py + packages/microcosm-build/tests/test_us_post_transfer_calibration.py + packages/microcosm-build/tests/test_us_postclone_geography_identity.py + packages/microcosm-build/tests/test_us_pregnancy.py + packages/microcosm-build/tests/test_us_prior_year_income.py + packages/microcosm-build/tests/test_us_property_completion_graph.py + packages/microcosm-build/tests/test_us_property_completion_routing.py + packages/microcosm-build/tests/test_us_puf55_heterogeneous_manifest_seals.py + packages/microcosm-build/tests/test_us_puf55_model_boundaries.py + packages/microcosm-build/tests/test_us_puf55_route_finalization.py + packages/microcosm-build/tests/test_us_puf55_route_numerical_finalization.py + packages/microcosm-build/tests/test_us_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_profile.py + packages/microcosm-build/tests/test_us_puf59_canonical.py + packages/microcosm-build/tests/test_us_puf_aggregate_records.py + packages/microcosm-build/tests/test_us_puf_capital_gains_tail.py + packages/microcosm-build/tests/test_us_puf_detail_transfer.py + packages/microcosm-build/tests/test_us_puf_donor_io.py + packages/microcosm-build/tests/test_us_puf_e01000_reconciliation.py + packages/microcosm-build/tests/test_us_puf_e19200_decomposition.py + packages/microcosm-build/tests/test_us_puf_price_baseline.py + packages/microcosm-build/tests/test_us_puf_support.py + packages/microcosm-build/tests/test_us_puf_support_base_builder.py + packages/microcosm-build/tests/test_us_puf_support_base_equivalence.py + packages/microcosm-build/tests/test_us_puf_tail_bound.py + packages/microcosm-build/tests/test_us_puf_target2024_growth.py + packages/microcosm-build/tests/test_us_puma_ladder.py + packages/microcosm-build/tests/test_us_puma_ladder_sources.py + packages/microcosm-build/tests/test_us_qbi_inputs.py + packages/microcosm-build/tests/test_us_relationship_inputs.py + packages/microcosm-build/tests/test_us_release_gate_preflight.py + packages/microcosm-build/tests/test_us_reported_coverage_vintage_gate.py + packages/microcosm-build/tests/test_us_retirement_contributions.py + packages/microcosm-build/tests/test_us_retirement_distributions.py + packages/microcosm-build/tests/test_us_runtime_facade_union.py + packages/microcosm-build/tests/test_us_runtime_import_order.py + packages/microcosm-build/tests/test_us_salt_refund_income.py + packages/microcosm-build/tests/test_us_scf_auto_loans.py + packages/microcosm-build/tests/test_us_scf_wealth.py + packages/microcosm-build/tests/test_us_sipp_financial_assets.py + packages/microcosm-build/tests/test_us_sipp_head_start.py + packages/microcosm-build/tests/test_us_sipp_tips.py + packages/microcosm-build/tests/test_us_sipp_vehicles.py + packages/microcosm-build/tests/test_us_snap_discretionary_exemption.py + packages/microcosm-build/tests/test_us_snap_state_take_up.py + packages/microcosm-build/tests/test_us_snap_take_up.py + packages/microcosm-build/tests/test_us_source_coverage.py + packages/microcosm-build/tests/test_us_source_runtime.py + packages/microcosm-build/tests/test_us_spec_bundle.py + packages/microcosm-build/tests/test_us_spine_agreement.py + packages/microcosm-build/tests/test_us_spine_assembly.py + packages/microcosm-build/tests/test_us_spine_blindness.py + packages/microcosm-build/tests/test_us_spm_resources.py + packages/microcosm-build/tests/test_us_spm_role_enrichment_builder.py + packages/microcosm-build/tests/test_us_spm_role_source.py + packages/microcosm-build/tests/test_us_ssi_disability_criteria.py + packages/microcosm-build/tests/test_us_ssi_take_up.py + packages/microcosm-build/tests/test_us_state_files_scorer.py + packages/microcosm-build/tests/test_us_survey_age_activation.py + packages/microcosm-build/tests/test_us_survey_age_artifact.py + packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_age_calibration_run.py + packages/microcosm-build/tests/test_us_survey_age_development.py + packages/microcosm-build/tests/test_us_survey_age_sources.py + packages/microcosm-build/tests/test_us_survey_calibration.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py + packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py + packages/microcosm-build/tests/test_us_survey_financial_successor.py + packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py + packages/microcosm-build/tests/test_us_survey_observed_age.py + packages/microcosm-build/tests/test_us_survey_origin_budget.py + packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_origin_budget_detached_views.py + packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py + packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py + packages/microcosm-build/tests/test_us_survey_population_preparation.py + packages/microcosm-build/tests/test_us_survey_population_replay.py + packages/microcosm-build/tests/test_us_survey_social_security.py + packages/microcosm-build/tests/test_us_survivor_benefits_exclusion.py + packages/microcosm-build/tests/test_us_take_up.py + packages/microcosm-build/tests/test_us_take_up_contract.py + packages/microcosm-build/tests/test_us_voluntary_filing.py + packages/microcosm-build/tests/test_us_warm_start_selection.py + packages/microcosm-build/tests/test_us_weeks_unemployed.py + packages/microcosm-build/tests/test_us_whole_series_selection_sites.py + packages/microcosm-build/tests/test_us_wic_claim.py + packages/microcosm-build/tests/test_us_wic_nutritional_risk_exclusion.py + packages/microcosm-build/tests/test_us_workers_compensation.py + packages/microcosm-build/tests/test_validation_input_coverage.py + packages/microcosm-calibrate/tests/test_compat.py + packages/microcosm-calibrate/tests/test_diagnostics.py + packages/microcosm-calibrate/tests/test_exact_k.py + packages/microcosm-calibrate/tests/test_group_bounds.py + packages/microcosm-calibrate/tests/test_grouped_fixed_support.py + packages/microcosm-calibrate/tests/test_hierarchy.py + packages/microcosm-calibrate/tests/test_informed_gates.py + packages/microcosm-calibrate/tests/test_kernels.py + packages/microcosm-calibrate/tests/test_matrix.py + packages/microcosm-calibrate/tests/test_monetary_binding.py + packages/microcosm-calibrate/tests/test_provider_labels.py + packages/microcosm-calibrate/tests/test_registry.py + packages/microcosm-calibrate/tests/test_score.py + packages/microcosm-calibrate/tests/test_solve.py + packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py + packages/microcosm-calibrate/tests/test_sparse_solve.py + packages/microcosm-calibrate/tests/test_target.py + packages/microcosm-calibrate/tests/test_target_snapshots.py + packages/microcosm-calibrate/tests/test_variable_labels.py + packages/microcosm-data/tests/test_compat.py + packages/microcosm-data/tests/test_contract.py + packages/microcosm-data/tests/test_h5_enrichment.py + packages/microcosm-data/tests/test_loader.py + packages/microcosm-data/tests/test_local_area_contract.py + packages/microcosm-data/tests/test_publish_guard.py + packages/microcosm-data/tests/test_registry.py + packages/microcosm-data/tests/test_release.py + packages/microcosm-data/tests/test_slack.py + packages/microcosm-data/tests/test_source_enrichment.py + packages/microcosm-data/tests/test_uk_dense_release_contract.py + packages/microcosm-fit/tests/test_cache.py + packages/microcosm-fit/tests/test_compat.py + packages/microcosm-fit/tests/test_dataframe_fit.py + packages/microcosm-fit/tests/test_graph_joint_empirical.py + packages/microcosm-fit/tests/test_graph_legacy_matrix.py + packages/microcosm-fit/tests/test_graph_legacy_qrf.py + packages/microcosm-fit/tests/test_graph_signed_reconciliation.py + packages/microcosm-fit/tests/test_joint_empirical.py + packages/microcosm-fit/tests/test_kernels.py + packages/microcosm-fit/tests/test_model.py + packages/microcosm-fit/tests/test_qrf.py + packages/microcosm-fit/tests/test_qrf_stateless.py + packages/microcosm-fit/tests/test_qrf_target.py + packages/microcosm-fit/tests/test_signed_reconciliation.py + packages/microcosm-fit/tests/test_weighted_fit_contract.py + packages/microcosm-frame/tests/test_accounting.py + packages/microcosm-frame/tests/test_adapters.py + packages/microcosm-frame/tests/test_axiom_adapter.py + packages/microcosm-frame/tests/test_bundle.py + packages/microcosm-frame/tests/test_contracts.py + packages/microcosm-frame/tests/test_kernels.py + packages/microcosm-frame/tests/test_materialize.py + packages/microcosm-frame/tests/test_place.py + packages/microcosm-frame/tests/test_policyengine_us_adapter.py + packages/microcosm-frame/tests/test_policyengine_us_consumer_index.py + packages/microcosm-frame/tests/test_policyengine_us_metadata_index.py + packages/microcosm-frame/tests/test_policyengine_us_ownership_index.py + packages/microcosm-frame/tests/test_rules_engine_contract.py + packages/microcosm-frame/tests/test_units.py + packages/microcosm-frame/tests/test_weights_schema.py + packages/microcosm-graph/tests/test_acceptance_a_identity.py + packages/microcosm-graph/tests/test_acceptance_b_ownership.py + packages/microcosm-graph/tests/test_acceptance_c_seeds.py + packages/microcosm-graph/tests/test_acceptance_d_weights.py + packages/microcosm-graph/tests/test_acceptance_e_store.py + packages/microcosm-graph/tests/test_acceptance_f_gates.py + packages/microcosm-graph/tests/test_acceptance_g_legibility.py + packages/microcosm-graph/tests/test_acceptance_h_parity.py + packages/microcosm-graph/tests/test_acceptance_replays.py + packages/microcosm-graph/tests/test_frame_metadata_store.py + packages/microcosm-graph/tests/test_graph_acceptance_burndown.py + packages/microcosm-graph/tests/test_graph_canonical.py + packages/microcosm-graph/tests/test_graph_codecs.py + packages/microcosm-graph/tests/test_graph_decl.py + packages/microcosm-graph/tests/test_graph_executor.py + packages/microcosm-graph/tests/test_graph_executor_scalar_float.py + packages/microcosm-graph/tests/test_graph_executor_series_stream.py + packages/microcosm-graph/tests/test_graph_executor_source_identity.py + packages/microcosm-graph/tests/test_graph_explain.py + packages/microcosm-graph/tests/test_graph_interface_lock.py + packages/microcosm-graph/tests/test_graph_kernel_contract.py + packages/microcosm-graph/tests/test_graph_keys.py + packages/microcosm-graph/tests/test_graph_manifest.py + packages/microcosm-graph/tests/test_graph_parity_pins.py + packages/microcosm-graph/tests/test_graph_parity_repin.py + packages/microcosm-graph/tests/test_graph_population.py + packages/microcosm-graph/tests/test_graph_randomness.py + packages/microcosm-graph/tests/test_graph_serialize.py + packages/microcosm-graph/tests/test_graph_storage_parts_whole_series.py + packages/microcosm-graph/tests/test_graph_store.py + packages/microcosm-graph/tests/test_graph_verification_epoch.py + packages/microcosm-graph/tests/test_graph_view.py +[engine] +shared-spec: files=87 procs=main shards_per_process=1 + packages/microcosm-build/tests/test_atomic_geography.py + packages/microcosm-build/tests/test_chronicle_epoch.py + packages/microcosm-build/tests/test_ci_test_groups.py + packages/microcosm-build/tests/test_compare_us_fiscal_diagnostics.py + packages/microcosm-build/tests/test_country_spec.py + packages/microcosm-build/tests/test_cross_grain.py + packages/microcosm-build/tests/test_demographics.py + packages/microcosm-build/tests/test_diagnose_us_target_support.py + packages/microcosm-build/tests/test_frame_checkpoint.py + packages/microcosm-build/tests/test_frame_sampling.py + packages/microcosm-build/tests/test_frame_serializer_registry.py + packages/microcosm-build/tests/test_gate_battery.py + packages/microcosm-build/tests/test_gate_battery_contract_example.py + packages/microcosm-build/tests/test_gate_battery_contract_pins.py + packages/microcosm-build/tests/test_gates.py + packages/microcosm-build/tests/test_holdout.py + packages/microcosm-build/tests/test_imputation_lineage_spec.py + packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py + packages/microcosm-build/tests/test_ledger_artifact.py + packages/microcosm-build/tests/test_ledger_targets.py + packages/microcosm-build/tests/test_logbook.py + packages/microcosm-build/tests/test_logbook_adoption.py + packages/microcosm-build/tests/test_logbook_archive.py + packages/microcosm-build/tests/test_logbook_backfill.py + packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py + packages/microcosm-build/tests/test_logbook_cli.py + packages/microcosm-build/tests/test_logbook_env.py + packages/microcosm-build/tests/test_logbook_isolation.py + packages/microcosm-build/tests/test_logbook_prediction_seed.py + packages/microcosm-build/tests/test_monetary_profile.py + packages/microcosm-build/tests/test_monetary_targets.py + packages/microcosm-build/tests/test_outer_stage_runtime.py + packages/microcosm-build/tests/test_plan.py + packages/microcosm-build/tests/test_reform_validation.py + packages/microcosm-build/tests/test_release_input_coverage.py + packages/microcosm-build/tests/test_release_target_parity.py + packages/microcosm-build/tests/test_score_uk_national_candidate.py + packages/microcosm-build/tests/test_source_runtime.py + packages/microcosm-build/tests/test_spec_engine_battery_semantics.py + packages/microcosm-build/tests/test_spec_engine_calibration_semantics.py + packages/microcosm-build/tests/test_spec_engine_compiler_ir.py + packages/microcosm-build/tests/test_spec_engine_country_bundles.py + packages/microcosm-build/tests/test_spec_engine_coverage_tool.py + packages/microcosm-build/tests/test_spec_engine_engine_abi.py + packages/microcosm-build/tests/test_spec_engine_field_usage.py + packages/microcosm-build/tests/test_spec_engine_identity_contracts.py + packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py + packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py + packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py + packages/microcosm-build/tests/test_spec_engine_loader.py + packages/microcosm-build/tests/test_spec_engine_plan_lock.py + packages/microcosm-build/tests/test_spec_engine_publication_semantics.py + packages/microcosm-build/tests/test_spec_engine_reemission.py + packages/microcosm-build/tests/test_spec_engine_resolver.py + packages/microcosm-build/tests/test_spec_engine_schema.py + packages/microcosm-build/tests/test_spec_engine_seeds.py + packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py + packages/microcosm-build/tests/test_spec_engine_take_up_semantics.py + packages/microcosm-build/tests/test_spec_engine_typed_closure.py + packages/microcosm-build/tests/test_spec_engine_yaml.py + packages/microcosm-build/tests/test_spec_only_country_packages.py + packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py + packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py + packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py + packages/microcosm-build/tests/test_spec_seed_identity_lock.py + packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py + packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py + packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py + packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py + packages/microcosm-build/tests/test_spec_worker_identity_lock.py + packages/microcosm-build/tests/test_stage_profile.py + packages/microcosm-build/tests/test_staging.py + packages/microcosm-build/tests/test_staging_v2.py + packages/microcosm-build/tests/test_stochastic_assignment.py + packages/microcosm-build/tests/test_target_materialization.py + packages/microcosm-build/tests/test_target_reference_authoring.py + packages/microcosm-build/tests/test_trace.py + packages/microcosm-build/tests/test_validation_input_coverage.py + packages/microcosm-frame/tests/test_accounting.py + packages/microcosm-frame/tests/test_adapters.py + packages/microcosm-frame/tests/test_axiom_adapter.py + packages/microcosm-frame/tests/test_bundle.py + packages/microcosm-frame/tests/test_contracts.py + packages/microcosm-frame/tests/test_materialize.py + packages/microcosm-frame/tests/test_place.py + packages/microcosm-frame/tests/test_units.py + packages/microcosm-frame/tests/test_weights_schema.py +us-p: files=33 procs=main shards_per_process=1 + packages/microcosm-build/tests/test_puf_qrf_chain.py + packages/microcosm-build/tests/test_us_parity_reference.py + packages/microcosm-build/tests/test_us_plan.py + packages/microcosm-build/tests/test_us_pool_input_consumers.py + packages/microcosm-build/tests/test_us_population_input_coverage.py + packages/microcosm-build/tests/test_us_post_transfer_calibration.py + packages/microcosm-build/tests/test_us_postclone_geography_identity.py + packages/microcosm-build/tests/test_us_pregnancy.py + packages/microcosm-build/tests/test_us_prior_year_income.py + packages/microcosm-build/tests/test_us_property_completion_graph.py + packages/microcosm-build/tests/test_us_property_completion_routing.py + packages/microcosm-build/tests/test_us_puf55_heterogeneous_manifest_seals.py + packages/microcosm-build/tests/test_us_puf55_model_boundaries.py + packages/microcosm-build/tests/test_us_puf55_route_finalization.py + packages/microcosm-build/tests/test_us_puf55_route_numerical_finalization.py + packages/microcosm-build/tests/test_us_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_profile.py + packages/microcosm-build/tests/test_us_puf59_canonical.py + packages/microcosm-build/tests/test_us_puf_aggregate_records.py + packages/microcosm-build/tests/test_us_puf_capital_gains_tail.py + packages/microcosm-build/tests/test_us_puf_detail_transfer.py + packages/microcosm-build/tests/test_us_puf_donor_io.py + packages/microcosm-build/tests/test_us_puf_e01000_reconciliation.py + packages/microcosm-build/tests/test_us_puf_e19200_decomposition.py + packages/microcosm-build/tests/test_us_puf_price_baseline.py + packages/microcosm-build/tests/test_us_puf_support.py + packages/microcosm-build/tests/test_us_puf_support_base_builder.py + packages/microcosm-build/tests/test_us_puf_support_base_equivalence.py + packages/microcosm-build/tests/test_us_puf_tail_bound.py + packages/microcosm-build/tests/test_us_puf_target2024_growth.py + packages/microcosm-build/tests/test_us_puma_ladder.py + packages/microcosm-build/tests/test_us_puma_ladder_sources.py +us-qs: files=61 procs=build, frame shards_per_process=1 + packages/microcosm-build/tests/test_us_qbi_inputs.py + packages/microcosm-build/tests/test_us_relationship_inputs.py + packages/microcosm-build/tests/test_us_release_gate_preflight.py + packages/microcosm-build/tests/test_us_release_head_to_head_scorer.py + packages/microcosm-build/tests/test_us_reported_coverage_vintage_gate.py + packages/microcosm-build/tests/test_us_retirement_contributions.py + packages/microcosm-build/tests/test_us_retirement_distributions.py + packages/microcosm-build/tests/test_us_runtime_facade_union.py + packages/microcosm-build/tests/test_us_runtime_import_order.py + packages/microcosm-build/tests/test_us_salt_refund_income.py + packages/microcosm-build/tests/test_us_scf_auto_loans.py + packages/microcosm-build/tests/test_us_scf_wealth.py + packages/microcosm-build/tests/test_us_sipp_financial_assets.py + packages/microcosm-build/tests/test_us_sipp_head_start.py + packages/microcosm-build/tests/test_us_sipp_tips.py + packages/microcosm-build/tests/test_us_sipp_vehicles.py + packages/microcosm-build/tests/test_us_snap_discretionary_exemption.py + packages/microcosm-build/tests/test_us_snap_state_take_up.py + packages/microcosm-build/tests/test_us_snap_take_up.py + packages/microcosm-build/tests/test_us_source_coverage.py + packages/microcosm-build/tests/test_us_source_runtime.py + packages/microcosm-build/tests/test_us_spec_bundle.py + packages/microcosm-build/tests/test_us_spine_agreement.py + packages/microcosm-build/tests/test_us_spine_assembly.py + packages/microcosm-build/tests/test_us_spine_blindness.py + packages/microcosm-build/tests/test_us_spm_resources.py + packages/microcosm-build/tests/test_us_spm_role_enrichment_builder.py + packages/microcosm-build/tests/test_us_spm_role_source.py + packages/microcosm-build/tests/test_us_ssi_disability_criteria.py + packages/microcosm-build/tests/test_us_ssi_take_up.py + packages/microcosm-build/tests/test_us_stacked_spine.py + packages/microcosm-build/tests/test_us_state_files_scorer.py + packages/microcosm-build/tests/test_us_survey_age_activation.py + packages/microcosm-build/tests/test_us_survey_age_artifact.py + packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_age_calibration_run.py + packages/microcosm-build/tests/test_us_survey_age_development.py + packages/microcosm-build/tests/test_us_survey_age_sources.py + packages/microcosm-build/tests/test_us_survey_calibration.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py + packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py + packages/microcosm-build/tests/test_us_survey_financial_successor.py + packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py + packages/microcosm-build/tests/test_us_survey_observed_age.py + packages/microcosm-build/tests/test_us_survey_origin_budget.py + packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_origin_budget_detached_views.py + packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py + packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py + packages/microcosm-build/tests/test_us_survey_population_preparation.py + packages/microcosm-build/tests/test_us_survey_population_replay.py + packages/microcosm-build/tests/test_us_survey_social_security.py + packages/microcosm-build/tests/test_us_survivor_benefits_exclusion.py + packages/microcosm-frame/tests/test_kernels.py + packages/microcosm-frame/tests/test_policyengine_us_adapter.py + packages/microcosm-frame/tests/test_policyengine_us_consumer_index.py + packages/microcosm-frame/tests/test_policyengine_us_metadata_index.py + packages/microcosm-frame/tests/test_policyengine_us_ownership_index.py + packages/microcosm-frame/tests/test_rules_engine_contract.py +us-not: files=22 procs=main shards_per_process=1 + packages/microcosm-build/tests/test_us_national_age_counts.py + packages/microcosm-build/tests/test_us_native_row_ceilings.py + packages/microcosm-build/tests/test_us_native_verify_once_epoch.py + packages/microcosm-build/tests/test_us_nonzero_shares.py + packages/microcosm-build/tests/test_us_org_wages.py + packages/microcosm-build/tests/test_us_other_health_insurance.py + packages/microcosm-build/tests/test_us_take_up.py + packages/microcosm-build/tests/test_us_take_up_contract.py + packages/microcosm-build/tests/test_us_trade_census_imports.py + packages/microcosm-build/tests/test_us_trade_crosscheck.py + packages/microcosm-build/tests/test_us_trade_entries_cli.py + packages/microcosm-build/tests/test_us_trade_entry_generator.py + packages/microcosm-build/tests/test_us_trade_facts.py + packages/microcosm-build/tests/test_us_trade_imdb_bulk.py + packages/microcosm-build/tests/test_us_trade_imdb_goldens.py + packages/microcosm-build/tests/test_us_voluntary_filing.py + packages/microcosm-build/tests/test_us_warm_start_selection.py + packages/microcosm-build/tests/test_us_weeks_unemployed.py + packages/microcosm-build/tests/test_us_whole_series_selection_sites.py + packages/microcosm-build/tests/test_us_wic_claim.py + packages/microcosm-build/tests/test_us_wic_nutritional_risk_exclusion.py + packages/microcosm-build/tests/test_us_workers_compensation.py +us-am: files=217 procs=build, other-shards shards_per_process=4 + packages/microcosm-build/tests/test_us_acs_housing_source.py + packages/microcosm-build/tests/test_us_acs_inputs.py + packages/microcosm-build/tests/test_us_acs_local_release_tool.py + packages/microcosm-build/tests/test_us_acs_multispine.py + packages/microcosm-build/tests/test_us_acs_multispine_base_builder.py + packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py + packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py + packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py + packages/microcosm-build/tests/test_us_acs_population_catalogue.py + packages/microcosm-build/tests/test_us_acs_pums.py + packages/microcosm-build/tests/test_us_acs_record_fence_scan.py + packages/microcosm-build/tests/test_us_acs_release_predictors.py + packages/microcosm-build/tests/test_us_acs_source_compile_cache.py + packages/microcosm-build/tests/test_us_acs_sources.py + packages/microcosm-build/tests/test_us_acs_transfer.py + packages/microcosm-build/tests/test_us_adult_care.py + packages/microcosm-build/tests/test_us_alimony.py + packages/microcosm-build/tests/test_us_asec_2024_native_population.py + packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py + packages/microcosm-build/tests/test_us_asec_checkpoint.py + packages/microcosm-build/tests/test_us_asec_coverage_authentication.py + packages/microcosm-build/tests/test_us_asec_current_money_source.py + packages/microcosm-build/tests/test_us_asec_demographic_source.py + packages/microcosm-build/tests/test_us_asec_household_coverage_fields.py + packages/microcosm-build/tests/test_us_asec_person_income_source.py + packages/microcosm-build/tests/test_us_asec_pool.py + packages/microcosm-build/tests/test_us_asec_prepared_resources.py + packages/microcosm-build/tests/test_us_asec_source_domain_hardening.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_de.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_national.py + packages/microcosm-build/tests/test_us_atomic_block_sources.py + packages/microcosm-build/tests/test_us_atomic_block_support.py + packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py + packages/microcosm-build/tests/test_us_base_pool.py + packages/microcosm-build/tests/test_us_block_ladder_sources.py + packages/microcosm-build/tests/test_us_buildj_base_script.py + packages/microcosm-build/tests/test_us_bundle_core_contracts.py + packages/microcosm-build/tests/test_us_capital_gain_details.py + packages/microcosm-build/tests/test_us_capital_gain_distributions.py + packages/microcosm-build/tests/test_us_casualty_losses.py + packages/microcosm-build/tests/test_us_child_property_income_graph.py + packages/microcosm-build/tests/test_us_child_property_income_graph_owner.py + packages/microcosm-build/tests/test_us_child_property_income_source.py + packages/microcosm-build/tests/test_us_child_property_income_source_owner.py + packages/microcosm-build/tests/test_us_child_support.py + packages/microcosm-build/tests/test_us_childcare.py + packages/microcosm-build/tests/test_us_common_frame_export_contract.py + packages/microcosm-build/tests/test_us_completion_support_custody.py + packages/microcosm-build/tests/test_us_congressional_district_geography.py + packages/microcosm-build/tests/test_us_congressional_district_vintage.py + packages/microcosm-build/tests/test_us_congressional_district_vintage_crosswalk.py + packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py + packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py + packages/microcosm-build/tests/test_us_current_asec_child_support_source.py + packages/microcosm-build/tests/test_us_current_asec_demographics.py + packages/microcosm-build/tests/test_us_current_asec_dividend_source.py + packages/microcosm-build/tests/test_us_current_asec_income_routing.py + packages/microcosm-build/tests/test_us_current_asec_interest_source.py + packages/microcosm-build/tests/test_us_current_asec_property_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py + packages/microcosm-build/tests/test_us_current_property_income_sources.py + packages/microcosm-build/tests/test_us_current_survey_amounts.py + packages/microcosm-build/tests/test_us_current_survey_geography.py + packages/microcosm-build/tests/test_us_current_survey_health_coverage.py + packages/microcosm-build/tests/test_us_current_survey_household_roles.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_graph.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_source.py + packages/microcosm-build/tests/test_us_current_survey_housing.py + packages/microcosm-build/tests/test_us_current_survey_person_status.py + packages/microcosm-build/tests/test_us_current_survey_person_status_graph.py + packages/microcosm-build/tests/test_us_current_survey_person_status_source.py + packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py + packages/microcosm-build/tests/test_us_current_survey_predictors.py + packages/microcosm-build/tests/test_us_current_survey_property_graph.py + packages/microcosm-build/tests/test_us_current_survey_puf_host.py + packages/microcosm-build/tests/test_us_current_survey_puf_transfer.py + packages/microcosm-build/tests/test_us_dc_ptc_take_up_exclusion.py + packages/microcosm-build/tests/test_us_disability_benefits.py + packages/microcosm-build/tests/test_us_documented_absent_inputs.py + packages/microcosm-build/tests/test_us_domestic_production.py + packages/microcosm-build/tests/test_us_early_head_start_exclusion.py + packages/microcosm-build/tests/test_us_education_assistance_source.py + packages/microcosm-build/tests/test_us_education_inputs.py + packages/microcosm-build/tests/test_us_educator_expenses.py + packages/microcosm-build/tests/test_us_eligibility_inputs.py + packages/microcosm-build/tests/test_us_energy_subsidy.py + packages/microcosm-build/tests/test_us_exact_k_ladder.py + packages/microcosm-build/tests/test_us_exact_k_ladder_e2e.py + packages/microcosm-build/tests/test_us_exact_k_ladder_launcher.py + packages/microcosm-build/tests/test_us_farm_business_income.py + packages/microcosm-build/tests/test_us_financial_assistance_exclusion.py + packages/microcosm-build/tests/test_us_fiscal_leaf_policy.py + packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py + packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py + packages/microcosm-build/tests/test_us_fiscal_targets.py + packages/microcosm-build/tests/test_us_form_4952.py + packages/microcosm-build/tests/test_us_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_full_puf_output_profiles.py + packages/microcosm-build/tests/test_us_geography_integration.py + packages/microcosm-build/tests/test_us_geography_ladder.py + packages/microcosm-build/tests/test_us_graph.py + packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py + packages/microcosm-build/tests/test_us_graph_atomic_person_status.py + packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py + packages/microcosm-build/tests/test_us_graph_current_survey_geography.py + packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py + packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py + packages/microcosm-build/tests/test_us_graph_fiscal_target_snapshots.py + packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_selection.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_stream.py + packages/microcosm-build/tests/test_us_graph_property_income.py + packages/microcosm-build/tests/test_us_graph_property_income_receipts.py + packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py + packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py + packages/microcosm-build/tests/test_us_graph_survey_completion.py + packages/microcosm-build/tests/test_us_graph_survey_population.py + packages/microcosm-build/tests/test_us_graph_survey_puf55.py + packages/microcosm-build/tests/test_us_graph_us_survey_enrichment.py + packages/microcosm-build/tests/test_us_hours_worked.py + packages/microcosm-build/tests/test_us_housing_inputs.py + packages/microcosm-build/tests/test_us_housing_participation.py + packages/microcosm-build/tests/test_us_immigration.py + packages/microcosm-build/tests/test_us_input_mass.py + packages/microcosm-build/tests/test_us_l0_refit_export.py + packages/microcosm-build/tests/test_us_late_producer_dag.py + packages/microcosm-build/tests/test_us_medicaid_take_up.py + packages/microcosm-build/tests/test_us_medicare_take_up.py + packages/microcosm-build/tests/test_us_misc_itemized.py + packages/microcosm-build/tests/test_us_multispine_pool.py + packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py + packages/microcosm-build/tests/test_us_multispine_pool_tool.py + packages/microcosm-build/tests/test_us_multispine_puf_clone.py + packages/microcosm-calibrate/tests/test_compat.py + packages/microcosm-calibrate/tests/test_diagnostics.py + packages/microcosm-calibrate/tests/test_exact_k.py + packages/microcosm-calibrate/tests/test_group_bounds.py + packages/microcosm-calibrate/tests/test_grouped_fixed_support.py + packages/microcosm-calibrate/tests/test_hierarchy.py + packages/microcosm-calibrate/tests/test_informed_gates.py + packages/microcosm-calibrate/tests/test_kernels.py + packages/microcosm-calibrate/tests/test_matrix.py + packages/microcosm-calibrate/tests/test_monetary_binding.py + packages/microcosm-calibrate/tests/test_provider_labels.py + packages/microcosm-calibrate/tests/test_registry.py + packages/microcosm-calibrate/tests/test_score.py + packages/microcosm-calibrate/tests/test_solve.py + packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py + packages/microcosm-calibrate/tests/test_sparse_solve.py + packages/microcosm-calibrate/tests/test_target.py + packages/microcosm-calibrate/tests/test_target_snapshots.py + packages/microcosm-calibrate/tests/test_variable_labels.py + packages/microcosm-data/tests/test_compat.py + packages/microcosm-data/tests/test_contract.py + packages/microcosm-data/tests/test_h5_enrichment.py + packages/microcosm-data/tests/test_loader.py + packages/microcosm-data/tests/test_local_area_contract.py + packages/microcosm-data/tests/test_publish_guard.py + packages/microcosm-data/tests/test_registry.py + packages/microcosm-data/tests/test_release.py + packages/microcosm-data/tests/test_slack.py + packages/microcosm-data/tests/test_source_enrichment.py + packages/microcosm-data/tests/test_uk_dense_release_contract.py + packages/microcosm-fit/tests/test_cache.py + packages/microcosm-fit/tests/test_compat.py + packages/microcosm-fit/tests/test_dataframe_fit.py + packages/microcosm-fit/tests/test_graph_joint_empirical.py + packages/microcosm-fit/tests/test_graph_legacy_matrix.py + packages/microcosm-fit/tests/test_graph_legacy_qrf.py + packages/microcosm-fit/tests/test_graph_signed_reconciliation.py + packages/microcosm-fit/tests/test_joint_empirical.py + packages/microcosm-fit/tests/test_kernels.py + packages/microcosm-fit/tests/test_model.py + packages/microcosm-fit/tests/test_qrf.py + packages/microcosm-fit/tests/test_qrf_stateless.py + packages/microcosm-fit/tests/test_qrf_target.py + packages/microcosm-fit/tests/test_signed_reconciliation.py + packages/microcosm-fit/tests/test_weighted_fit_contract.py + packages/microcosm-graph/tests/test_acceptance_a_identity.py + packages/microcosm-graph/tests/test_acceptance_b_ownership.py + packages/microcosm-graph/tests/test_acceptance_c_seeds.py + packages/microcosm-graph/tests/test_acceptance_d_weights.py + packages/microcosm-graph/tests/test_acceptance_e_store.py + packages/microcosm-graph/tests/test_acceptance_f_gates.py + packages/microcosm-graph/tests/test_acceptance_g_legibility.py + packages/microcosm-graph/tests/test_acceptance_h_parity.py + packages/microcosm-graph/tests/test_acceptance_replays.py + packages/microcosm-graph/tests/test_frame_metadata_store.py + packages/microcosm-graph/tests/test_graph_acceptance_burndown.py + packages/microcosm-graph/tests/test_graph_canonical.py + packages/microcosm-graph/tests/test_graph_codecs.py + packages/microcosm-graph/tests/test_graph_decl.py + packages/microcosm-graph/tests/test_graph_executor.py + packages/microcosm-graph/tests/test_graph_executor_scalar_float.py + packages/microcosm-graph/tests/test_graph_executor_series_stream.py + packages/microcosm-graph/tests/test_graph_executor_source_identity.py + packages/microcosm-graph/tests/test_graph_explain.py + packages/microcosm-graph/tests/test_graph_interface_lock.py + packages/microcosm-graph/tests/test_graph_kernel_contract.py + packages/microcosm-graph/tests/test_graph_keys.py + packages/microcosm-graph/tests/test_graph_manifest.py + packages/microcosm-graph/tests/test_graph_parity_pins.py + packages/microcosm-graph/tests/test_graph_parity_repin.py + packages/microcosm-graph/tests/test_graph_population.py + packages/microcosm-graph/tests/test_graph_randomness.py + packages/microcosm-graph/tests/test_graph_serialize.py + packages/microcosm-graph/tests/test_graph_storage_parts_whole_series.py + packages/microcosm-graph/tests/test_graph_store.py + packages/microcosm-graph/tests/test_graph_verification_epoch.py + packages/microcosm-graph/tests/test_graph_view.py +uk: files=137 procs=build, frame shards_per_process=1 + packages/microcosm-build/tests/test_uk_age_tail.py + packages/microcosm-build/tests/test_uk_atomic_area_support.py + packages/microcosm-build/tests/test_uk_atomic_household_lineage.py + packages/microcosm-build/tests/test_uk_battery_bindings.py + packages/microcosm-build/tests/test_uk_brma_resource.py + packages/microcosm-build/tests/test_uk_bus_targets.py + packages/microcosm-build/tests/test_uk_bus_use_incidence.py + packages/microcosm-build/tests/test_uk_calibration_run.py + packages/microcosm-build/tests/test_uk_calibration_seam_driver.py + packages/microcosm-build/tests/test_uk_cgt_calibration.py + packages/microcosm-build/tests/test_uk_cgt_imputation.py + packages/microcosm-build/tests/test_uk_cgt_observation_period.py + packages/microcosm-build/tests/test_uk_cgt_source_manifest.py + packages/microcosm-build/tests/test_uk_cgt_structure.py + packages/microcosm-build/tests/test_uk_cgt_targets.py + packages/microcosm-build/tests/test_uk_childcare_targets.py + packages/microcosm-build/tests/test_uk_chronicle_feed.py + packages/microcosm-build/tests/test_uk_consumption_resources.py + packages/microcosm-build/tests/test_uk_content_identity.py + packages/microcosm-build/tests/test_uk_corrected_measurement_receipt.py + packages/microcosm-build/tests/test_uk_council_tax_targets.py + packages/microcosm-build/tests/test_uk_data_target_parity.py + packages/microcosm-build/tests/test_uk_dense_release_assembler.py + packages/microcosm-build/tests/test_uk_diagnostics.py + packages/microcosm-build/tests/test_uk_donor_uprating.py + packages/microcosm-build/tests/test_uk_e5_identity.py + packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py + packages/microcosm-build/tests/test_uk_energy_pricing.py + packages/microcosm-build/tests/test_uk_etb_services.py + packages/microcosm-build/tests/test_uk_etb_vat.py + packages/microcosm-build/tests/test_uk_evaluate_dataset_size_tool.py + packages/microcosm-build/tests/test_uk_fact_raking.py + packages/microcosm-build/tests/test_uk_firm_generation.py + packages/microcosm-build/tests/test_uk_firms_targets.py + packages/microcosm-build/tests/test_uk_frame_linkage_preflight.py + packages/microcosm-build/tests/test_uk_frs_council_tax.py + packages/microcosm-build/tests/test_uk_frs_disability.py + packages/microcosm-build/tests/test_uk_frs_education.py + packages/microcosm-build/tests/test_uk_frs_education_grants.py + packages/microcosm-build/tests/test_uk_frs_employment.py + packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py + packages/microcosm-build/tests/test_uk_frs_legacy_proxies.py + packages/microcosm-build/tests/test_uk_frs_relationships.py + packages/microcosm-build/tests/test_uk_frs_release.py + packages/microcosm-build/tests/test_uk_frs_spine.py + packages/microcosm-build/tests/test_uk_frs_stochastic.py + packages/microcosm-build/tests/test_uk_geography_ladder.py + packages/microcosm-build/tests/test_uk_geography_sources.py + packages/microcosm-build/tests/test_uk_graph.py + packages/microcosm-build/tests/test_uk_h5_payload_compare.py + packages/microcosm-build/tests/test_uk_hmrc_calibration.py + packages/microcosm-build/tests/test_uk_hmrc_capital_gains.py + packages/microcosm-build/tests/test_uk_hmrc_income.py + packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py + packages/microcosm-build/tests/test_uk_hmrc_replay.py + packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py + packages/microcosm-build/tests/test_uk_identity_stability_receipts.py + packages/microcosm-build/tests/test_uk_incumbent_surface_evaluation.py + packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py + packages/microcosm-build/tests/test_uk_ladder_sni_sources.py + packages/microcosm-build/tests/test_uk_ladder_targets.py + packages/microcosm-build/tests/test_uk_lcfs_consumption.py + packages/microcosm-build/tests/test_uk_ledger_fact_vendoring.py + packages/microcosm-build/tests/test_uk_ledger_targets.py + packages/microcosm-build/tests/test_uk_local_area_crosswalk.py + packages/microcosm-build/tests/test_uk_local_candidate_scorer.py + packages/microcosm-build/tests/test_uk_local_doctrine.py + packages/microcosm-build/tests/test_uk_local_gate_battery.py + packages/microcosm-build/tests/test_uk_local_geography.py + packages/microcosm-build/tests/test_uk_local_holdout.py + packages/microcosm-build/tests/test_uk_local_incumbent_extractor.py + packages/microcosm-build/tests/test_uk_local_release_preflight.py + packages/microcosm-build/tests/test_uk_local_rowwise.py + packages/microcosm-build/tests/test_uk_local_target_census.py + packages/microcosm-build/tests/test_uk_local_targets.py + packages/microcosm-build/tests/test_uk_local_validation_levels.py + packages/microcosm-build/tests/test_uk_measure_simulation.py + packages/microcosm-build/tests/test_uk_national_calibration.py + packages/microcosm-build/tests/test_uk_national_frame.py + packages/microcosm-build/tests/test_uk_national_sampling.py + packages/microcosm-build/tests/test_uk_nhs_allocation.py + packages/microcosm-build/tests/test_uk_ods_tables.py + packages/microcosm-build/tests/test_uk_parity_reference.py + packages/microcosm-build/tests/test_uk_payload_diff_classifier.py + packages/microcosm-build/tests/test_uk_pipr_deferrals.py + packages/microcosm-build/tests/test_uk_population_targets.py + packages/microcosm-build/tests/test_uk_provenance_flags.py + packages/microcosm-build/tests/test_uk_raking.py + packages/microcosm-build/tests/test_uk_regional_uprating.py + packages/microcosm-build/tests/test_uk_release_assembler.py + packages/microcosm-build/tests/test_uk_release_certification.py + packages/microcosm-build/tests/test_uk_release_identity.py + packages/microcosm-build/tests/test_uk_release_input_coverage.py + packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py + packages/microcosm-build/tests/test_uk_rowwise_build_driver.py + packages/microcosm-build/tests/test_uk_rowwise_candidate.py + packages/microcosm-build/tests/test_uk_rowwise_dataset.py + packages/microcosm-build/tests/test_uk_rowwise_dry_run.py + packages/microcosm-build/tests/test_uk_rowwise_geography.py + packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py + packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py + packages/microcosm-build/tests/test_uk_salary_sacrifice.py + packages/microcosm-build/tests/test_uk_signed_differences.py + packages/microcosm-build/tests/test_uk_size_evaluation.py + packages/microcosm-build/tests/test_uk_source_runtime.py + packages/microcosm-build/tests/test_uk_source_stages.py + packages/microcosm-build/tests/test_uk_spi_income.py + packages/microcosm-build/tests/test_uk_spi_spine.py + packages/microcosm-build/tests/test_uk_spi_support.py + packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py + packages/microcosm-build/tests/test_uk_spine_parity_instrument.py + packages/microcosm-build/tests/test_uk_stage_checkpoints.py + packages/microcosm-build/tests/test_uk_stage_health.py + packages/microcosm-build/tests/test_uk_staging_integration.py + packages/microcosm-build/tests/test_uk_staging_provisioning.py + packages/microcosm-build/tests/test_uk_stochastic_tools.py + packages/microcosm-build/tests/test_uk_student_loans.py + packages/microcosm-build/tests/test_uk_take_up_contract.py + packages/microcosm-build/tests/test_uk_take_up_gate.py + packages/microcosm-build/tests/test_uk_target_references.py + packages/microcosm-build/tests/test_uk_terminal_gates.py + packages/microcosm-build/tests/test_uk_tfc_routed_share.py + packages/microcosm-build/tests/test_uk_uc_capital_coherence.py + packages/microcosm-build/tests/test_uk_uc_deduction_attributes.py + packages/microcosm-build/tests/test_uk_uc_family.py + packages/microcosm-build/tests/test_uk_uc_lp_evidence.py + packages/microcosm-build/tests/test_uk_uc_matrix_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_relationships.py + packages/microcosm-build/tests/test_uk_uc_reporter_redraw.py + packages/microcosm-build/tests/test_uk_uc_source_periods.py + packages/microcosm-build/tests/test_uk_uc_support_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_target_measurements.py + packages/microcosm-build/tests/test_uk_was_wealth.py + packages/microcosm-build/tests/test_uk_wealth_resources.py + packages/microcosm-build/tests/test_uk_weighted_integrity.py + packages/microcosm-build/tests/test_uk_weighted_integrity_measurement.py + packages/microcosm-frame/tests/test_policyengine_uk_adapter.py +[wheels] +wheels: files=557 procs=main shards_per_process=8 + packages/microcosm-build/tests/test_atomic_geography.py + packages/microcosm-build/tests/test_chronicle_epoch.py + packages/microcosm-build/tests/test_ci_test_groups.py + packages/microcosm-build/tests/test_compare_us_fiscal_diagnostics.py + packages/microcosm-build/tests/test_country_spec.py + packages/microcosm-build/tests/test_cross_grain.py + packages/microcosm-build/tests/test_demographics.py + packages/microcosm-build/tests/test_diagnose_us_target_support.py + packages/microcosm-build/tests/test_frame_checkpoint.py + packages/microcosm-build/tests/test_frame_sampling.py + packages/microcosm-build/tests/test_frame_serializer_registry.py + packages/microcosm-build/tests/test_gate_battery.py + packages/microcosm-build/tests/test_gate_battery_contract_example.py + packages/microcosm-build/tests/test_gate_battery_contract_pins.py + packages/microcosm-build/tests/test_gates.py + packages/microcosm-build/tests/test_holdout.py + packages/microcosm-build/tests/test_imputation_lineage_spec.py + packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py + packages/microcosm-build/tests/test_ledger_artifact.py + packages/microcosm-build/tests/test_ledger_targets.py + packages/microcosm-build/tests/test_logbook.py + packages/microcosm-build/tests/test_logbook_adoption.py + packages/microcosm-build/tests/test_logbook_archive.py + packages/microcosm-build/tests/test_logbook_backfill.py + packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py + packages/microcosm-build/tests/test_logbook_cli.py + packages/microcosm-build/tests/test_logbook_env.py + packages/microcosm-build/tests/test_logbook_isolation.py + packages/microcosm-build/tests/test_logbook_prediction_seed.py + packages/microcosm-build/tests/test_monetary_profile.py + packages/microcosm-build/tests/test_monetary_targets.py + packages/microcosm-build/tests/test_outer_stage_runtime.py + packages/microcosm-build/tests/test_plan.py + packages/microcosm-build/tests/test_puf_qrf_chain.py + packages/microcosm-build/tests/test_reform_validation.py + packages/microcosm-build/tests/test_release_input_coverage.py + packages/microcosm-build/tests/test_release_target_parity.py + packages/microcosm-build/tests/test_score_uk_national_candidate.py + packages/microcosm-build/tests/test_source_runtime.py + packages/microcosm-build/tests/test_spec_engine_battery_semantics.py + packages/microcosm-build/tests/test_spec_engine_calibration_semantics.py + packages/microcosm-build/tests/test_spec_engine_compiler_ir.py + packages/microcosm-build/tests/test_spec_engine_country_bundles.py + packages/microcosm-build/tests/test_spec_engine_coverage_tool.py + packages/microcosm-build/tests/test_spec_engine_engine_abi.py + packages/microcosm-build/tests/test_spec_engine_field_usage.py + packages/microcosm-build/tests/test_spec_engine_identity_contracts.py + packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py + packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py + packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py + packages/microcosm-build/tests/test_spec_engine_loader.py + packages/microcosm-build/tests/test_spec_engine_plan_lock.py + packages/microcosm-build/tests/test_spec_engine_publication_semantics.py + packages/microcosm-build/tests/test_spec_engine_reemission.py + packages/microcosm-build/tests/test_spec_engine_resolver.py + packages/microcosm-build/tests/test_spec_engine_schema.py + packages/microcosm-build/tests/test_spec_engine_seeds.py + packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py + packages/microcosm-build/tests/test_spec_engine_take_up_semantics.py + packages/microcosm-build/tests/test_spec_engine_typed_closure.py + packages/microcosm-build/tests/test_spec_engine_yaml.py + packages/microcosm-build/tests/test_spec_only_country_packages.py + packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py + packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py + packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py + packages/microcosm-build/tests/test_spec_seed_identity_lock.py + packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py + packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py + packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py + packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py + packages/microcosm-build/tests/test_spec_worker_identity_lock.py + packages/microcosm-build/tests/test_stage_profile.py + packages/microcosm-build/tests/test_staging.py + packages/microcosm-build/tests/test_staging_v2.py + packages/microcosm-build/tests/test_stochastic_assignment.py + packages/microcosm-build/tests/test_target_materialization.py + packages/microcosm-build/tests/test_target_reference_authoring.py + packages/microcosm-build/tests/test_trace.py + packages/microcosm-build/tests/test_uk_age_tail.py + packages/microcosm-build/tests/test_uk_atomic_area_support.py + packages/microcosm-build/tests/test_uk_atomic_household_lineage.py + packages/microcosm-build/tests/test_uk_battery_bindings.py + packages/microcosm-build/tests/test_uk_brma_resource.py + packages/microcosm-build/tests/test_uk_bus_targets.py + packages/microcosm-build/tests/test_uk_bus_use_incidence.py + packages/microcosm-build/tests/test_uk_calibration_run.py + packages/microcosm-build/tests/test_uk_calibration_seam_driver.py + packages/microcosm-build/tests/test_uk_cgt_calibration.py + packages/microcosm-build/tests/test_uk_cgt_imputation.py + packages/microcosm-build/tests/test_uk_cgt_observation_period.py + packages/microcosm-build/tests/test_uk_cgt_source_manifest.py + packages/microcosm-build/tests/test_uk_cgt_structure.py + packages/microcosm-build/tests/test_uk_cgt_targets.py + packages/microcosm-build/tests/test_uk_childcare_targets.py + packages/microcosm-build/tests/test_uk_chronicle_feed.py + packages/microcosm-build/tests/test_uk_consumption_resources.py + packages/microcosm-build/tests/test_uk_content_identity.py + packages/microcosm-build/tests/test_uk_corrected_measurement_receipt.py + packages/microcosm-build/tests/test_uk_council_tax_targets.py + packages/microcosm-build/tests/test_uk_data_target_parity.py + packages/microcosm-build/tests/test_uk_dense_release_assembler.py + packages/microcosm-build/tests/test_uk_diagnostics.py + packages/microcosm-build/tests/test_uk_donor_uprating.py + packages/microcosm-build/tests/test_uk_e5_identity.py + packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py + packages/microcosm-build/tests/test_uk_energy_pricing.py + packages/microcosm-build/tests/test_uk_etb_services.py + packages/microcosm-build/tests/test_uk_etb_vat.py + packages/microcosm-build/tests/test_uk_evaluate_dataset_size_tool.py + packages/microcosm-build/tests/test_uk_fact_raking.py + packages/microcosm-build/tests/test_uk_firm_generation.py + packages/microcosm-build/tests/test_uk_firms_targets.py + packages/microcosm-build/tests/test_uk_frame_linkage_preflight.py + packages/microcosm-build/tests/test_uk_frs_council_tax.py + packages/microcosm-build/tests/test_uk_frs_disability.py + packages/microcosm-build/tests/test_uk_frs_education.py + packages/microcosm-build/tests/test_uk_frs_education_grants.py + packages/microcosm-build/tests/test_uk_frs_employment.py + packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py + packages/microcosm-build/tests/test_uk_frs_legacy_proxies.py + packages/microcosm-build/tests/test_uk_frs_relationships.py + packages/microcosm-build/tests/test_uk_frs_release.py + packages/microcosm-build/tests/test_uk_frs_spine.py + packages/microcosm-build/tests/test_uk_frs_stochastic.py + packages/microcosm-build/tests/test_uk_geography_ladder.py + packages/microcosm-build/tests/test_uk_geography_sources.py + packages/microcosm-build/tests/test_uk_graph.py + packages/microcosm-build/tests/test_uk_h5_payload_compare.py + packages/microcosm-build/tests/test_uk_hmrc_calibration.py + packages/microcosm-build/tests/test_uk_hmrc_capital_gains.py + packages/microcosm-build/tests/test_uk_hmrc_income.py + packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py + packages/microcosm-build/tests/test_uk_hmrc_replay.py + packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py + packages/microcosm-build/tests/test_uk_identity_stability_receipts.py + packages/microcosm-build/tests/test_uk_incumbent_surface_evaluation.py + packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py + packages/microcosm-build/tests/test_uk_ladder_sni_sources.py + packages/microcosm-build/tests/test_uk_ladder_targets.py + packages/microcosm-build/tests/test_uk_lcfs_consumption.py + packages/microcosm-build/tests/test_uk_ledger_fact_vendoring.py + packages/microcosm-build/tests/test_uk_ledger_targets.py + packages/microcosm-build/tests/test_uk_local_area_crosswalk.py + packages/microcosm-build/tests/test_uk_local_candidate_scorer.py + packages/microcosm-build/tests/test_uk_local_doctrine.py + packages/microcosm-build/tests/test_uk_local_gate_battery.py + packages/microcosm-build/tests/test_uk_local_geography.py + packages/microcosm-build/tests/test_uk_local_holdout.py + packages/microcosm-build/tests/test_uk_local_incumbent_extractor.py + packages/microcosm-build/tests/test_uk_local_release_preflight.py + packages/microcosm-build/tests/test_uk_local_rowwise.py + packages/microcosm-build/tests/test_uk_local_target_census.py + packages/microcosm-build/tests/test_uk_local_targets.py + packages/microcosm-build/tests/test_uk_local_validation_levels.py + packages/microcosm-build/tests/test_uk_measure_simulation.py + packages/microcosm-build/tests/test_uk_national_calibration.py + packages/microcosm-build/tests/test_uk_national_frame.py + packages/microcosm-build/tests/test_uk_national_sampling.py + packages/microcosm-build/tests/test_uk_nhs_allocation.py + packages/microcosm-build/tests/test_uk_ods_tables.py + packages/microcosm-build/tests/test_uk_parity_reference.py + packages/microcosm-build/tests/test_uk_payload_diff_classifier.py + packages/microcosm-build/tests/test_uk_pipr_deferrals.py + packages/microcosm-build/tests/test_uk_population_targets.py + packages/microcosm-build/tests/test_uk_provenance_flags.py + packages/microcosm-build/tests/test_uk_raking.py + packages/microcosm-build/tests/test_uk_regional_uprating.py + packages/microcosm-build/tests/test_uk_release_assembler.py + packages/microcosm-build/tests/test_uk_release_certification.py + packages/microcosm-build/tests/test_uk_release_identity.py + packages/microcosm-build/tests/test_uk_release_input_coverage.py + packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py + packages/microcosm-build/tests/test_uk_rowwise_build_driver.py + packages/microcosm-build/tests/test_uk_rowwise_candidate.py + packages/microcosm-build/tests/test_uk_rowwise_dataset.py + packages/microcosm-build/tests/test_uk_rowwise_dry_run.py + packages/microcosm-build/tests/test_uk_rowwise_geography.py + packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py + packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py + packages/microcosm-build/tests/test_uk_salary_sacrifice.py + packages/microcosm-build/tests/test_uk_signed_differences.py + packages/microcosm-build/tests/test_uk_size_evaluation.py + packages/microcosm-build/tests/test_uk_source_runtime.py + packages/microcosm-build/tests/test_uk_source_stages.py + packages/microcosm-build/tests/test_uk_spi_income.py + packages/microcosm-build/tests/test_uk_spi_spine.py + packages/microcosm-build/tests/test_uk_spi_support.py + packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py + packages/microcosm-build/tests/test_uk_spine_parity_instrument.py + packages/microcosm-build/tests/test_uk_stage_checkpoints.py + packages/microcosm-build/tests/test_uk_stage_health.py + packages/microcosm-build/tests/test_uk_staging_integration.py + packages/microcosm-build/tests/test_uk_staging_provisioning.py + packages/microcosm-build/tests/test_uk_stochastic_tools.py + packages/microcosm-build/tests/test_uk_student_loans.py + packages/microcosm-build/tests/test_uk_take_up_contract.py + packages/microcosm-build/tests/test_uk_take_up_gate.py + packages/microcosm-build/tests/test_uk_target_references.py + packages/microcosm-build/tests/test_uk_terminal_gates.py + packages/microcosm-build/tests/test_uk_tfc_routed_share.py + packages/microcosm-build/tests/test_uk_uc_capital_coherence.py + packages/microcosm-build/tests/test_uk_uc_deduction_attributes.py + packages/microcosm-build/tests/test_uk_uc_family.py + packages/microcosm-build/tests/test_uk_uc_lp_evidence.py + packages/microcosm-build/tests/test_uk_uc_matrix_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_relationships.py + packages/microcosm-build/tests/test_uk_uc_reporter_redraw.py + packages/microcosm-build/tests/test_uk_uc_source_periods.py + packages/microcosm-build/tests/test_uk_uc_support_diagnostics.py + packages/microcosm-build/tests/test_uk_uc_target_measurements.py + packages/microcosm-build/tests/test_uk_was_wealth.py + packages/microcosm-build/tests/test_uk_wealth_resources.py + packages/microcosm-build/tests/test_uk_weighted_integrity.py + packages/microcosm-build/tests/test_uk_weighted_integrity_measurement.py + packages/microcosm-build/tests/test_us_acs_housing_source.py + packages/microcosm-build/tests/test_us_acs_inputs.py + packages/microcosm-build/tests/test_us_acs_local_release_tool.py + packages/microcosm-build/tests/test_us_acs_multispine.py + packages/microcosm-build/tests/test_us_acs_multispine_base_builder.py + packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py + packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py + packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py + packages/microcosm-build/tests/test_us_acs_population_catalogue.py + packages/microcosm-build/tests/test_us_acs_pums.py + packages/microcosm-build/tests/test_us_acs_record_fence_scan.py + packages/microcosm-build/tests/test_us_acs_release_predictors.py + packages/microcosm-build/tests/test_us_acs_source_compile_cache.py + packages/microcosm-build/tests/test_us_acs_sources.py + packages/microcosm-build/tests/test_us_acs_transfer.py + packages/microcosm-build/tests/test_us_adult_care.py + packages/microcosm-build/tests/test_us_alimony.py + packages/microcosm-build/tests/test_us_asec_2024_native_population.py + packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py + packages/microcosm-build/tests/test_us_asec_checkpoint.py + packages/microcosm-build/tests/test_us_asec_coverage_authentication.py + packages/microcosm-build/tests/test_us_asec_current_money_source.py + packages/microcosm-build/tests/test_us_asec_demographic_source.py + packages/microcosm-build/tests/test_us_asec_household_coverage_fields.py + packages/microcosm-build/tests/test_us_asec_person_income_source.py + packages/microcosm-build/tests/test_us_asec_pool.py + packages/microcosm-build/tests/test_us_asec_prepared_resources.py + packages/microcosm-build/tests/test_us_asec_source_domain_hardening.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_de.py + packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_national.py + packages/microcosm-build/tests/test_us_atomic_block_sources.py + packages/microcosm-build/tests/test_us_atomic_block_support.py + packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py + packages/microcosm-build/tests/test_us_base_pool.py + packages/microcosm-build/tests/test_us_block_ladder_sources.py + packages/microcosm-build/tests/test_us_buildj_base_script.py + packages/microcosm-build/tests/test_us_bundle_core_contracts.py + packages/microcosm-build/tests/test_us_capital_gain_details.py + packages/microcosm-build/tests/test_us_capital_gain_distributions.py + packages/microcosm-build/tests/test_us_casualty_losses.py + packages/microcosm-build/tests/test_us_child_property_income_graph.py + packages/microcosm-build/tests/test_us_child_property_income_graph_owner.py + packages/microcosm-build/tests/test_us_child_property_income_source.py + packages/microcosm-build/tests/test_us_child_property_income_source_owner.py + packages/microcosm-build/tests/test_us_child_support.py + packages/microcosm-build/tests/test_us_childcare.py + packages/microcosm-build/tests/test_us_common_frame_export_contract.py + packages/microcosm-build/tests/test_us_completion_support_custody.py + packages/microcosm-build/tests/test_us_congressional_district_geography.py + packages/microcosm-build/tests/test_us_congressional_district_vintage.py + packages/microcosm-build/tests/test_us_congressional_district_vintage_crosswalk.py + packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py + packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py + packages/microcosm-build/tests/test_us_current_asec_child_support_source.py + packages/microcosm-build/tests/test_us_current_asec_demographics.py + packages/microcosm-build/tests/test_us_current_asec_dividend_source.py + packages/microcosm-build/tests/test_us_current_asec_income_routing.py + packages/microcosm-build/tests/test_us_current_asec_interest_source.py + packages/microcosm-build/tests/test_us_current_asec_property_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py + packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py + packages/microcosm-build/tests/test_us_current_property_income_sources.py + packages/microcosm-build/tests/test_us_current_survey_amounts.py + packages/microcosm-build/tests/test_us_current_survey_geography.py + packages/microcosm-build/tests/test_us_current_survey_health_coverage.py + packages/microcosm-build/tests/test_us_current_survey_household_roles.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_graph.py + packages/microcosm-build/tests/test_us_current_survey_household_roles_source.py + packages/microcosm-build/tests/test_us_current_survey_housing.py + packages/microcosm-build/tests/test_us_current_survey_person_status.py + packages/microcosm-build/tests/test_us_current_survey_person_status_graph.py + packages/microcosm-build/tests/test_us_current_survey_person_status_source.py + packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py + packages/microcosm-build/tests/test_us_current_survey_predictors.py + packages/microcosm-build/tests/test_us_current_survey_property_graph.py + packages/microcosm-build/tests/test_us_current_survey_puf_host.py + packages/microcosm-build/tests/test_us_current_survey_puf_transfer.py + packages/microcosm-build/tests/test_us_dc_ptc_take_up_exclusion.py + packages/microcosm-build/tests/test_us_disability_benefits.py + packages/microcosm-build/tests/test_us_documented_absent_inputs.py + packages/microcosm-build/tests/test_us_domestic_production.py + packages/microcosm-build/tests/test_us_early_head_start_exclusion.py + packages/microcosm-build/tests/test_us_education_assistance_source.py + packages/microcosm-build/tests/test_us_education_inputs.py + packages/microcosm-build/tests/test_us_educator_expenses.py + packages/microcosm-build/tests/test_us_eligibility_inputs.py + packages/microcosm-build/tests/test_us_energy_subsidy.py + packages/microcosm-build/tests/test_us_exact_k_ladder.py + packages/microcosm-build/tests/test_us_exact_k_ladder_e2e.py + packages/microcosm-build/tests/test_us_exact_k_ladder_launcher.py + packages/microcosm-build/tests/test_us_farm_business_income.py + packages/microcosm-build/tests/test_us_financial_assistance_exclusion.py + packages/microcosm-build/tests/test_us_fiscal_leaf_policy.py + packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py + packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py + packages/microcosm-build/tests/test_us_fiscal_targets.py + packages/microcosm-build/tests/test_us_form_4952.py + packages/microcosm-build/tests/test_us_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_full_puf_output_profiles.py + packages/microcosm-build/tests/test_us_geography_integration.py + packages/microcosm-build/tests/test_us_geography_ladder.py + packages/microcosm-build/tests/test_us_graph.py + packages/microcosm-build/tests/test_us_graph_atomic_completion_host.py + packages/microcosm-build/tests/test_us_graph_atomic_person_status.py + packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py + packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py + packages/microcosm-build/tests/test_us_graph_current_survey_geography.py + packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py + packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py + packages/microcosm-build/tests/test_us_graph_fiscal_target_snapshots.py + packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_selection.py + packages/microcosm-build/tests/test_us_graph_full_puf_seal_stream.py + packages/microcosm-build/tests/test_us_graph_property_income.py + packages/microcosm-build/tests/test_us_graph_property_income_receipts.py + packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py + packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py + packages/microcosm-build/tests/test_us_graph_survey_completion.py + packages/microcosm-build/tests/test_us_graph_survey_population.py + packages/microcosm-build/tests/test_us_graph_survey_puf55.py + packages/microcosm-build/tests/test_us_graph_us_survey_enrichment.py + packages/microcosm-build/tests/test_us_hours_worked.py + packages/microcosm-build/tests/test_us_housing_inputs.py + packages/microcosm-build/tests/test_us_housing_participation.py + packages/microcosm-build/tests/test_us_immigration.py + packages/microcosm-build/tests/test_us_input_mass.py + packages/microcosm-build/tests/test_us_l0_refit_export.py + packages/microcosm-build/tests/test_us_late_producer_dag.py + packages/microcosm-build/tests/test_us_medicaid_take_up.py + packages/microcosm-build/tests/test_us_medicare_take_up.py + packages/microcosm-build/tests/test_us_misc_itemized.py + packages/microcosm-build/tests/test_us_multispine_pool.py + packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py + packages/microcosm-build/tests/test_us_multispine_pool_tool.py + packages/microcosm-build/tests/test_us_multispine_puf_clone.py + packages/microcosm-build/tests/test_us_national_age_counts.py + packages/microcosm-build/tests/test_us_native_row_ceilings.py + packages/microcosm-build/tests/test_us_native_verify_once_epoch.py + packages/microcosm-build/tests/test_us_nonzero_shares.py + packages/microcosm-build/tests/test_us_org_wages.py + packages/microcosm-build/tests/test_us_other_health_insurance.py + packages/microcosm-build/tests/test_us_parity_reference.py + packages/microcosm-build/tests/test_us_plan.py + packages/microcosm-build/tests/test_us_pool_input_consumers.py + packages/microcosm-build/tests/test_us_population_input_coverage.py + packages/microcosm-build/tests/test_us_post_transfer_calibration.py + packages/microcosm-build/tests/test_us_postclone_geography_identity.py + packages/microcosm-build/tests/test_us_pregnancy.py + packages/microcosm-build/tests/test_us_prior_year_income.py + packages/microcosm-build/tests/test_us_property_completion_graph.py + packages/microcosm-build/tests/test_us_property_completion_routing.py + packages/microcosm-build/tests/test_us_puf55_heterogeneous_manifest_seals.py + packages/microcosm-build/tests/test_us_puf55_model_boundaries.py + packages/microcosm-build/tests/test_us_puf55_route_finalization.py + packages/microcosm-build/tests/test_us_puf55_route_numerical_finalization.py + packages/microcosm-build/tests/test_us_puf55_survey_recipients.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py + packages/microcosm-build/tests/test_us_puf55_survey_ss_profile.py + packages/microcosm-build/tests/test_us_puf59_canonical.py + packages/microcosm-build/tests/test_us_puf_aggregate_records.py + packages/microcosm-build/tests/test_us_puf_capital_gains_tail.py + packages/microcosm-build/tests/test_us_puf_detail_transfer.py + packages/microcosm-build/tests/test_us_puf_donor_io.py + packages/microcosm-build/tests/test_us_puf_e01000_reconciliation.py + packages/microcosm-build/tests/test_us_puf_e19200_decomposition.py + packages/microcosm-build/tests/test_us_puf_price_baseline.py + packages/microcosm-build/tests/test_us_puf_support.py + packages/microcosm-build/tests/test_us_puf_support_base_builder.py + packages/microcosm-build/tests/test_us_puf_support_base_equivalence.py + packages/microcosm-build/tests/test_us_puf_tail_bound.py + packages/microcosm-build/tests/test_us_puf_target2024_growth.py + packages/microcosm-build/tests/test_us_puma_ladder.py + packages/microcosm-build/tests/test_us_puma_ladder_sources.py + packages/microcosm-build/tests/test_us_qbi_inputs.py + packages/microcosm-build/tests/test_us_relationship_inputs.py + packages/microcosm-build/tests/test_us_release_gate_preflight.py + packages/microcosm-build/tests/test_us_release_head_to_head_scorer.py + packages/microcosm-build/tests/test_us_reported_coverage_vintage_gate.py + packages/microcosm-build/tests/test_us_retirement_contributions.py + packages/microcosm-build/tests/test_us_retirement_distributions.py + packages/microcosm-build/tests/test_us_runtime_facade_union.py + packages/microcosm-build/tests/test_us_runtime_import_order.py + packages/microcosm-build/tests/test_us_salt_refund_income.py + packages/microcosm-build/tests/test_us_scf_auto_loans.py + packages/microcosm-build/tests/test_us_scf_wealth.py + packages/microcosm-build/tests/test_us_sipp_financial_assets.py + packages/microcosm-build/tests/test_us_sipp_head_start.py + packages/microcosm-build/tests/test_us_sipp_tips.py + packages/microcosm-build/tests/test_us_sipp_vehicles.py + packages/microcosm-build/tests/test_us_snap_discretionary_exemption.py + packages/microcosm-build/tests/test_us_snap_state_take_up.py + packages/microcosm-build/tests/test_us_snap_take_up.py + packages/microcosm-build/tests/test_us_source_coverage.py + packages/microcosm-build/tests/test_us_source_runtime.py + packages/microcosm-build/tests/test_us_spec_bundle.py + packages/microcosm-build/tests/test_us_spine_agreement.py + packages/microcosm-build/tests/test_us_spine_assembly.py + packages/microcosm-build/tests/test_us_spine_blindness.py + packages/microcosm-build/tests/test_us_spm_resources.py + packages/microcosm-build/tests/test_us_spm_role_enrichment_builder.py + packages/microcosm-build/tests/test_us_spm_role_source.py + packages/microcosm-build/tests/test_us_ssi_disability_criteria.py + packages/microcosm-build/tests/test_us_ssi_take_up.py + packages/microcosm-build/tests/test_us_stacked_spine.py + packages/microcosm-build/tests/test_us_state_files_scorer.py + packages/microcosm-build/tests/test_us_survey_age_activation.py + packages/microcosm-build/tests/test_us_survey_age_artifact.py + packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_age_calibration_run.py + packages/microcosm-build/tests/test_us_survey_age_development.py + packages/microcosm-build/tests/test_us_survey_age_sources.py + packages/microcosm-build/tests/test_us_survey_calibration.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py + packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py + packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py + packages/microcosm-build/tests/test_us_survey_financial_successor.py + packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py + packages/microcosm-build/tests/test_us_survey_observed_age.py + packages/microcosm-build/tests/test_us_survey_origin_budget.py + packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py + packages/microcosm-build/tests/test_us_survey_origin_budget_detached_views.py + packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py + packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py + packages/microcosm-build/tests/test_us_survey_population_preparation.py + packages/microcosm-build/tests/test_us_survey_population_replay.py + packages/microcosm-build/tests/test_us_survey_social_security.py + packages/microcosm-build/tests/test_us_survivor_benefits_exclusion.py + packages/microcosm-build/tests/test_us_take_up.py + packages/microcosm-build/tests/test_us_take_up_contract.py + packages/microcosm-build/tests/test_us_trade_census_imports.py + packages/microcosm-build/tests/test_us_trade_crosscheck.py + packages/microcosm-build/tests/test_us_trade_entries_cli.py + packages/microcosm-build/tests/test_us_trade_entry_generator.py + packages/microcosm-build/tests/test_us_trade_facts.py + packages/microcosm-build/tests/test_us_trade_imdb_bulk.py + packages/microcosm-build/tests/test_us_trade_imdb_goldens.py + packages/microcosm-build/tests/test_us_voluntary_filing.py + packages/microcosm-build/tests/test_us_warm_start_selection.py + packages/microcosm-build/tests/test_us_weeks_unemployed.py + packages/microcosm-build/tests/test_us_whole_series_selection_sites.py + packages/microcosm-build/tests/test_us_wic_claim.py + packages/microcosm-build/tests/test_us_wic_nutritional_risk_exclusion.py + packages/microcosm-build/tests/test_us_workers_compensation.py + packages/microcosm-build/tests/test_validation_input_coverage.py + packages/microcosm-calibrate/tests/test_compat.py + packages/microcosm-calibrate/tests/test_diagnostics.py + packages/microcosm-calibrate/tests/test_exact_k.py + packages/microcosm-calibrate/tests/test_group_bounds.py + packages/microcosm-calibrate/tests/test_grouped_fixed_support.py + packages/microcosm-calibrate/tests/test_hierarchy.py + packages/microcosm-calibrate/tests/test_informed_gates.py + packages/microcosm-calibrate/tests/test_kernels.py + packages/microcosm-calibrate/tests/test_matrix.py + packages/microcosm-calibrate/tests/test_monetary_binding.py + packages/microcosm-calibrate/tests/test_provider_labels.py + packages/microcosm-calibrate/tests/test_registry.py + packages/microcosm-calibrate/tests/test_score.py + packages/microcosm-calibrate/tests/test_solve.py + packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py + packages/microcosm-calibrate/tests/test_sparse_solve.py + packages/microcosm-calibrate/tests/test_target.py + packages/microcosm-calibrate/tests/test_target_snapshots.py + packages/microcosm-calibrate/tests/test_variable_labels.py + packages/microcosm-data/tests/test_compat.py + packages/microcosm-data/tests/test_contract.py + packages/microcosm-data/tests/test_h5_enrichment.py + packages/microcosm-data/tests/test_loader.py + packages/microcosm-data/tests/test_local_area_contract.py + packages/microcosm-data/tests/test_publish_guard.py + packages/microcosm-data/tests/test_registry.py + packages/microcosm-data/tests/test_release.py + packages/microcosm-data/tests/test_slack.py + packages/microcosm-data/tests/test_source_enrichment.py + packages/microcosm-data/tests/test_uk_dense_release_contract.py + packages/microcosm-fit/tests/test_cache.py + packages/microcosm-fit/tests/test_compat.py + packages/microcosm-fit/tests/test_dataframe_fit.py + packages/microcosm-fit/tests/test_graph_joint_empirical.py + packages/microcosm-fit/tests/test_graph_legacy_matrix.py + packages/microcosm-fit/tests/test_graph_legacy_qrf.py + packages/microcosm-fit/tests/test_graph_signed_reconciliation.py + packages/microcosm-fit/tests/test_joint_empirical.py + packages/microcosm-fit/tests/test_kernels.py + packages/microcosm-fit/tests/test_model.py + packages/microcosm-fit/tests/test_qrf.py + packages/microcosm-fit/tests/test_qrf_stateless.py + packages/microcosm-fit/tests/test_qrf_target.py + packages/microcosm-fit/tests/test_signed_reconciliation.py + packages/microcosm-fit/tests/test_weighted_fit_contract.py + packages/microcosm-frame/tests/test_accounting.py + packages/microcosm-frame/tests/test_adapters.py + packages/microcosm-frame/tests/test_axiom_adapter.py + packages/microcosm-frame/tests/test_bundle.py + packages/microcosm-frame/tests/test_contracts.py + packages/microcosm-frame/tests/test_kernels.py + packages/microcosm-frame/tests/test_materialize.py + packages/microcosm-frame/tests/test_place.py + packages/microcosm-frame/tests/test_policyengine_uk_adapter.py + packages/microcosm-frame/tests/test_policyengine_us_adapter.py + packages/microcosm-frame/tests/test_policyengine_us_consumer_index.py + packages/microcosm-frame/tests/test_policyengine_us_metadata_index.py + packages/microcosm-frame/tests/test_policyengine_us_ownership_index.py + packages/microcosm-frame/tests/test_rules_engine_contract.py + packages/microcosm-frame/tests/test_units.py + packages/microcosm-frame/tests/test_weights_schema.py + packages/microcosm-graph/tests/test_acceptance_a_identity.py + packages/microcosm-graph/tests/test_acceptance_b_ownership.py + packages/microcosm-graph/tests/test_acceptance_c_seeds.py + packages/microcosm-graph/tests/test_acceptance_d_weights.py + packages/microcosm-graph/tests/test_acceptance_e_store.py + packages/microcosm-graph/tests/test_acceptance_f_gates.py + packages/microcosm-graph/tests/test_acceptance_g_legibility.py + packages/microcosm-graph/tests/test_acceptance_h_parity.py + packages/microcosm-graph/tests/test_acceptance_replays.py + packages/microcosm-graph/tests/test_frame_metadata_store.py + packages/microcosm-graph/tests/test_graph_acceptance_burndown.py + packages/microcosm-graph/tests/test_graph_canonical.py + packages/microcosm-graph/tests/test_graph_codecs.py + packages/microcosm-graph/tests/test_graph_decl.py + packages/microcosm-graph/tests/test_graph_executor.py + packages/microcosm-graph/tests/test_graph_executor_scalar_float.py + packages/microcosm-graph/tests/test_graph_executor_series_stream.py + packages/microcosm-graph/tests/test_graph_executor_source_identity.py + packages/microcosm-graph/tests/test_graph_explain.py + packages/microcosm-graph/tests/test_graph_interface_lock.py + packages/microcosm-graph/tests/test_graph_kernel_contract.py + packages/microcosm-graph/tests/test_graph_keys.py + packages/microcosm-graph/tests/test_graph_manifest.py + packages/microcosm-graph/tests/test_graph_parity_pins.py + packages/microcosm-graph/tests/test_graph_parity_repin.py + packages/microcosm-graph/tests/test_graph_population.py + packages/microcosm-graph/tests/test_graph_randomness.py + packages/microcosm-graph/tests/test_graph_serialize.py + packages/microcosm-graph/tests/test_graph_storage_parts_whole_series.py + packages/microcosm-graph/tests/test_graph_store.py + packages/microcosm-graph/tests/test_graph_verification_epoch.py + packages/microcosm-graph/tests/test_graph_view.py +[engine-only] excluded from the fast tier, run in the engine tier + packages/microcosm-build/tests/test_us_multispine_pool_tool.py -> us-am + packages/microcosm-build/tests/test_us_release_head_to_head_scorer.py -> us-qs +[defaulted] + packages/microcosm-build/tests/test_chronicle_epoch.py -> shared-spec + packages/microcosm-build/tests/test_compare_us_fiscal_diagnostics.py -> shared-spec + packages/microcosm-build/tests/test_country_spec.py -> shared-spec + packages/microcosm-build/tests/test_demographics.py -> shared-spec + packages/microcosm-build/tests/test_diagnose_us_target_support.py -> shared-spec + packages/microcosm-build/tests/test_frame_checkpoint.py -> shared-spec + packages/microcosm-build/tests/test_frame_sampling.py -> shared-spec + packages/microcosm-build/tests/test_frame_serializer_registry.py -> shared-spec + packages/microcosm-build/tests/test_gate_battery.py -> shared-spec + packages/microcosm-build/tests/test_gate_battery_contract_example.py -> shared-spec + packages/microcosm-build/tests/test_gate_battery_contract_pins.py -> shared-spec + packages/microcosm-build/tests/test_gates.py -> shared-spec + packages/microcosm-build/tests/test_holdout.py -> shared-spec + packages/microcosm-build/tests/test_imputation_lineage_spec.py -> shared-spec + packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py -> shared-spec + packages/microcosm-build/tests/test_ledger_artifact.py -> shared-spec + packages/microcosm-build/tests/test_ledger_targets.py -> shared-spec + packages/microcosm-build/tests/test_logbook.py -> shared-spec + packages/microcosm-build/tests/test_logbook_adoption.py -> shared-spec + packages/microcosm-build/tests/test_logbook_archive.py -> shared-spec + packages/microcosm-build/tests/test_logbook_backfill.py -> shared-spec + packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py -> shared-spec + packages/microcosm-build/tests/test_logbook_cli.py -> shared-spec + packages/microcosm-build/tests/test_logbook_env.py -> shared-spec + packages/microcosm-build/tests/test_logbook_isolation.py -> shared-spec + packages/microcosm-build/tests/test_logbook_prediction_seed.py -> shared-spec + packages/microcosm-build/tests/test_monetary_profile.py -> shared-spec + packages/microcosm-build/tests/test_monetary_targets.py -> shared-spec + packages/microcosm-build/tests/test_outer_stage_runtime.py -> shared-spec + packages/microcosm-build/tests/test_plan.py -> shared-spec + packages/microcosm-build/tests/test_reform_validation.py -> shared-spec + packages/microcosm-build/tests/test_release_input_coverage.py -> shared-spec + packages/microcosm-build/tests/test_release_target_parity.py -> shared-spec + packages/microcosm-build/tests/test_score_uk_national_candidate.py -> shared-spec + packages/microcosm-build/tests/test_source_runtime.py -> shared-spec + packages/microcosm-build/tests/test_stage_profile.py -> shared-spec + packages/microcosm-build/tests/test_staging.py -> shared-spec + packages/microcosm-build/tests/test_staging_v2.py -> shared-spec + packages/microcosm-build/tests/test_stochastic_assignment.py -> shared-spec + packages/microcosm-build/tests/test_target_materialization.py -> shared-spec + packages/microcosm-build/tests/test_target_reference_authoring.py -> shared-spec + packages/microcosm-build/tests/test_trace.py -> shared-spec + packages/microcosm-build/tests/test_validation_input_coverage.py -> shared-spec + packages/microcosm-frame/tests/test_accounting.py -> shared-spec + packages/microcosm-frame/tests/test_adapters.py -> shared-spec + packages/microcosm-frame/tests/test_axiom_adapter.py -> shared-spec + packages/microcosm-frame/tests/test_bundle.py -> shared-spec + packages/microcosm-frame/tests/test_contracts.py -> shared-spec + packages/microcosm-frame/tests/test_materialize.py -> shared-spec + packages/microcosm-frame/tests/test_place.py -> shared-spec + packages/microcosm-frame/tests/test_units.py -> shared-spec + packages/microcosm-frame/tests/test_weights_schema.py -> shared-spec +verification=ok diff --git a/experiments/acs-serial-membership/original.json b/experiments/acs-serial-membership/original.json new file mode 100644 index 000000000..e79b477f5 --- /dev/null +++ b/experiments/acs-serial-membership/original.json @@ -0,0 +1,106 @@ +{ + "all_masks_match_pandas": true, + "benchmark_sha256": "5d50e685ea4ebeb2769bba6cd434eece17e126b436383726c1a6ae62d3beff86", + "chunks": 20, + "first_lookup": { + "cpu_seconds": 0.33849600000000013, + "wall_seconds": 0.339910457842052 + }, + "index_deep_bytes_including_shared_strings": null, + "keys": 200000, + "mode": "original", + "module_sha256": "893fd570cf74bf87c133d98e2a450793ad031ce60a40778fb5ea03b3cb9280c8", + "pandas": "3.0.3", + "preparation": { + "cpu_seconds": 2.0000000000575113e-06, + "wall_seconds": 3.0838418751955032e-06 + }, + "prepared_index_engine_reused": false, + "pyarrow": "25.0.0", + "python": "3.14.4 (main, Apr 14 2026, 14:46:33) [Clang 22.1.3 ]", + "rows_per_chunk": 20000, + "rss_after_lookups_bytes": 409485312, + "rss_before_lookup_bytes": 407896064, + "scope": "invented_arrays_only; not native preparation performance", + "total_cpu_seconds": 6.691459000000003, + "total_wall_seconds": 6.729966166429222, + "warm_lookups": [ + { + "cpu_seconds": 0.3393189999999999, + "wall_seconds": 0.34137666691094637 + }, + { + "cpu_seconds": 0.33743600000000007, + "wall_seconds": 0.3397197499871254 + }, + { + "cpu_seconds": 0.3358890000000003, + "wall_seconds": 0.3384829161223024 + }, + { + "cpu_seconds": 0.3301660000000002, + "wall_seconds": 0.33123829215765 + }, + { + "cpu_seconds": 0.3281459999999998, + "wall_seconds": 0.3287552078254521 + }, + { + "cpu_seconds": 0.328306, + "wall_seconds": 0.3294437499716878 + }, + { + "cpu_seconds": 0.32299299999999986, + "wall_seconds": 0.3233217499218881 + }, + { + "cpu_seconds": 0.3580370000000004, + "wall_seconds": 0.37026783311739564 + }, + { + "cpu_seconds": 0.3380220000000005, + "wall_seconds": 0.33984450018033385 + }, + { + "cpu_seconds": 0.33664000000000005, + "wall_seconds": 0.3371821250766516 + }, + { + "cpu_seconds": 0.3283490000000002, + "wall_seconds": 0.3295408331323415 + }, + { + "cpu_seconds": 0.335947, + "wall_seconds": 0.33823804184794426 + }, + { + "cpu_seconds": 0.3306930000000001, + "wall_seconds": 0.332630000077188 + }, + { + "cpu_seconds": 0.3262930000000006, + "wall_seconds": 0.3279706249013543 + }, + { + "cpu_seconds": 0.3291460000000006, + "wall_seconds": 0.3301870001014322 + }, + { + "cpu_seconds": 0.3377799999999995, + "wall_seconds": 0.33863633312284946 + }, + { + "cpu_seconds": 0.34295900000000046, + "wall_seconds": 0.34419145807623863 + }, + { + "cpu_seconds": 0.3380799999999997, + "wall_seconds": 0.339578292099759 + }, + { + "cpu_seconds": 0.32875999999999994, + "wall_seconds": 0.32944725011475384 + } + ], + "whole_process_peak_rss_bytes_macos": 428949504 +} diff --git a/experiments/acs-serial-membership/prepared.json b/experiments/acs-serial-membership/prepared.json new file mode 100644 index 000000000..ad46e2fec --- /dev/null +++ b/experiments/acs-serial-membership/prepared.json @@ -0,0 +1,106 @@ +{ + "all_masks_match_pandas": true, + "benchmark_sha256": "5d50e685ea4ebeb2769bba6cd434eece17e126b436383726c1a6ae62d3beff86", + "chunks": 20, + "first_lookup": { + "cpu_seconds": 0.012153999999999998, + "wall_seconds": 0.013858875026926398 + }, + "index_deep_bytes_including_shared_strings": 17627112, + "keys": 200000, + "mode": "prepared", + "module_sha256": "893fd570cf74bf87c133d98e2a450793ad031ce60a40778fb5ea03b3cb9280c8", + "pandas": "3.0.3", + "preparation": { + "cpu_seconds": 0.011846000000000023, + "wall_seconds": 0.012051749974489212 + }, + "prepared_index_engine_reused": true, + "pyarrow": "25.0.0", + "python": "3.14.4 (main, Apr 14 2026, 14:46:33) [Clang 22.1.3 ]", + "rows_per_chunk": 20000, + "rss_after_lookups_bytes": 407994368, + "rss_before_lookup_bytes": 407060480, + "scope": "invented_arrays_only; not native preparation performance", + "total_cpu_seconds": 0.05246100000000009, + "total_wall_seconds": 0.05446754000149667, + "warm_lookups": [ + { + "cpu_seconds": 0.0018370000000000886, + "wall_seconds": 0.0018601249903440475 + }, + { + "cpu_seconds": 0.0016259999999999053, + "wall_seconds": 0.0016246249433606863 + }, + { + "cpu_seconds": 0.0015140000000000153, + "wall_seconds": 0.001539166085422039 + }, + { + "cpu_seconds": 0.001682999999999879, + "wall_seconds": 0.0016859162133187056 + }, + { + "cpu_seconds": 0.001527000000000056, + "wall_seconds": 0.0015269580762833357 + }, + { + "cpu_seconds": 0.0015130000000000976, + "wall_seconds": 0.0015121668111532927 + }, + { + "cpu_seconds": 0.00143099999999996, + "wall_seconds": 0.001430625095963478 + }, + { + "cpu_seconds": 0.0013640000000001429, + "wall_seconds": 0.0013634159695357084 + }, + { + "cpu_seconds": 0.0016320000000000778, + "wall_seconds": 0.001649708952754736 + }, + { + "cpu_seconds": 0.001562999999999981, + "wall_seconds": 0.001562874997034669 + }, + { + "cpu_seconds": 0.0015339999999999243, + "wall_seconds": 0.0015337499789893627 + }, + { + "cpu_seconds": 0.0013920000000000599, + "wall_seconds": 0.001392250182107091 + }, + { + "cpu_seconds": 0.0013369999999999216, + "wall_seconds": 0.0013375419657677412 + }, + { + "cpu_seconds": 0.001346999999999987, + "wall_seconds": 0.001363749848678708 + }, + { + "cpu_seconds": 0.0015739999999999643, + "wall_seconds": 0.0015869159251451492 + }, + { + "cpu_seconds": 0.0015049999999998676, + "wall_seconds": 0.0015045409090816975 + }, + { + "cpu_seconds": 0.0014180000000001414, + "wall_seconds": 0.0014184580650180578 + }, + { + "cpu_seconds": 0.0013360000000000039, + "wall_seconds": 0.001335417153313756 + }, + { + "cpu_seconds": 0.0013279999999999959, + "wall_seconds": 0.0013287088368088007 + } + ], + "whole_process_peak_rss_bytes_macos": 420200448 +} diff --git a/experiments/acs-serial-membership/tests-with-owner.txt b/experiments/acs-serial-membership/tests-with-owner.txt new file mode 100644 index 000000000..ea71cf6f3 --- /dev/null +++ b/experiments/acs-serial-membership/tests-with-owner.txt @@ -0,0 +1,4 @@ +........................................................................ [ 38%] +........................................................................ [ 77%] +......................................... [100%] +185 passed in 8.02s diff --git a/experiments/acs-serial-membership/tests.txt b/experiments/acs-serial-membership/tests.txt new file mode 100644 index 000000000..94a80a0a4 --- /dev/null +++ b/experiments/acs-serial-membership/tests.txt @@ -0,0 +1,2 @@ +............................................................... [100%] +63 passed in 0.70s diff --git a/experiments/asec-usual-hours-observations-20260919.json b/experiments/asec-usual-hours-observations-20260919.json new file mode 100644 index 000000000..ef3a0724e --- /dev/null +++ b/experiments/asec-usual-hours-observations-20260919.json @@ -0,0 +1,84 @@ +{ + "scope": "Pure ASEC all-age observations and invented source-owner regressions; no actual data or all-person output", + "base": "41fe226d82a2b41fe911f8e04b4dc1d503d14af3", + "source": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_usual_hours.py": "9660045cd2df2d9a69157e47f626eb77e6ebcfa98a8bb12e66e320307e1476cd", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_hours.py": "bfc79e2c580fa24c0e8afac6f5cfacd643d2cdf7fcbf1d2ffb960eaa5e26ca89", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_hours_source.py": "953c61db93d1c5b85755dc84287a8d5535571ac265f968a1d5c451592eb06b74", + "packages/microcosm-build/tests/test_us_current_asec_usual_hours.py": "4c33fff8f50177a5f5e8755a8a6f60c4c1c793f1de00c34d6d356f7bf725f050" + }, + "tests": { + "all_age_observations": 37, + "source_owner": 24, + "existing_pure_recoder": 54, + "failed": 0, + "skipped": 0 + }, + "result_sha256": "f1e9ac23f774c56d958c3f23bd83b3680d46b6152a21567dad4e20fb4479f4a8", + "xml_sha256": "0de4b0f841c4a18cce845044f64cf51b6ac61208345f6b174c9bd711371202f1", + "wall_seconds": 149.05698354099877, + "cpu_seconds": 115.30500799999999, + "peak_rss_bytes": 494436352, + "primary_dictionary": { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "visually_checked_pdf_pages": [ + 38, + 41 + ] + }, + "primary_api_metadata": [ + { + "name": "HRSWK", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/HRSWK.json", + "sha256": "adf6a9e2034b18d7216eff32bfbf0df6938e76b2032fb3d3d3adb2e27af4d56f" + }, + { + "name": "WKSWORK", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/WKSWORK.json", + "sha256": "4bcff6dbc5569c1341857cebae96cd8acd5bc3ae6fd875448a99b510eb06802a" + }, + { + "name": "WORKYN", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/WORKYN.json", + "sha256": "eba9950f0b21c96ffa4cf1526e5429e5daecbd91000c10f1751b3f9ca7949fb5" + }, + { + "name": "WTEMP", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/WTEMP.json", + "sha256": "48451e221582750fa8b4031d5d8a653ccfeaaccba82345ffdeb55054de3cae18" + }, + { + "name": "WRK_CK", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/WRK_CK.json", + "sha256": "09c41254643089d6bbb776d4332574e709c4215f9a76c6af353d1890de2f0dc5" + }, + { + "name": "A_AGE", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/A_AGE.json", + "sha256": "4b2222c321d110f6b4e7536046ec02bc5727772cb84034af654ee9f14e87a98a" + }, + { + "name": "WSAL_VAL", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/WSAL_VAL.json", + "sha256": "d4cd19311d821b481da5707ea93d9ed03df409e3f2d6288438b509109d9428a7" + }, + { + "name": "SEMP_VAL", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/SEMP_VAL.json", + "sha256": "2f2c969321d776622d807a52550cfe929f33b2dd86392ee4464459efea1861cd" + }, + { + "name": "FRSE_VAL", + "url": "https://api.census.gov/data/2025/cps/asec/mar/variables/FRSE_VAL.json", + "sha256": "f8d8b57de8e4b55eb2997153622977f5ec6416d716b520a704ff9f5616e97451" + } + ], + "review": "Root source review; no independent peer after quota limitation", + "remaining": [ + "Actual all-age source audit", + "Explicit under15 modeled completion with earnings conflict checks", + "Source owner/graph/clone integration", + "Release no-worse comparison" + ] +} diff --git a/experiments/child-property-empirical-candidate-20260913.md b/experiments/child-property-empirical-candidate-20260913.md new file mode 100644 index 000000000..1e9d8f58a --- /dev/null +++ b/experiments/child-property-empirical-candidate-20260913.md @@ -0,0 +1,23 @@ +# Child property-income empirical candidate + +This source-only candidate implements an explicit transfer from original ASEC persons aged15–17 to explicitly unmeasured children aged0–14. It is opt-in methodology under review, not a released default or a measured child-income distribution. No native fit or test execution has occurred in this commit. + +`microcosm.fit.joint_empirical` fits a pooled weighted joint law for two nonnegative targets. It classifies each donor into neither, first-only, second-only or both-positive, then draws one paired observation within the chosen pattern. It uses Microcosm's typed/explicit weight resolvers. It does not run QRF or independently sample marginal amounts. + +For observed weighted pattern probabilities p, receipt factor r produces q_g=r*p_g for positive patterns and q_00=1-r*sum(p_positive). It refuses r<0, nonfinite values or r*sum(p_positive)>1; no probability is clipped. Two strictly positive amount factors scale the paired donor coordinates. Source-positive amounts lost to underflow or overflow refuse. Reference factors are one. r=0 is a separately labeled all-zero stress, still requiring the same fitted support-qualified donor population. + +Cumulative positive weights are normalized by their own finite positive endpoint. This declared normalization makes the final CDF endpoint one; non-increasing positive intervals refuse. Uniform coordinates use half-open intervals. The numerical operator accepts supplied uniforms; the separately owned graph adapter uses the maintained typed `keyed_uniform`, with exact original source coordinates rather than row positions, clone IDs or graph keys. Model JSON preserves integer coordinate types with decimal-string tags, including IDs above2**53. Descriptive model bytes cannot issue source authority. + +Reference transport retains the fitted zero-pattern mass directly rather than reconstructing it by subtracting rounded positive probabilities from one. This preserves representable leading intervals such as a zero donor with weight `2**-54` beside a positive donor with weight one. Every present positive-mass pattern, including the zero-income pattern when not named as required, must meet the declared pattern minima. Binary64 is the numerical input contract: floating dtypes wider than 64 bits refuse before target, explicit weight or uniform conversion. Where NumPy's `longdouble` aliases binary64, it remains supported; tests do not claim to exercise an unavailable extended-exponent representation. Already constructed Frame weights retain the Frame's own binary64 input contract. + +Support policy is mandatory and has no native default. It records positive-mass row/household counts, person and household effective sizes, separate overall/positive/pattern minima, and required patterns. Zero-weight observations stay visible in raw diagnostics but are never sampled or counted as positive support. Absent nonrequired patterns retain empirical zero and an explicit absence diagnostic; observed rare patterns cannot bypass the declared minima. Invented tests use named test policies. Native adequacy thresholds and sensitivity criteria require separate review. + +The US source qualifier calls the existing actual preparation and complete ASEC catalogue owners. Existing interest/dividend qualifiers already authenticate all current-year member rows before projecting selected `.person` tables; their full `.asec_literals` supply the donor pool. Exact full roster joins retain original household weights using `Fraction(HSUP_WGT,100)`, with no sample or clone multiplier. Ordinary interest is `TRDINT_VAL`, not combined `INT_VAL`; dividends are `DIV_VAL`. Missing retirement/rental information does not disqualify a known pair. Allocation/edit/topcode descriptors, including the existing I_DIVYN codebook ambiguity, remain descriptive provenance. + +The recipient path admits only jointly unknown, explicitly unmeasured under15 O/D observations. Source errors take precedence. It refuses contradictory or positive out-of-universe ASEC values, numeric out-of-universe ACS INTP, invalid ACS adjustments hidden by blank-status precedence, and unsupported known/partially-known incumbents. Source amounts and knownness are never rewritten by this module. Future model zeros must be labeled imputed, not observed nonreceipt. + +The qualifier extends maintained source identity and live function/default/closure fences around initial and final actual owner borrows. Its returned tables remain descriptive: downstream owners must retain and requalify original source owners, verify model/scenario artifacts and independently reconstruct the complete receiving population. A copied receipt or DataFrame cannot grant that authority. The new source tables are bounded to600,000 full source rows; the64MiB encoding ceiling applies to encoded metadata/individual coordinate data here, not a demonstrated resident-memory bound. The existing parent and future runtime guards retain their independent limits. + +The interpretation seal includes the routing domain, receipt, account and allocation constants used by the literal projectors. The unpublished-allocation set is represented by its type and sorted immutable values, so changing its contents cannot silently change provenance. Authored actual-owner controls preserve an allocation flag on the excluded teenager, compare raw original catalogue weights with projected weights, refuse malformed or contradictory excluded-donor literals pinned before issuance, and mutate detached projections during final catalogue/preparation callbacks. These controls remain unrun pending guarded acceptance. + +This commit changes no receiving graph, canonical columns, source-knownness, tax rules, PUF allocator, calibration, dataset, paper/site deployment or release gate. The graph integration lane owns separate private model-support CREATE branches with exact schema Slices; it will leave the receiving survey spine, clone expansion and geography unchanged. Whole-Frame reconstruction, composed cold/required replay, child-containing PUF admission, native donor diagnostics, household holdouts and scenario sensitivity remain pending. diff --git a/experiments/fiscal-target-snapshot-host-20260912.md b/experiments/fiscal-target-snapshot-host-20260912.md new file mode 100644 index 000000000..824fe94e2 --- /dev/null +++ b/experiments/fiscal-target-snapshot-host-20260912.md @@ -0,0 +1,113 @@ +# Target snapshots through the US fiscal graph + +The actual `FiscalDenseCalibrationKernel` now accepts an optional host-owned +`TargetSnapshotObserver` and forwards it to its existing grouped `calibrate` +call. Default construction and explicit `None` preserve the unobserved path. +Observer configuration stays on the kernel instance; it is absent from graph +parameters, implementation identity inputs, cache keys, artifacts and receipts. +The [usage documentation](../docs/calibration-target-snapshots.md#us-fiscal-host) +shows how a caller supplies the sink. + +This integration combines grouped core `2f62bb8448010bcc9b9e1419ea24aafcb85917d1` +with PR #914 head `16fc6cb501ccb58ceb158e9064bf4f39ee23479c`, including reviewed +repair `fbb109987e30f09ccc87fc402c47190d7c3ecfd2`. The grouped loop now reports +completed optimizer updates, starting at zero, while cadence continues to use +the one-based evaluation ordinal. Its selected snapshot remains the final +closing state at `epoch == epochs`, with no retained best. Ordinary Adam, +proximal and winning budget-probe labels preserve their reviewed repairs. +No numeric update, accepted projection, optimizer selection or RNG operation +was changed by the added fiscal observer boundary. + +## Integrated execution evidence + +The full calibration test directory, existing dense fiscal tests and nine new +fiscal snapshot controls completed with **473 passed, one skipped and no +failures or errors**. The run took 12.051 seconds wall time, 12.375 CPU seconds +and 491,569,152 bytes peak RSS. All 478 Python source files under shard source +directories retained their hashes; Torch threads were 1/1. + +The one skip is the saved prepatch exact-byte fixture. It requires its recorded +Torch interop setting of 18, while this bounded run used one. The current +on/off numerical comparisons and all new fiscal controls executed; the skipped +historical comparison is not claimed as a pass. The earlier grouped lane's +443-test evidence remains separate from this integrated run. + +The new controls exercise the real solver and tiny invented fiscal graph: + +- Default, explicit `None`, and observers with different labels/cadences produce + exactly equal weights, graph artifacts and receipts. Kernel implementation + hashes and declared node identities are equal across observer configurations. +- Snapshots use the actual measurement's ordered target names and values. The + emitted in-loop estimates equal the existing float32 loss tensors, and final + estimates equal the final float64 diagnostics. A six-update run at cadence + three emits completed-update labels 0, 2, 5 and selected 6. +- Every delivered dictionary is detached. A sink can damage its own aggregate + metadata and target rows without changing later snapshots or returned results. +- Exceptions from current and selected sinks propagate. Selected sinks that + corrupt live measurement values, household IDs or accepted weights are + refused by the existing final checks after callback completion. +- Actual graph executions with observation off/on have identical node keys, + weights and artifact bytes. Required replay with a fresh failing observer and + forbidden optimizer hook hits every cached node and invokes neither hook. + A cache replay therefore generates no new optimizer snapshots. + +The first attempt is preserved as failed test evidence. Its final-weight +corruption control tried normal assignment to an already read-only array and +stopped before reaching the intended final guard. The corrected test explicitly +injects a corrupted live weight container; final acceptance then refuses it. +This was a test-only correction; production solver and host bytes were unchanged. + +| Local evidence | SHA256 | +| --- | --- | +| Integrated execution receipt | `b4fb9b48a1d1ca4739e6d22d78968eae14a4855789548c9805068b7ffac2d38f` | +| Integrated JUnit result | `68b110709ade099efc3084af2e0adae0944d6a251172bce8496a3c265e20ed00` | +| Fiscal host source | `c26a9484b596015b4ffaf6a6a1e68e45f0ab4e81e63b5b612149777b100d56b9` | +| New fiscal snapshot test source | `f60d71aa9a4ac6bcf897dd937fe787eb21b0866cdd04223dba513af684849fcb` | +| Reconciled solver source | `8f22f325ed06cde31c4c25467efa39a43ba8ccef9598b14e468701f3263c1010` | + +These bounded wrappers apply runtime/thread limits and source hashes. They are +not the strict native pilot audit guard. The reviewed execution path uses +invented sources and no country simulation; flags or table counts alone do not +establish native/source admission. + +## Source-derived identities + +The merge had five derived-identity conflicts. Final values were recomputed +from the combined checkout using the existing spec loader/compiler, complete +coverage verifier and selective calibration parity tool. No parent branch's +hash was accepted as the final value. Fit and simulation parity pins were left +byte-identical. The parity tool also checks that the local direct calibration +output remains equal to its recorded bytes before rewriting its key. + +| Identity | Final value | +| --- | --- | +| Seed protocol | `5f93b3ec98ada30338b06ad978a6e1b47d0f9f916af89418500cea435235ad06` | +| Seed map | `da07a54ab2bc4e297ac7d0693a8559b359de230787757e4b6e26dcb619c8e5a0` | +| US resolved spec | `54aa5d96b062a66207dfe49d4a03817e9658bd0ac11acb37546b1bf2a01f533d` | +| Minimal loader golden | `a866bfe36a9eeb3b9a9888466b4b906faf4d8da57daf380d9bbc8ccf22e1e048` | +| Fiscal kernel implementation | `3d68e0f9b95a6d07f7a61b93f2044157ba4ab925e09a0a9600beebc16b60b8b7` | +| Calibration parity node | `f85d2af2276acc617779567e7ce8c97878febda0062acd7d2ddf326f4629d520` | + +The first identity-helper invocation correctly refused two advertised source +roots from the shared editable environment. Its failure is preserved. The +successful invocation disabled automatic site initialization and supplied the +owned shard roots plus dependency site-packages explicitly. No source-identity +check was relaxed. Its report SHA256 is +`69666f687e025d9a27d9d76b2cc87fefc2622734a56fc4530674b9de019ff23d`. + +Five focused checks of the final minimal/country spec goldens and calibration +parity identities then passed without failures or skips. They took 58.312 +seconds wall time with all 478 source pins unchanged. The receipt SHA256 is +`ee662e419c17acc27b76761b32f2bdbb096f2dc993e1efeb437d3f01006573e8`; +the JUnit result SHA256 is +`a60a4ffcc11b4afde6fe64a332c2bcfe54b8ac298f9aa523d9a4279089ccd116`. +No UK or US country-simulation test was selected. + +## Remaining acceptance + +This is solver and fiscal-host instrumentation, not a dataset release or +dashboard deployment. Complete-population/source authority remains external. +Native calibration quality, observer overhead at production target counts, +cadence choice, staging upload and a dashboard consumer need their own evidence. +Retain prior histories with their actual run identities; do not label cached +replay as new optimizer progress. diff --git a/experiments/grouped-target-snapshot-integration-20260912.json b/experiments/grouped-target-snapshot-integration-20260912.json new file mode 100644 index 000000000..5f7c0e72b --- /dev/null +++ b/experiments/grouped-target-snapshot-integration-20260912.json @@ -0,0 +1,90 @@ +{ + "lane": "grouped-target-snapshot-integration-20260912", + "date": "2026-09-12", + "scope": "shared calibration solver seam only; no host wiring, no native build, no release action, nothing pushed", + "base": { + "origin_main": "116d46ee9dc2aafdc68259b7c06e4c3462522e8b", + "moved_since_pin": false, + "note": "origin/main was fetched and is byte-identical to the shared base both reviewed heads were cut from, so no main drift had to be preserved" + }, + "integrated_heads": { + "grouped_us_solver": "536f1ceefcdafda3cc619c14b4da18e012a7be57", + "target_snapshots_pr914": "b43369dc49e175803f62020cc1e72fa53926aed8", + "merge_base_of_the_two": "116d46ee9dc2aafdc68259b7c06e4c3462522e8b" + }, + "pinned_source_hashes_verified_before_merge": { + "grouped/solve.py": "925a1b6bae1c714de93df3ee44e26afeed7f3912377b93942f3a5173281d851a", + "grouped/calibrate__init__.py": "69ac527f4732cdb67c4807bcb8ee9faec56174621f663ec491140dfc324a705f", + "snapshots/solve.py": "f43be0efecdd21b3f5caa9789edc8463a127423c665523ac7f0c8bc48f4b4eed", + "snapshots/calibrate__init__.py": "6b66552827852dd16a36512bdef76c28e47842d9a6b0b59d176a4a18a71f48ae", + "all_four_matched_the_pin_file": true + }, + "commits": [ + {"sha": "fff20fca56f41060db55742b884a6df2693e139f", "what": "merge of the two reviewed heads; neither solver replaced"}, + {"sha": "e64d5a8850ab301b7e819a6b972f235c51688568", "what": "grouped Adam snapshot emission and retain-best separation"}, + {"sha": "7ac2b864a93977ffa98fa86674c09e04ea6b20b5", "what": "source-derived identity recomputation"}, + {"sha": "2f62bb8448010bcc9b9e1419ea24aafcb85917d1", "what": "grouped x snapshot cross-product controls"} + ], + "final_source_hashes": { + "packages/microcosm-calibrate/src/microcosm/calibrate/solve.py": "e031ab5823d0f7bff43642e6ebb160ad867ec387e88658534ccd1e6ef62b273d", + "packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py": "1841264fe0c09a96ccf6aacf467cfa201de92ee67185b00cd378456fc7631a78", + "packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py": "a86958e3e94ca2454d997567883e2223e832f7edb5d994f689f0a64ddad1a88a", + "packages/microcosm-calibrate/src/microcosm/calibrate/group_bounds.py": "19a440a31d164832f61c84b2c1277a7a8798681f6830700bf03511dce7524557" + }, + "identity_recomputation": { + "rule": "recomputed against the merged checkout and the isolated interpreter; no value equals either branch's, so none is an ours/theirs choice", + "values": [ + {"artifact": "spec_engine/inventory_coverage.py EXPECTED_HASHES.seed_protocol", "grouped": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47", "snapshots": "d052fd875d014f6fa28d6f5b052735912f464d4efb2c98ecf373346b8cc454d3", "recomputed": "57e21b6522cfccc9b2d5e725503d9d39255abe5d5c9b391563d08ee664dea988"}, + {"artifact": "spec_engine/inventory_coverage.py EXPECTED_HASHES.seed_map", "grouped": "be2d24ad74bdc6339dfa073d01469074baee4e3a691eaa350451d36fa17d13dc", "snapshots": "d5a9694aac3e351f987b4f08ef8add0c25fcd3390d6bb2a851246da03adde516", "recomputed": "4ae2a0634e3a92061beba5bf5059db7c30b4ff6f1898e639697d246cb7e73878"}, + {"artifact": "us resolved spec_sha256 (us-f0-coverage.json and test_us_multispine_pool_tool.py)", "grouped": "f03e4d08d0ba4e15da53aab86bd2fbf50286e19e248657ac859b760103af2405", "snapshots": "ff2c97039853e18bfc0eb5388eb08c688cae33068b742f232d4c06a18b8efd88", "recomputed": "15e61194b4077de58272b14d83af95f7f54427aeab251429ada3cda01d93cbc8"}, + {"artifact": "test_spec_engine_loader.py minimal-loader golden", "grouped": "77f043a13853bbc109b6aa8cb42635ab1b2ebea69a1997b5bdb4c78a3726aeb3", "snapshots": "8a24089831b3a8f0cfe1be094769af7a30b7b74dbc7d4009e26e9f56444cba4c", "recomputed": "3868d14d5df04b5ab10ed4f0a56c2d9652c15c219840ed88d437f847fa481de7"}, + {"artifact": "parity kernels/calibrate/pins.json implementation_hash", "grouped": "59ca046b7c26bd872e56bd4cf3dd6170748fbe04e5a5da7b711e94aea8d3cc74", "snapshots": "46b52a20a7a5fac27d354c735a09ae5ba171d62bd9010fbc3079b06614cc2a32", "recomputed": "f2dd92eaaebba782e100cb92d90524d3ae6e2bab9ffedbacbe8df0257d5cae3d"}, + {"artifact": "parity kernels/calibrate/pins.json node_key", "grouped": "faaccb273d0aedeb1cca2e1b52c3e27678867512493f3c0379f596890c258cd1", "snapshots": "970720b71cb2579a5e2f714ca47deabc28c9249a83007ebfda83e2e7446d9311", "recomputed": "ae92142a3bf0934ac6b6751aca904c1847cdff3ca8533af83141d2cbf1ccf102"}, + {"artifact": "test_spec_engine_country_bundles.py am", "grouped": "c15bd12508731eb11fe061df810d9047b4c62c2887645cb19d3c54c7a104aa1c", "snapshots": "untouched-by-S", "recomputed": "258b940476e9f611d5e12d471dead7427479f36267eb40ed459fcd2f3dbd18ae"}, + {"artifact": "test_spec_engine_country_bundles.py be", "grouped": "5285b530974a9b8f7c21497fb140fdbd6060c3f1cc983987f35e9f244e71ee62", "snapshots": "untouched-by-S", "recomputed": "a61b65f7a1424c090081632fb1ddacdb85dae1b85920faff5dd7cb144cb1b5a1"}, + {"artifact": "test_spec_engine_country_bundles.py uk", "grouped": "1ac642230eb3166dd9ea2a113536985e35bf2898f914e289f5564ac8fde6e0b4", "snapshots": "untouched-by-S", "recomputed": "66837fd0fad4956566655f558d10f8d39e83523ad873b17e9f37ce0cb5012724"} + ], + "deliberately_not_repinned": [ + "packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json (G's value preserved; it hashes microcosm/frame/bundle.py, not calibrate.solve)", + "packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/ (neither head changed microcosm/fit/qrf.py)", + "parity direct.csv outputs (the calibrate kernel's outputs did not move, only its implementation hash)" + ], + "tooling": { + "us-f0-coverage.json": "tools/spec_engine_coverage.py (then --check exits 0)", + "calibrate parity pin": "tools/graph_parity_repin.py calibrate", + "loader golden and bundle digests": "taken from the observed value the failing identity assertion reports" + } + }, + "diff_summary": { + "vs_grouped_head": {"files": 25, "insertions": 4287, "deletions": 20, "note": "every deletion is an identity pin line, an extended import, an extended docstring, or a line S replaced; nothing from the grouped head was lost"}, + "vs_snapshot_head": {"files": 432, "insertions": 148099, "deletions": 3251, "note": "the grouped branch's whole delta"} + }, + "controls": { + "microcosm-calibrate shard": {"passed": 443, "failed": 0}, + "microcosm-graph shard + test_us_multispine_pool_tool + 4 spec-engine identity files": {"passed": 775, "failed": 0}, + "per_file": { + "test_target_snapshots.py": {"cases": 52, "added": 0}, + "test_grouped_fixed_support.py": {"cases": 42, "added": 18}, + "test_group_bounds.py": {"cases": 82, "added": 3}, + "test_solve_us_launch_integration.py": {"cases": 12, "added": 4} + }, + "tools/spec_engine_coverage.py --check": "exit 0", + "tools/ci_test_groups.py --verify": "verification=ok; no test file changed lane and none is [defaulted]", + "ruff": "format clean, check clean on packages/microcosm-calibrate" + }, + "environment": { + "interpreter": "borrowed root .venv/bin/python 3.14.4 run with -I -B -S and an explicit sys.path", + "sys_path_order": "this worktree's packages/*/src first, then the repo root, then the borrowed site-packages", + "imports_asserted_in_worktree": ["microcosm.frame", "microcosm.calibrate", "microcosm.build", "microcosm.fit", "microcosm.graph"], + "threads": "OMP/OPENBLAS/MKL/NUMEXPR/VECLIB all 1", + "pytest": "PYTEST_DISABLE_PLUGIN_AUTOLOAD=1, -p no:randomly", + "locked_versions": {"numpy": "2.4.6", "pandas": "3.0.3", "scipy": "1.17.1", "torch": "2.12.0", "platform": "arm64/darwin/py3.14"}, + "nothing_installed": true + }, + "not_done_on_purpose": [ + "host observer wiring in us_runtime/graph_fiscal_dense_calibration.py", + "any global observer registry, Node.params callback or filesystem path in graph params", + "any claim that a required replay reruns the optimizer", + "native build, PUF fixture, country build, dependency install, push, PR, publication or merge" + ] +} diff --git a/experiments/grouped-target-snapshot-integration-20260912.md b/experiments/grouped-target-snapshot-integration-20260912.md new file mode 100644 index 000000000..5e317e0bf --- /dev/null +++ b/experiments/grouped-target-snapshot-integration-20260912.md @@ -0,0 +1,130 @@ +# Grouped solver x calibration target snapshot integration — 2026-09-12 + +Historical scope note, 12 September 2026: this records grouped core +`2f62bb8448010bcc9b9e1419ea24aafcb85917d1` and its earlier test run. The later +[fiscal-host integration](fiscal-target-snapshot-host-20260912.md) adds the +host observer and PR #914's completed-update convention. The 443-test result +and host-wiring gaps below remain evidence of this earlier revision. + +Scope, evidence and residual risks for branch +`grouped-target-snapshot-integration-20260912`. Source-only; no native build, +no PUF fixture, no country build, no release action, nothing pushed. + +## What this is + +The bounded integration of the reviewed calibration target snapshot observer +(microcosm#908, head `b43369dc49e175803f62020cc1e72fa53926aed8`) with the +reviewed US grouped/fixed-zero Adam solver (head +`536f1ceefcdafda3cc619c14b4da18e012a7be57`), on a fresh `origin/main` +(`116d46ee9dc2aafdc68259b7c06e4c3462522e8b`). Main had not moved from the +shared base, so no main drift had to be preserved. + +Executed against the read-only design checklist +`grouped-snapshot-integration-review.md` and its pins. All four pinned source +hashes on both heads matched the pin file byte for byte before the merge. + +## What it is not + +- **Not** host wiring. `us_runtime/graph_fiscal_dense_calibration.py` still + calls `calibrate` without an observer, so this branch produces no snapshot + file for the actual US fiscal path. Wiring it is a separate, explicitly + host-owned step. No global observer registry, no callback or filesystem path + in graph `Node.params`, and no claim that any replay reruns the optimizer. +- **Not** a dashboard, a staging upload, or a cadence/performance acceptance. +- **Not** a release, a publication or a promotion of any artifact. + +## The one real code change + +A clean textual merge leaves grouped runs uninstrumented: `_optimize` returns +early into `_optimize_grouped` before reaching any snapshot hook S added to the +ordinary Adam loop, so a caller passing `target_snapshots` to a grouped +`calibrate` received exactly one closing snapshot and no trajectory. + +1. `_optimize` forwards its bound observer into `_optimize_grouped` + (`snapshots=snapshots`), and `_optimize_grouped` takes a default-`None` + `snapshots` argument. +2. `_optimize_grouped` emits inside the epoch loop, after the existing progress + callback and before `backward()`, from the exact float32 `estimate` tensor + the epoch's loss was computed from: `iterate=current`, `precision=float32`, + `loss=trajectory[epoch]`, `best_retained={available: False, epoch: None, + loss: None}`. +3. `calibrate` reads retain-best off the solver actually used rather than + inferring it from an empty `iterate_selection_receipt`, and stamps the + grouped closing snapshot at `epoch == epochs`. The receipt itself stays + empty, exactly as before. +4. Grouped snapshots carry three bounded aggregate scalars in `selection`: + `rule: "closing_state"`, `constraint_mode: "grouped_upper_bounds"`, + `grouped_preserve_zeros: `. + +Nothing else in either solver moved. No optimizer or matrix dtype, active mask, +zero template, L2 denominator, log/exp operation, optimizer construction, seed, +projection call, accepted-buffer assignment or `log_w` overwrite changed; there +is still exactly one `log_w` overwrite per accepted projection, the final no-op +projection and accepted-byte equality are unchanged, and the stored-result +admission (weight bytes, fixed-zero mask, bounds, ordered household IDs) still +runs after the last sink. + +### Why the `selection` labels + +Grouped Adam is a closing-state algorithm; it never runs the retain-best rule. +Without a label, its `best_retained.available: False` is indistinguishable from +an ordinary run whose retain-best rule happened to be off, and a consumer would +read the two the same way. Checklist section 2 sanctions exactly this — +"Optional grouped labels must be bounded aggregate scalars, e.g. constraint +mode and whether fixed-zero support is enabled". The labels mirror the run's +own `options["iterate_selection"]` and reuse the receipt's `rule` vocabulary; +they are three JSON scalars carrying no record-level content. This is the one +place the integration goes beyond the checklist's literal field list for +in-loop snapshots, and it is a single expression to remove if root disagrees. + +## Evidence + +- Grouped runs are bit-identical with the observer on and off across + dense x CSR, positive x fixed-zero support, and plain / warm-started / L2: + returned weights, loss trajectory, options, closing loss, stored frame + weights, ordered IDs, fixed-zero bytes, and the full sequence of private + post-projection proof payloads. +- Exactly 20 `_apply_constraint` and 3 `problem.estimates` calls with the + observer on and off on the 20-epoch fixture: cadence buys observation, not + evaluation. Torch and numpy RNG state are identical after each pair of runs. +- Each in-loop row carries the exact float32 tensor the epoch's loss was + computed from, verified against a wrapped `_apply_constraint`, and differs on + all 20 epochs from a recomputation on the accepted vector the private + observer reports next. +- The closing row equals the reused float64 final diagnostics estimates. +- A sink that ruins every delivered payload changes neither the weights nor the + next payload. A sink that mutates live frame IDs at the closing emission + still trips the ordered-ID admission guard. +- Zero epochs through the internal grouped seam emits nothing and evaluates + nothing. +- The deliberately oscillating example still selects grouped closing output + while ordinary Adam selects an earlier best, and each solver's closing + snapshot now says which rule produced it. + +Identity recomputation. Editing `solve.py` — an attested kernel module — moves +the seed protocol digest, every resolved-spec digest that folds it in, and the +calibrate kernel's implementation hash. Each value below was recomputed against +this checkout with the isolated Python 3.14 interpreter; none equals either +branch's value, so none is an ours/theirs choice. The simulate parity pin and +the fit.qrf pin were deliberately left alone: neither hashes +`microcosm.calibrate.solve`, and both are byte-unchanged from the grouped head. + +## Residual risks + +1. **Cadence cost on a real US run is unmeasured here.** The only overhead + evidence in the tree is #908's synthetic benchmark; this branch adds no + native measurement. A country-scale cadence choice is still unowned. +2. **The `selection` labels are an integration judgement**, not a checklist + requirement, on the in-loop rows specifically. See above. +3. **Host wiring remains the gap between this and any dashboard.** Green tests + here say the solver seam is correct; they say nothing about whether any real + US calibration emits anything, because none does yet. +4. **The identity recomputation is only as good as its interpreter.** Digests + fold locked dependency versions; a lane on different locked versions will + compute different parity keys. The values here were taken under + numpy 2.4.6 / pandas 3.0.3 / scipy 1.17.1 / torch 2.12.0 on + arm64/darwin/py3.14, which is what the calibrate parity pin records. +5. **The engine-gated identity tests were exercised with the engines present** + in the borrowed environment. A lane without the US extra will skip + `test_spec_engine_loader.py`'s golden and the country-bundle proofs rather + than check them. diff --git a/experiments/native-all-person-hours-20260919.json b/experiments/native-all-person-hours-20260919.json new file mode 100644 index 000000000..d74218984 --- /dev/null +++ b/experiments/native-all-person-hours-20260919.json @@ -0,0 +1,53 @@ +{ + "actual_asec_observation_audit": { + "counts": { + "age/age15": 2174, + "age/age16plus": 112272, + "age/under15": 27679, + "result/age15/asec_positive_source_hours": 243, + "result/age15/asec_source_nonwork_completion": 1931, + "result/age16plus/asec_positive_source_hours": 72217, + "result/age16plus/asec_source_nonwork_completion": 40055, + "result/under15/asec_source_universe_unavailable": 27679, + "rows": 142125, + "topcoded_hours/age16plus": 77, + "unresolved/under15": 27679 + }, + "member": "pppub25.csv", + "member_sha256": "06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5", + "receipt_sha256": "03b924aadbf49d5b66dd55fbdd9ffb8ae7a2ee449ce4531a1210f4ff4edb3fbc", + "refusals": 0, + "rows": 142125, + "scope": "Unweighted full-member observation diagnostic; original9bcd recoder, not source-owner authority or new child-completion execution" + }, + "base_commit": "9bcd039631ae9865f8caeaa8fe87b9b528265106", + "date": "2026-09-19", + "earlier_failed_attempt": "Attempt1 refused incomplete invented earnings columns; fixture repaired before fresh attempt2. Production field checks unchanged.", + "invented_tests": { + "cpu_seconds": 158.045512, + "failed": 0, + "passed": 144, + "peak_rss_bytes": 447184896, + "result_sha256": "7e62b4b78a0f1bb55aa6cc3820d9689ac029f78aeca67764a8f1b62f309ada24", + "skipped": 0, + "source_unchanged": true, + "wall_seconds": 226.9725150829181, + "xml_sha256": "1ba5fe413bd251639ea169175ba6b90f3436d2e2a734b57d310db20a572652bf" + }, + "remaining": [ + "Actual complete source-owner execution", + "Graph leaf and exact clone attachment", + "Engine input and release no-worse checks" + ], + "review": { + "independent_new_peer": "Unavailable after quota exhaustion; not claimed", + "root": "Complete source diff, source diagnostic, XML and current exact-byte checks reviewed" + }, + "scope": "Original survey person hours proposals; no clone/graph/engine/release qualification", + "source_and_test_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_usual_hours.py": "77878c90f89384077dd5d3f3397144b4fdcaaf2a6117d899295dc959fb151d55", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_hours_source.py": "86ab8112c7a34c4c3059352480949ec22e037698ba46bd98aa7a6d0b4c2c3b5c", + "packages/microcosm-build/tests/test_us_current_asec_usual_hours.py": "875a51b4afe5226d2a271f2b5d81da9c34b6f7808a8872edec3c90088432ca18", + "packages/microcosm-build/tests/test_us_current_survey_hours_source.py": "8e3d7383863c457bc7d3643fc9f05253cbd093f4014c3107aa64579e411f1d3f" + } +} diff --git a/experiments/native-byte-transports/RUN-RECORD-fifteenth.txt b/experiments/native-byte-transports/RUN-RECORD-fifteenth.txt new file mode 100644 index 000000000..26422bdad --- /dev/null +++ b/experiments/native-byte-transports/RUN-RECORD-fifteenth.txt @@ -0,0 +1,62 @@ +native-byte-transports lane -- 1/15 run record +launched: 2026-09-18T22:24:11Z (a first launch at 22:23:15Z died at parse time: + the launch script's venv-argument line carried an apostrophe inside a + ${VAR:?message} expansion, fixed in 703c9d5fe before this launch) +harness pid: 90568 / pgid 90568 / session leader (the waiter shell exec'd into + setsid_exec.py, which exec'd into the harness, so the pid the shell + printed is the harness's own) +kill, if ever needed: kill 90568 -- exact pid only, never a pattern kill +gate: available memory > 45 GB, open at launch (72.4 GB); no after-run to wait for +source tree: /Users/maxghenis/PolicyEngine/_worktrees/microcosm-byte-transports-fifteenth + detached at 9af56aa8a7a210e18674d484b11d36f417de6759 (git status clean; + packages/ byte-identical to the lane worktree at that head, verified + by diff -rq; the two later lane commits touch experiments/ only) +venv: that tree's own .venv, built from the merged lock with + `uv sync --all-packages --locked --extra us` (policyengine-us 2.2.1) +sources: /Users/maxghenis/PolicyEngine/_worktrees/native-scale-runs/run-fifteenth, + the retention lane's staged 1/15 clone (selection-request.json carries + fraction [1,15]); read only -- the harness writes only under + .measure/fifteenth (TMPDIR included) +harness: .measure/harness19_fifteenth.py, byte-identical to the committed + experiments/native-byte-transports/fifteenth/harness19_fifteenth.py, + which is the retention lane's committed harness19_fifteenth.py with + three named edits: the venv is an argument (this tree's lock moved), + and the two receipt fields that lane found stale (sample.fraction and + the scope sentence) say 1/15. Nothing measured changes. +ceilings, as set and never lowered: RLIMIT_CPU hard 21,660 s (setsid_exec), + harness soft 21,600 CPU-s, 43,200 wall-s, 48 GiB RSS +label: native-byte-transports-1-15 +disk at launch: 28 GiB free on the data volume. Estimated need: ~3.7 GB of + fixed ACS captures plus a graph store that scales from the 1/1000 + run's 127 MB (~8-9 GB at 1/15) plus temporaries -- about 12-13 GB. + Recorded as a risk, not a gate. +concurrency: this lane's 57-file dependent pytest battery (4 processes) runs on + other cores during the run; the ceilings are byte and row comparisons + and cannot turn on load, the CPU and wall figures can, and the report + does not lead with them. + +--- prediction on record BEFORE the outcome --- +At 1/15 (105,825 selected households; about 102,151 ACS households, 228,192 +ACS persons, 237,667 stacked persons), no byte transport on the nineteen-node +path refuses at this head: + selected SERIALNO list ~1.63 MB under coverage.MAX_ROSTER_BYTES (4 GiB); + refused at 1 MiB on the base branch + coverage body charge ~25.8 MB (113 B/person bound) under 4 GiB; + the base charge was 6*len(raw)+1024 + against 64 MiB, refusing above 12,911 + coverage body ~20.6 MB one 64 MiB accumulation + raw_person_keys digest ~4.6 MB streamed + issuance receipt ~1.6 MB under 4 GiB; refused at 2 MiB before + preparation receipt ~83 MB two 64 MiB segments, under 4 GiB + (PREPARATION_ROSTER_BYTES on the + consumer side, 64 MiB before) + allocation payload ~33 MB one segment + predictor projection origins ~5.7 MB one segment +and every row-count ceiling the row lane lifted sits above these counts. So the +run is expected to pass the point where the retention lane's 1/15 refused +(605 CPU-s, CANONICAL_SIZE in issue_acs_native_coverage), write +snapshots/preparation/header.json, and execute all nineteen nodes and the +replay within the 21,600 CPU-s ceiling. If it refuses, the harness's receipt +names PREPARATION_ISSUANCE_REFUSED at most, because the catch-all chain is only +visible in a traceback; the diagnostic harness (sys.monitoring RAISE observer) +recovers the cause without changing a byte of the measured tree. diff --git a/experiments/native-byte-transports/agreement-sweep-receipt.json b/experiments/native-byte-transports/agreement-sweep-receipt.json new file mode 100644 index 000000000..8264b4cf0 --- /dev/null +++ b/experiments/native-byte-transports/agreement-sweep-receipt.json @@ -0,0 +1,9 @@ +{ + "scope": "Seeded agreement sweep over the segmented, digested and matched byte transports this lane introduced, against their whole-bytes oracles. Not a build, not a certification, not release eligible.", + "release_eligible": false, + "seed": 20260918, + "documents": 2000, + "comparisons": 34000, + "refusals": 12000, + "disagreements": [] +} diff --git a/experiments/native-byte-transports/agreement_sweep.py b/experiments/native-byte-transports/agreement_sweep.py new file mode 100644 index 000000000..00355e18c --- /dev/null +++ b/experiments/native-byte-transports/agreement_sweep.py @@ -0,0 +1,284 @@ +"""Seeded agreement sweep over every byte transport this lane changed the shape of. + +Max's standard from the retention lane (its report §11, question 3): wherever a +predicate becomes a fold, a seeded sweep runs beside the hand-built battery. +Five predicates became folds here, one per encoder, and each is swept against +the whole-bytes oracle it replaces: + +* ``graph_survey_population._segmented_json`` against ``_bounded_json`` at + random segment sizes down to the longest token; ``_json_sha256`` against + ``_sha(_bounded_json)``; ``_json_matches`` against ``==`` on the whole + bytes, on every mutation (a flipped byte, a truncation, an extension, an + empty payload) and on the unmutated bytes. +* ``acs_person_coverage_authentication._json_roster`` against ``_json`` at + random segment sizes; ``_json_sha256`` against ``_sha(_json)``. +* ``graph_child_property_income._json`` against ``adapter._json`` + (``json.dumps`` with the adapter's settings) at random accumulation sizes. +* ``survey_origin_budget``'s streamed ``household_ids``/``group_indices`` + lists: ``append_list`` is exercised through the module by encoding a list + element by element and comparing with ``_json(list)``; the sweep reproduces + the loop's bytes here because the loop is a closure inside ``_document``. + +Every document is drawn from a seeded generator over nested dicts and lists of +ints, floats (finite, including -0.0 and subnormals), ASCII and non-ASCII +strings, None and booleans, at random depth and width. Refusals are also +swept: below the longest token every segmented encoder must refuse with its +per-accumulation code, and one byte over the total every one must refuse with +its total code. + +Not a build, not a certification, not release eligible. + + python agreement_sweep.py [pairs] +""" + +from __future__ import annotations + +import hashlib +import json +import pathlib +import random +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[2] +sys.path[:0] = [str(path) for path in sorted((ROOT / "packages").glob("*/src"))] + +from microcosm.build.us_runtime import acs_person_coverage_authentication as auth +from microcosm.build.us_runtime import graph_child_property_income as child +from microcosm.build.us_runtime import graph_survey_population as graph +from microcosm.build.us_runtime import survey_origin_budget as budget +from microcosm.fit import graph_joint_empirical as adapter + +for _module in (auth, child, graph, budget, adapter): + if not pathlib.Path(_module.__file__).resolve().is_relative_to(ROOT): + raise SystemExit(f"{_module.__name__} resolved outside {ROOT}") + +SEED = 20260918 +ALPHABET = "abcXYZ019 _-./\\\"'\n\té中\U0001f600" + + +def _scalar(rng, floats): + # The coverage encoder admits str, int, bool, None, list and dict only + # (acs_person_coverage_authentication._json_size refuses CANONICAL_TYPE on + # anything else), so floats are drawn only for the encoders that take them. + kind = rng.random() + if kind < 0.25: + return rng.randint(-(2**62), 2**62) + if kind < 0.45 and floats: + return rng.choice( + [ + 0.0, + -0.0, + 1.5, + 137.0 / 7, + 5e-324, + 1e300, + rng.random() * 1e6, + -rng.random(), + ] + ) + if kind < 0.75: + return "".join(rng.choice(ALPHABET) for _ in range(rng.randint(0, 24))) + if kind < 0.85: + return None + return rng.random() < 0.5 + + +def _document(rng, depth=0, floats=True): + if depth >= 3 or rng.random() < 0.3: + return _scalar(rng, floats) + if rng.random() < 0.5: + return [_document(rng, depth + 1, floats) for _ in range(rng.randint(0, 12))] + keys = [ + "".join(rng.choice(ALPHABET[:12]) for _ in range(rng.randint(1, 8))) + for _ in range(rng.randint(0, 8)) + ] + return {k: _document(rng, depth + 1, floats) for k in keys} + + +def _roster(rng, floats=True): + # A document with a dict at the top and a long list somewhere, like every + # whole-roster document on the path. + return { + "protocol": "invented/sweep", + "rows": [_document(rng, 1, floats) for _ in range(rng.randint(1, 60))], + "header": _document(rng, 1, floats), + } + + +def _mutations(rng, whole): + yield "truncated", whole[:-1] + yield "extended", whole + b"x" + yield "empty", b"" + if whole: + position = rng.randrange(len(whole)) + flipped = bytearray(whole) + flipped[position] ^= 1 << rng.randrange(8) + yield "flipped", bytes(flipped) + + +def sweep_graph(rng, receipt): + document = _roster(rng) + whole = graph._bounded_json(document, 64 * 1024**2) + pieces = list(graph._canonical_pieces(document, 64 * 1024**2)) + longest = max(map(len, pieces)) + for _ in range(3): + segment = rng.randint(longest, max(longest, len(whole)) + 8) + receipt["comparisons"] += 1 + if ( + graph._segmented_json(document, segment=segment, maximum=len(whole)) + != whole + ): + receipt["disagreements"].append( + ("graph._segmented_json", segment, whole[:80]) + ) + receipt["comparisons"] += 1 + if ( + graph._json_sha256(document, maximum=len(whole)) + != hashlib.sha256(whole).hexdigest() + ): + receipt["disagreements"].append(("graph._json_sha256", whole[:80])) + receipt["comparisons"] += 1 + if graph._json_matches(document, whole, maximum=len(whole)) is not True: + receipt["disagreements"].append(("graph._json_matches whole", whole[:80])) + for name, mutated in _mutations(rng, whole): + receipt["comparisons"] += 1 + if graph._json_matches(document, mutated, maximum=2**40) is not False: + receipt["disagreements"].append(("graph._json_matches " + name, whole[:80])) + if longest > 1 and longest < len(whole): + receipt["refusals"] += 1 + try: + graph._segmented_json(document, segment=longest - 1, maximum=len(whole)) + receipt["disagreements"].append( + ("graph accumulation refusal missing", whole[:80]) + ) + except graph.SurveyPopulationGraphError as error: + if str(error) != "TRANSPORT_LIMIT": + receipt["disagreements"].append(("graph accumulation code", str(error))) + receipt["refusals"] += 1 + try: + graph._segmented_json(document, segment=longest, maximum=len(whole) - 1) + receipt["disagreements"].append(("graph total refusal missing", whole[:80])) + except graph.SurveyPopulationGraphError as error: + if str(error) != "TRANSPORT_ROSTER_LIMIT": + receipt["disagreements"].append(("graph total code", str(error))) + + +def sweep_coverage(rng, receipt): + document = _roster(rng, floats=False) + whole = auth._json(document, auth.MAX_ROSTER_BYTES) + pieces = list(auth._json_chunks(document, auth.MAX_ROSTER_BYTES)) + longest = max(map(len, pieces)) + for _ in range(3): + segment = rng.randint(longest, max(longest, len(whole)) + 8) + receipt["comparisons"] += 1 + if auth._json_roster(document, segment=segment, maximum=len(whole)) != whole: + receipt["disagreements"].append(("auth._json_roster", segment, whole[:80])) + receipt["comparisons"] += 1 + if auth._json_sha256(document, len(whole)) != hashlib.sha256(whole).hexdigest(): + receipt["disagreements"].append(("auth._json_sha256", whole[:80])) + if longest > 1 and longest < len(whole): + receipt["refusals"] += 1 + try: + auth._json_roster(document, segment=longest - 1, maximum=len(whole)) + receipt["disagreements"].append( + ("auth accumulation refusal missing", whole[:80]) + ) + except auth.ACSCoverageAuthenticationError as error: + if str(error) != "CANONICAL_SIZE": + receipt["disagreements"].append(("auth accumulation code", str(error))) + receipt["refusals"] += 1 + try: + auth._json_roster(document, segment=longest, maximum=len(whole) - 1) + receipt["disagreements"].append(("auth total refusal missing", whole[:80])) + except auth.ACSCoverageAuthenticationError as error: + if str(error) != "CANONICAL_SIZE": + receipt["disagreements"].append(("auth total code", str(error))) + + +def sweep_child(rng, receipt, monkey): + document = _roster(rng) + whole = adapter._json(document) + encoder = json.JSONEncoder(sort_keys=True, separators=(",", ":"), allow_nan=False) + longest = max(len(piece.encode()) for piece in encoder.iterencode(document)) + for _ in range(3): + segment = rng.randint(longest, max(longest, len(whole)) + 8) + monkey(child, "MAX_ARTIFACT_BYTES", segment) + monkey(child, "MAX_ROSTER_BYTES", len(whole)) + receipt["comparisons"] += 1 + if child._json(document) != whole: + receipt["disagreements"].append(("child._json", segment, whole[:80])) + if longest > 1 and longest < len(whole): + monkey(child, "MAX_ARTIFACT_BYTES", longest - 1) + monkey(child, "MAX_ROSTER_BYTES", 2**40) + receipt["refusals"] += 1 + try: + child._json(document) + receipt["disagreements"].append( + ("child accumulation refusal missing", whole[:80]) + ) + except ValueError as error: + if "GRAPH_ARTIFACT_SIZE" not in str(error): + receipt["disagreements"].append(("child accumulation code", str(error))) + monkey(child, "MAX_ARTIFACT_BYTES", 64 * 1024**2) + monkey(child, "MAX_ROSTER_BYTES", len(whole) - 1) + receipt["refusals"] += 1 + try: + child._json(document) + receipt["disagreements"].append(("child total refusal missing", whole[:80])) + except ValueError as error: + if "GRAPH_ARTIFACT_LIMIT" not in str(error): + receipt["disagreements"].append(("child total code", str(error))) + + +def sweep_budget_lists(rng, receipt): + # The two header lists are streamed one element at a time inside + # _document's append_list; the bytes those appends produce are + # b"[" + b",".join(_json(v)) + b"]", which must equal _json(list). + values = [rng.randint(-(2**62), 2**62) for _ in range(rng.randint(0, 200))] + streamed = b"[" + b",".join(budget._json(v) for v in values) + b"]" + receipt["comparisons"] += 1 + if streamed != budget._json(values): + receipt["disagreements"].append(("budget append_list", values[:5])) + + +def main() -> int: + out = pathlib.Path(sys.argv[1]) + pairs = int(sys.argv[2]) if len(sys.argv) > 2 else 2000 + rng = random.Random(SEED) + receipt = { + "scope": "Seeded agreement sweep over the segmented, digested and matched byte transports this lane introduced, against their whole-bytes oracles. Not a build, not a certification, not release eligible.", + "release_eligible": False, + "seed": SEED, + "documents": pairs, + "comparisons": 0, + "refusals": 0, + "disagreements": [], + } + saved = {} + + def monkey(module, name, value): + saved.setdefault((module, name), getattr(module, name)) + setattr(module, name, value) + + try: + for _ in range(pairs): + sweep_graph(rng, receipt) + sweep_coverage(rng, receipt) + sweep_child(rng, receipt, monkey) + sweep_budget_lists(rng, receipt) + finally: + for (module, name), value in saved.items(): + setattr(module, name, value) + receipt["disagreements"] = [ + [str(part)[:200] for part in item] for item in receipt["disagreements"] + ] + out.write_text(json.dumps(receipt, indent=1) + "\n") + print( + f"documents={pairs} comparisons={receipt['comparisons']} " + f"refusals={receipt['refusals']} disagreements={len(receipt['disagreements'])}" + ) + return 1 if receipt["disagreements"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/native-byte-transports/byte-census-measurements.json b/experiments/native-byte-transports/byte-census-measurements.json new file mode 100644 index 000000000..51e313efd --- /dev/null +++ b/experiments/native-byte-transports/byte-census-measurements.json @@ -0,0 +1,170 @@ +{ + "scope": "Byte measurements for the byte-transport lane, through each module's own encoder, over the recovered 1/1000 artifact's catalogue counts and the full captured public ACS PUMS archive. Not a build, not a certification, not release eligible.", + "release_eligible": false, + "counts": { + "acs_households": 1531614, + "acs_persons": 3422888, + "asec_households": 55762, + "asec_persons": 142125, + "stacked_households": 1587376, + "stacked_persons": 3565013, + "clone_households": 3174752, + "clone_persons": 7130026 + }, + "origin_budget": { + "encoder": "survey_origin_budget._json (graph._bounded_json) per record", + "rows": { + "1587376": { + "record_bytes": 709, + "per_group_bytes": 742 + }, + "3174752": { + "record_bytes": 709, + "per_group_bytes": 742 + } + }, + "per_group_bytes": 742, + "full_source_bytes": 1177832992, + "tenth_bytes": 117782854, + "households_the_64MiB_cap_admits": 90443, + "household_ids_list_bytes": 25398017, + "group_indices_list_bytes": 25398017 + }, + "numeric_bounds": { + "encoder": "graph._bounded_json(document, graph_survey_calibration.MAX_BYTES)", + "bytes_at_groups": { + "2": 585, + "2002": 304525 + }, + "per_group_bytes": 151.97, + "per_clone_row_bytes": 75.98, + "full_source_bytes": 241233530, + "tenth_bytes": 24123261, + "clone_rows_the_64MiB_cap_admits": 883185, + "max_rows": 524288, + "clone_rows_full_source": 3174752 + }, + "age_artifact": { + "encoder": "MAGIC + header + values.tobytes(order='C'), values encode_recipient_matrix, FEATURES=5", + "bytes_per_tax_unit": 48, + "tax_units_per_household_at_1_1000": 1.343434, + "clone_one_tax_units_full_source_SCALED_FROM_1_1000": 2132535, + "full_source_bytes_SCALED": 102361680, + "tenth_bytes_SCALED": 10236144, + "tax_units_the_64MiB_cap_admits": 1398101, + "note": "tax units per household is a roster ratio from the 1/1000 artifact, not a catalogue count" + }, + "child_draw": { + "encoder": "graph_child_property_income._json(adapter._draw_document(...)) [producer]; adapter._document(payload, MAX_DRAW_BYTES) [consumer]", + "per_eligible_child_bytes": 273.0, + "eligible_children_full_source_ESTIMATE": null, + "children_the_64MiB_cap_admits": 245820, + "max_recipients_fit": 1048576, + "max_draw_bytes_fit": 67108864, + "max_artifact_bytes_graph": 67108864, + "note": "eligible-child count is not in the 1/1000 artifact; a census agent estimates it from ACS/ASEC age shares" + }, + "selection_block": { + "encoder": "survey_origin_budget._json(view.receipt['selection']) for selection_sha256 (graph._bounded_json, 64 MiB)", + "per_selected_row_bytes_at_1_1000": 243.13, + "full_source_bytes_SCALED": 385942815, + "tenth_bytes_SCALED": 38594135, + "rows_the_64MiB_cap_admits": 276017, + "note": "row widths at 1/1000 (inclusion_probability pairs, ids) differ slightly from full-source widths; scaled, not encoded at full-source widths" + }, + "acs_household_archive": { + "records": 1631969, + "occupied_records": 1531614, + "raw_bytes": 977481380, + "old_charge_bytes_all": 7536024536, + "old_charge_bytes_occupied": 7095161910, + "np_total": 3422888, + "serialno_widths": { + "13": 1631969 + }, + "members": { + "psam_husa.csv": 827133, + "psam_husb.csv": 804836 + }, + "serialno_list_bytes_full_source": 24505825, + "seconds": 13.9, + "derived": { + "old_charge_occupied_over_64MiB": 105.72615131735802, + "np_total_equals_person_records": true, + "serialno_list_over_1MiB": 23.37057590484619, + "serialno_list_over_2MiB": 11.685287952423096, + "serialnos_the_1MiB_cap_admits": 65535 + } + }, + "acs_person_archive": { + "records": 3422888, + "raw_bytes": 2405464349, + "raw_max": 877, + "old_charge_bytes": 17937823406, + "body_bytes": 308941698, + "body_row_max": 102, + "cells_json_bytes": 198264171, + "cells_json_max": 59, + "key_pair_bytes": 68461899, + "sporder_two_digit_rows": 4139, + "states": { + "observed_code": 5704812, + "outside_age_universe": 1140964 + }, + "members": { + "psam_pusa.csv": 1743751, + "psam_pusb.csv": 1679137 + }, + "key_list_bytes": 68461900, + "seconds": 182.7, + "derived": { + "old_charge_admits_rows_at_64MiB": 12805, + "old_charge_full_source_over_64MiB": 267.2943980395794, + "body_full_source_over_64MiB": 4.603590041399002, + "body_admits_rows_at_64MiB": 743525, + "key_list_full_source_over_64MiB": 1.0201618075370789, + "body_bytes_per_row_mean": 90.25761228529826, + "old_charge_over_body": 58.06216358013285 + } + } +} diff --git a/experiments/native-byte-transports/byte_census_measure.py b/experiments/native-byte-transports/byte_census_measure.py new file mode 100644 index 000000000..8349edafe --- /dev/null +++ b/experiments/native-byte-transports/byte_census_measure.py @@ -0,0 +1,543 @@ +"""Measure every byte transport this lane censuses, through each module's own encoder. + +The row-count lane established the full-source counts from the recovered +1/1000 artifact's catalogues (1,531,614 selectable ACS households, 3,422,888 +ACS persons, 55,762 ASEC households, 142,125 ASEC persons, 1,587,376 stacked +households, 3,565,013 stacked persons, 3,174,752 clone households). This +measures the *bytes* each transport meets at those counts: + +* the ACS archive is streamed in full -- every person record and every + household record of the pilot's captured public ACS PUMS archive -- so the + coverage body, the pre-allocation charge, the selected-key list and the + serialno list are sums over the real rows, not samples scaled up; +* each JSON roster document is encoded through its own module's encoder on + rows shaped at full-source id widths, and the marginal cost per row is a + difference between two row counts so fixed preambles cancel; +* each binary artifact is computed from the module's own width constants. + +Where a count is a roster ratio scaled from 1/1000 rather than a catalogue +count (tax units per household, eligible children), the receipt says so. + +Nothing outside the output path is written. Not a build, not a certification, +not release eligible. + + python byte_census_measure.py +""" + +from __future__ import annotations + +import csv +import io +import json +import pathlib +import sys +import time +import zipfile +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +ROOT = pathlib.Path(__file__).resolve().parents[2] +sys.path[:0] = [str(path) for path in sorted((ROOT / "packages").glob("*/src"))] + +from microcosm.build.us_runtime import acs_person_coverage_authentication as auth +from microcosm.build.us_runtime import acs_person_coverage_columns as literal +from microcosm.build.us_runtime import current_survey_household_roles as roles +from microcosm.build.us_runtime import current_survey_predictors as predictors +from microcosm.build.us_runtime import graph_child_property_income as child_graph +from microcosm.build.us_runtime import graph_combined_clone as clone +from microcosm.build.us_runtime import ( + graph_current_survey_household_roles as roles_graph, +) +from microcosm.build.us_runtime import graph_survey_age_artifact as ages +from microcosm.build.us_runtime import graph_survey_calibration as numeric +from microcosm.build.us_runtime import graph_survey_population as graph +from microcosm.build.us_runtime import puf_detail_transfer as detail +from microcosm.build.us_runtime import survey_origin_budget as budget_owner +from microcosm.fit import graph_joint_empirical as adapter + +for _module in (auth, literal, graph, budget_owner, numeric, ages, roles, adapter): + if not pathlib.Path(_module.__file__).resolve().is_relative_to(ROOT): + raise SystemExit(f"{_module.__name__} resolved outside {ROOT}") + +# Measured full-source counts; see experiments/native-row-ceilings/roster-census.json. +ACS_HOUSEHOLDS = 1_531_614 +ACS_PERSONS = 3_422_888 +ASEC_HOUSEHOLDS = 55_762 +ASEC_PERSONS = 142_125 +STACKED_HOUSEHOLDS = 1_587_376 +STACKED_PERSONS = 3_565_013 +CLONE_HOUSEHOLDS = 3_174_752 +CLONE_PERSONS = 7_130_026 +MIB = 1024**2 + + +def _members(archive, prefix): + with zipfile.ZipFile(archive) as zf: + for info in sorted(zf.infolist(), key=lambda m: m.filename): + name = info.filename.casefold() + if name.startswith(prefix) and name.endswith(".csv"): + yield zf, info + + +def _rows(stream, first): + """Raw record bytes and decoded cells, the way the owner decodes them.""" + header = None + for raw in stream: + text = raw.decode("utf-8-sig" if header is None and first else "utf-8") + values = next(csv.reader(io.StringIO(text, newline=""))) + if header is None: + header = values + yield raw, header, None + continue + yield raw, header, values + + +def acs_persons(archive, contract): + fields = contract["fields"] + out = { + "records": 0, + "raw_bytes": 0, + "raw_max": 0, + "old_charge_bytes": 0, # 6 * len(raw) + 1024, the shipped pre-charge + "body_bytes": 0, # exact NDJSON body: _json(row) + newline + "body_row_max": 0, + "cells_json_bytes": 0, # _json([*READ_COLUMNS cells, member, ordinal]) + "cells_json_max": 0, + "key_pair_bytes": 0, # _json([SERIALNO, SPORDER]) per person, plus commas + "sporder_two_digit_rows": 0, + "states": {}, + "members": {}, + } + started = time.monotonic() + for zf, info in _members(archive, "psam_pus"): + positions = None + ordinal = 0 + with zf.open(info) as stream: + for raw, header, values in _rows(stream, True): + if values is None: + positions = [header.index(c) for c in literal.READ_COLUMNS] + continue + ordinal += 1 + cells = [values[p] for p in positions] + serial, sporder, agep, mil, esr = cells + out["records"] += 1 + out["raw_bytes"] += len(raw) + out["raw_max"] = max(out["raw_max"], len(raw)) + out["old_charge_bytes"] += 6 * len(raw) + 1024 + states = [ + literal._field_state( + agep, + value, + minimum_age=fields[name]["minimum_age"], + codes=fields[name]["codes"], + ) + for name, value in (("MIL", mil), ("ESR", esr)) + ] + for state in states: + out["states"][state] = out["states"].get(state, 0) + 1 + row = auth._json( + [ + serial, + int(sporder), + agep, + mil, + esr, + *states, + info.filename, + ordinal, + ], + auth.MAX_RECORD_BYTES - 1, + ) + out["body_bytes"] += len(row) + 1 + out["body_row_max"] = max(out["body_row_max"], len(row) + 1) + cells_json = len( + auth._json( + [*cells, info.filename, ordinal], auth.MAX_RECORD_BYTES - 1 + ) + ) + out["cells_json_bytes"] += cells_json + out["cells_json_max"] = max(out["cells_json_max"], cells_json) + out["key_pair_bytes"] += len(auth._json([serial, int(sporder)])) + 1 + if int(sporder) >= 10: + out["sporder_two_digit_rows"] += 1 + out["members"][info.filename] = ordinal + out["key_list_bytes"] = out["key_pair_bytes"] - 1 + 2 # no trailing comma; brackets + out["seconds"] = round(time.monotonic() - started, 1) + return out + + +def acs_households(archive): + out = { + "records": 0, + "occupied_records": 0, + "raw_bytes": 0, + "old_charge_bytes_all": 0, + "old_charge_bytes_occupied": 0, + "np_total": 0, + "serialno_widths": {}, + "members": {}, + } + started = time.monotonic() + for zf, info in _members(archive, "psam_hus"): + positions = None + ordinal = 0 + with zf.open(info) as stream: + for raw, header, values in _rows(stream, True): + if values is None: + positions = [header.index(c) for c in ("SERIALNO", "NP")] + continue + ordinal += 1 + serial, np_ = (values[p] for p in positions) + out["records"] += 1 + out["raw_bytes"] += len(raw) + charge = 6 * len(raw) + 1024 + out["old_charge_bytes_all"] += charge + width = str(len(serial)) + out["serialno_widths"][width] = out["serialno_widths"].get(width, 0) + 1 + n = int(np_) + if n > 0: + out["occupied_records"] += 1 + out["old_charge_bytes_occupied"] += charge + out["np_total"] += n + out["members"][info.filename] = ordinal + # 16 canonical bytes per 13-character key: quotes, 13 chars, separator. + out["serialno_list_bytes_full_source"] = 16 * out["occupied_records"] + 1 + out["seconds"] = round(time.monotonic() - started, 1) + return out + + +def origin_budget(): + from fractions import Fraction + + design, probability, share = Fraction(137, 1), Fraction(1, 1), Fraction(1, 2) + actual = float(design * share / probability) + reference = budget_owner._reference(design, probability, share, actual) + rows = {} + for magnitude in (STACKED_HOUSEHOLDS, CLONE_HOUSEHOLDS): + record = { + "source": "acs", + "source_year": 2024, + "survey_year": 2024, + "raw_native_id": f"2024HU{magnitude % 10_000_000:07d}", + "selected_receiving_household_id": magnitude, + "combined_household_id": magnitude, + "statistical_unit": "occupied_housing_unit", + "original_design_float64_bytes": "0" * 16, + **reference, + "members": [[magnitude, 0], [magnitude * 2, 1]], + "incoming_clone_float64_bytes": ["0" * 16, "0" * 16], + } + encoded = len(budget_owner._json(record)) + ids = len(str(magnitude * 2)) * 2 + 2 + groups = len(str(magnitude)) * 2 + 2 + rows[str(magnitude)] = { + "record_bytes": encoded, + "per_group_bytes": encoded + 1 + ids + groups, + } + per_group = rows[str(STACKED_HOUSEHOLDS)]["per_group_bytes"] + return { + "encoder": "survey_origin_budget._json (graph._bounded_json) per record", + "rows": rows, + "per_group_bytes": per_group, + "full_source_bytes": per_group * STACKED_HOUSEHOLDS, + "tenth_bytes": per_group * (STACKED_HOUSEHOLDS // 10), + "households_the_64MiB_cap_admits": budget_owner.MAX_PAYLOAD_BYTES // per_group, + # The two header lists alone, each one entry per clone household row. + "household_ids_list_bytes": (len(str(CLONE_HOUSEHOLDS)) + 1) * CLONE_HOUSEHOLDS + + 1, + "group_indices_list_bytes": (len(str(STACKED_HOUSEHOLDS)) + 1) + * CLONE_HOUSEHOLDS + + 1, + } + + +def numeric_bounds(): + def document(groups): + ids = list(range(CLONE_HOUSEHOLDS - 2 * groups, CLONE_HOUSEHOLDS)) + indices = [STACKED_HOUSEHOLDS - groups + i // 2 for i in range(2 * groups)] + upper = [float(137 * 2 * (1 + i / 7)).hex() for i in range(groups)] + row_upper = [float(137 * 8 * (1 + i / 7)).hex() for i in range(2 * groups)] + incoming = [float(137 * (1 + i / 7)).hex() for i in range(2 * groups)] + return { + "protocol": numeric.BOUNDS_PROTOCOL, + "budget_sha256": "a" * 64, + "population": clone.COMBINED_CLONE_NODE, + "household_ids": ids, + "group_indices": indices, + "group_upper_hex": upper, + "row_upper_hex": row_upper, + "incoming_hex": incoming, + } + + small, large = 2, 2_002 + bytes_small = len(graph._bounded_json(document(small), numeric.MAX_BYTES)) + bytes_large = len(graph._bounded_json(document(large), numeric.MAX_BYTES)) + per_group = (bytes_large - bytes_small) / (large - small) + return { + "encoder": "graph._bounded_json(document, graph_survey_calibration.MAX_BYTES)", + "bytes_at_groups": {str(small): bytes_small, str(large): bytes_large}, + "per_group_bytes": round(per_group, 2), + "per_clone_row_bytes": round(per_group / 2, 2), + "full_source_bytes": int(per_group * STACKED_HOUSEHOLDS), + "tenth_bytes": int(per_group * (STACKED_HOUSEHOLDS // 10)), + "clone_rows_the_64MiB_cap_admits": int(numeric.MAX_BYTES // (per_group / 2)), + "max_rows": numeric.MAX_ROWS, + "clone_rows_full_source": CLONE_HOUSEHOLDS, + } + + +def age_artifact(): + header = ages._json( + { + "protocol": "microcosm.us.survey-household-age-counts.v1", + "population": clone.COMBINED_CLONE_NODE, + "columns": list(ages._COLUMNS), + "rows": CLONE_HOUSEHOLDS, + "people": CLONE_PERSONS, + "age_convention": "integer_years_completed", + } + ) + row = ages._WIDTH * 8 + fixed = len(ages.MAGIC) + 4 + len(header) + reserved = len(ages.MAGIC) + 4 + ages.MAX_HEADER_BYTES + return { + "encoder": "MAGIC + header + values.tobytes(order='C'), values encode_recipient_matrix, FEATURES={len(detail.FEATURES)}", + "bytes_per_tax_unit": row, + "tax_units_per_household_at_1_1000": round(ratio, 6), + "clone_one_tax_units_full_source_SCALED_FROM_1_1000": clone_one, + "full_source_bytes_SCALED": row * clone_one, + "tenth_bytes_SCALED": row * (clone_one // 10), + "tax_units_the_64MiB_cap_admits": (64 * MIB) // row, + "note": "tax units per household is a roster ratio from the 1/1000 artifact, not a catalogue count", + } + + +def child_draw(): + key = ("acs", 2024, 2024, "2024HU0001234567", "12", "12") + n = 1_001 + draw = SimpleNamespace( + values=np.tile(np.array([[1234.5, 0.0]]), (n, 1)), + patterns=np.ones(n, dtype=np.int64), + donor_keys=[("asec", 2024, 2025, "1234567", "12", "1")] * n, + model_sha256="b" * 64, + ) + params = { + "support_sha256": "c" * 64, + "donor_source_sha256": "d" * 64, + "source_sha256": "e" * 64, + "scenario_sha256": "f" * 64, + "transport_json": '{"kind":"nearest"}', + "transport_sha256": "0" * 64, + "stream": ["child-property", 1], + "coordinate_suffix": ("child-property-v1",), + } + + def encoded(rows): + ids = np.arange(STACKED_PERSONS - rows, STACKED_PERSONS) + keys = [key] * rows + partial = SimpleNamespace( + values=draw.values[:rows], + patterns=draw.patterns[:rows], + donor_keys=draw.donor_keys[:rows], + model_sha256=draw.model_sha256, + ) + return len(adapter._json(adapter._draw_document(partial, ids, keys, params))) + + a, b = encoded(1), encoded(n) + per_row = (b - a) / (n - 1) + return { + "encoder": "graph_child_property_income._json(adapter._draw_document(...)) [producer]; adapter._document(payload, MAX_DRAW_BYTES) [consumer]", + "per_eligible_child_bytes": round(per_row, 2), + "eligible_children_full_source_ESTIMATE": None, + "children_the_64MiB_cap_admits": int(64 * MIB // per_row), + "max_recipients_fit": adapter.MAX_RECIPIENTS, + "max_draw_bytes_fit": adapter.MAX_DRAW_BYTES, + "max_artifact_bytes_graph": child_graph.MAX_ARTIFACT_BYTES, + "note": "eligible-child count is not in the 1/1000 artifact; a census agent estimates it from ACS/ASEC age shares", + } + + +def selection_block(preparation): + selected = preparation["selection"]["selected"] + + def enc(v): + return len( + json.dumps( + v, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode() + ) + + per_row = enc(selected) / len(selected) + return { + "encoder": "survey_origin_budget._json(view.receipt['selection']) for selection_sha256 (graph._bounded_json, 64 MiB)", + "per_selected_row_bytes_at_1_1000": round(per_row, 2), + "full_source_bytes_SCALED": int(per_row * STACKED_HOUSEHOLDS), + "tenth_bytes_SCALED": int(per_row * (STACKED_HOUSEHOLDS // 10)), + "rows_the_64MiB_cap_admits": int(64 * MIB // per_row), + "note": "row widths at 1/1000 (inclusion_probability pairs, ids) differ slightly from full-source widths; scaled, not encoded at full-source widths", + } + + +def main() -> int: + snapshot = pathlib.Path(sys.argv[1]) + preparation = json.loads(pathlib.Path(sys.argv[2]).read_bytes()) + out = pathlib.Path(sys.argv[3]) + contract = literal.coverage_field_contract() + record = { + "scope": ( + "Byte measurements for the byte-transport lane, through each module's own " + "encoder, over the recovered 1/1000 artifact's catalogue counts and the full " + "captured public ACS PUMS archive. Not a build, not a certification, not " + "release eligible." + ), + "release_eligible": False, + "counts": { + "acs_households": ACS_HOUSEHOLDS, + "acs_persons": ACS_PERSONS, + "asec_households": ASEC_HOUSEHOLDS, + "asec_persons": ASEC_PERSONS, + "stacked_households": STACKED_HOUSEHOLDS, + "stacked_persons": STACKED_PERSONS, + "clone_households": CLONE_HOUSEHOLDS, + "clone_persons": CLONE_PERSONS, + }, + "origin_budget": origin_budget(), + "numeric_bounds": numeric_bounds(), + "age_artifact": age_artifact(), + "roles_projection": roles_projection(), + "predictor_projection": predictor_projection(), + "diagnostic_matrix": diagnostic_matrix(preparation), + "child_draw": child_draw(), + "selection_block": selection_block(preparation), + } + out.write_text(json.dumps(record, indent=1) + "\n") + print("encoder measurements written; streaming the ACS archive ...", flush=True) + record["acs_household_archive"] = acs_households(snapshot / "csv_hus.zip") + out.write_text(json.dumps(record, indent=1) + "\n") + print( + "household archive done", + record["acs_household_archive"]["seconds"], + "s", + flush=True, + ) + record["acs_person_archive"] = acs_persons(snapshot / "csv_pus.zip", contract) + people = record["acs_person_archive"] + people["derived"] = { + "old_charge_admits_rows_at_64MiB": int( + auth.MAX_BODY_BYTES / (people["old_charge_bytes"] / people["records"]) + ), + "old_charge_full_source_over_64MiB": people["old_charge_bytes"] + / auth.MAX_BODY_BYTES, + "body_full_source_over_64MiB": people["body_bytes"] / auth.MAX_BODY_BYTES, + "body_admits_rows_at_64MiB": int( + auth.MAX_BODY_BYTES / (people["body_bytes"] / people["records"]) + ), + "key_list_full_source_over_64MiB": people["key_list_bytes"] + / auth.MAX_BODY_BYTES, + "body_bytes_per_row_mean": people["body_bytes"] / people["records"], + "old_charge_over_body": people["old_charge_bytes"] / people["body_bytes"], + } + hh = record["acs_household_archive"] + hh["derived"] = { + "old_charge_occupied_over_64MiB": hh["old_charge_bytes_occupied"] + / auth.MAX_BODY_BYTES, + "np_total_equals_person_records": hh["np_total"] == people["records"], + "serialno_list_over_1MiB": hh["serialno_list_bytes_full_source"] / MIB, + "serialno_list_over_2MiB": hh["serialno_list_bytes_full_source"] / (2 * MIB), + "serialnos_the_1MiB_cap_admits": (MIB - 1) // 16, + } + out.write_text(json.dumps(record, indent=1) + "\n") + print( + json.dumps({"people": people["derived"], "households": hh["derived"]}, indent=1) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/native-byte-transports/census-brief.md b/experiments/native-byte-transports/census-brief.md new file mode 100644 index 000000000..1723e4a6a --- /dev/null +++ b/experiments/native-byte-transports/census-brief.md @@ -0,0 +1,156 @@ +# Byte-transport census: brief for every census and verification agent + +Read this whole file before reading any module. Every line number you cite is +this tree's, at the head you read it. You are read-only: do not edit, format, +or create any file under `packages/`, `docs/` or `tools/`; write nothing +outside the scratchpad path you are given. Do not run pytest. Do not read +gated data; the only data inputs are the two files named in §4. + +## 1. What a row is + +One row per **byte bound**: a module-level constant or inline literal that +bounds a byte count (`MAX_*_BYTES`, `*_MAX_BYTES`, `_MAX_*`, a `_bounded_json` +/ `_encode` / `_json` limit argument, a `len(payload) <= N` check, a +`read(N + 1)` read cap, a struct width times a row ceiling, a JSON-token or +record cap). A bound that is defined in one module and enforced in another is +one row, keyed by its definition, listing every enforcement site. + +Each row must carry, from the code you read at this head: + +- `constant`: `module.py:line NAME` (or `module.py:line ` for a literal). +- `value`: the expression and its integer value. +- `enforced_at`: every site `module.py:line`, with the refusal code literal and + exception type raised there. +- `encodes`: what document the bound sits in front of. Say precisely: a + per-row roster (which roster: ACS households, ACS persons, ASEC households, + ASEC persons, stacked households, stacked persons, clone households, clone + persons, selected serialnos, eligible children, tax units, groups), a + fixed-shape receipt, a catalogue, an evidence list, a header, a numpy body, + an upstream file, a single scalar/token. +- `shape`: `roster` (grows with the selection fraction), `fixed` (does not), + `upstream` (an input file's own size, identical at every fraction), + `width` (a fixed-width field or encoder width). +- `site_kind`: `materialises` (the bytes exist whole before or after the check) + or `streams` (a digest or a token-at-a-time accumulation; no whole copy). +- `bytes_at_tenth`, `bytes_at_full_source`: for a `roster` row, the bytes the + document meets at 1/10 and at full source **through the module's own + encoder** at the counts in §3. Cite `byte-census-measurements.json` when it + already measured the document; otherwise measure it yourself with the venv + python (`.venv/bin/python`) on invented rows at full-source id widths, as a + difference between two row counts so fixed preambles cancel, and put the + script and its receipt in your scratchpad. For a `fixed`/`upstream`/`width` + row say why the bytes do not move with the fraction and give the one number + you have (the recovered 1/1000 artifact's, or the input file's). +- `binds_at_tenth`, `binds_at_full_source`: `true` only if the measured bytes + exceed the value **at the value the base branch shipped**; record the base + value and this head's value separately when they differ (`base_value`, + `head_value`), because several bounds were lifted on this branch already. +- `reachable_from`: `19` (the nineteen-node financial graph), `45` (the + forty-five-node pilot graph), `age` (`survey_age_calibration`), or `none`, + with the call chain in one line (runner → function → module). §2 gives the + rosters. A bound in a module the runners import but whose enforcing function + no runner reaches is `none`, and you say which function would reach it. +- `classification`, one of: + - `segment`: a whole-roster canonical stream that meets its bound; takes the + transport lane's segmented shape (bytes unchanged, accumulation ceiling + unchanged, explicit total). + - `consumer`: a consumer-side cap on bytes a segmented producer issues; takes + the producer's total. + - `resource`: a single body that exists whole before the bound (a numpy + buffer) and meets its bound; becomes a fixed multiple of full source. + - `stays`: a roster-shaped document that does not meet its bound at full + source (give the headroom multiple). + - `must-not-move`: protects an encoder width, an upstream file's real size, + a fixed-width field, or a token/scalar. + - `not-a-transport`: a row count wearing a byte name (say which row-count + rule applies), or a memory ceiling with no document behind it. +- `notes`: anything a reviewer needs; keep it to what the code says. + +## 2. Reachability, from the code + +**The nineteen-node financial graph** (`_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/graph.json`, run by `graph_atomic_survey_financial.run_atomic_survey_financial` with no property, tax, status or roles): + +| node | kernel | +|---|---| +| `survey_population.create`, `.allocate`, `.observed_geography` | `graph_survey_population`, `graph_atomic_survey_population`, `survey_population_preparation` (source admission), `acs_native_coverage_binding`, `acs_person_coverage_authentication`, `acs_housing_universe_source`, `acs_population_catalogue`, `asec_2024_native_population`, `asec_population_catalogue`, `survey_catalogue_selection`, `survey_population_domains`, `survey_observed_age`, `native_household_origin` | +| `combined_survey_puf_support_clone`, `.owned` | `graph_combined_clone` | +| `geography.support.0`, `.assign`, `.derive`, `.gate` | `survey_atomic_geography`, `graph_geography`, `microcosm.graph.codecs` (`raw-bytes-v1`) | +| `survey_predictors.source_projection`, `.asec_design_donor`, `.asec_current_columns`, `.fit.000-002`, `.apply.000-002`, `.attach` | `graph_current_survey_predictors`, `current_survey_predictors`, `asec_current_money*`, `_asec_current_money_codec`, `microcosm.fit.model_input`, `microcosm.fit.qrf`, `microcosm.fit.graph_qrf*` | + +After the graph returns, the runner replays and verifies: `survey_population_replay`, `graph_atomic_survey_financial` verification, `current_survey_predictors.verify_materialized_current_survey_predictors`. + +**The forty-five-node pilot graph** (`_recovered/pilot-runs/native45-v5/`, run through `graph_survey_completion_host` with `property_income`, `rebase_property_taxes=True`, `person_status=False`, `household_roles=False`): the nineteen above, plus the property-income nodes of `graph_current_survey_property.current_survey_property_nodes` (`graph_property_income`, `graph_property_income_receipts`, `current_property_income_sources`, `current_asec_property_basis`, `property_income_constants`), the three property-tax nodes of `graph_property_tax_leaves`, and the ten completion nodes: `survey_completion.receiving` (`graph_survey_completion`), the six `child_property.*` nodes (`graph_child_property_income`, `current_child_property_income_source`, `current_property_completion_routing`, `microcosm.fit.graph_joint_empirical`, `microcosm.fit.joint_empirical`), and the tax trio again over the completed population. Household roles (`current_survey_household_roles`, `graph_current_survey_household_roles`) and person status (`graph_current_survey_person_status`) are reachable only from the 49/51-node variants: record them, mark `reachable_from: none` with that note. + +**`survey_age_calibration`** is neither graph. It runs `survey_origin_budget.freeze_survey_origin_budget` and `.checked_view`, `graph_survey_budget`, `graph_survey_calibration`, `graph_survey_age_artifact`, `survey_financial_successor`, `demographic_calibration_graph`, `graph_national_age_counts`, `survey_calibration_diagnostics`. The brief that commissioned this census names the origin budget explicitly, so these are in scope under `reachable_from: age`. + +**Not on any of these paths** (record with `reachable_from: none` if you meet them; do not spend measurement effort): `graph_survey_puf55`, `puf55_survey_recipients`, `graph_current_survey_puf_transfer`, `puf_diagnostic_consumer.current_survey_recipient_matrix`, `current_survey_amounts`, `graph_us_survey_enrichment`, `graph_fiscal_*`, `graph_acs_housing_universe` (the housing *graph*; the housing *source* `acs_housing_universe_source` is on the path through `acs_native_coverage_binding`). + +## 3. The counts (measured, not extrapolated) + +From the recovered 1/1000 artifact's catalogues, which are counts of the whole +upstream files and so are full-source counts (`experiments/native-row-ceilings/roster-census.json`): + +| quantity | full source | at 1/10 | at 1/15 | +|---|---:|---:|---:| +| ACS households (selectable) | 1,531,614 | 153,161 | 102,107 | +| ACS persons | 3,422,888 | 342,288 | 228,192 | +| ASEC households | 55,762 | 5,576 | 3,717 | +| ASEC persons | 142,125 | 14,212 | 9,475 | +| stacked households (= selected = groups) | 1,587,376 | 158,737 | 105,825 | +| stacked persons | 3,565,013 | 356,501 | 237,667 | +| clone households | 3,174,752 | 317,475 | 211,650 | +| clone persons | 7,130,026 | 713,002 | 475,335 | + +Entities other than person and household (tax units, families, marital units, +SPM units) are roster ratios from the 1/1000 artifact (1,584 households carry +2,128 tax units, 1,594 families, 2,774 marital units, 1,585 SPM units); say +`SCALED` when you use one. + +## 4. Inputs you may read + +- `experiments/native-byte-transports/byte-census-measurements.json`: this + lane's measurements through each module's own encoder (origin budget, + numeric bounds, age artifact, roles projection, predictor projection, + diagnostic matrix, child draw, selection block, and the whole captured ACS + archive: coverage body, pre-allocation charge, key list, serialno list). +- `experiments/native-row-ceilings/census.json`: the row-count lane's census + (146 bounds). Its byte rows are a starting list to **verify at this head**, + not a source of truth: several of its values and verdicts predate lifts on + this branch, and it missed modules (the property family, the housing source's + prepared receipt, consumer-side re-encodings). +- `_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json` + (the 1/1000 preparation receipt, 1,136,063 bytes) for row shapes. +- `docs/us-native-scale-transport.md` §2 and `docs/us-native-row-ceilings.md` + for the two neighbouring arguments. + +## 5. What has already moved on this branch (record base and head values) + +- `acs_person_coverage_authentication`: `MAX_ROSTER_BYTES = 64 * MAX_BODY_BYTES` + added; the coverage body, the issuance receipt and the key-list digest are + segmented/streamed; `SELECTED_BODY_BUDGET` charges the NDJSON line bound. +- `acs_native_coverage_binding`: the serialno pre-check sizes without encoding + under `coverage.MAX_ROSTER_BYTES`; the issuance receipt is `_json_roster`. +- `graph_survey_population`: `_canonical_pieces`, `_segmented_json`, + `_json_sha256`, `_json_matches`; `PREPARATION_ROSTER_BYTES`. +- `survey_origin_budget`: `MAX_ROSTER_BYTES`; the budget document is + segmented; `selection_sha256` streamed; `checked_view` streams its + comparison. +- `graph_survey_budget`, `graph_survey_calibration` (`MAX_ROSTER_BYTES`, + `MAX_ROWS`), `graph_survey_age_artifact` (`MAX_BYTES` 2 GiB), + `current_survey_predictors` (segmented projection), + `graph_child_property_income` (segmented `_json`, `MAX_ROSTER_BYTES`, + `MAX_ORIGINALS`), `current_child_property_income_source` + (`MAX_RECIPIENT_ROWS`), `microcosm.fit.graph_joint_empirical` + (`MAX_DRAW_BYTES`). + +Use `git diff origin/native-row-ceilings -- ` to see exactly what moved +in a module; the base value is the `-` side. + +## 6. Verification agents + +You receive one row and must try to refute it by re-reading the code: the +value, every enforcement site, the encoded document's shape, whether the site +streams or materialises, the reachability chain, the byte figures (recompute +one), and the classification. Return `agrees: true/false`, the exact +correction if false, and the line-cited evidence either way. Default to +`agrees: false` when you cannot verify a claim from the code. diff --git a/experiments/native-byte-transports/census-raw.json b/experiments/native-byte-transports/census-raw.json new file mode 100644 index 000000000..1dd5fac1d --- /dev/null +++ b/experiments/native-byte-transports/census-raw.json @@ -0,0 +1,8047 @@ +{ + "scope": "Raw census rows from the nine family agents at head 475c9ba9d (verification pending). Not a build, not a certification, not release eligible.", + "release_eligible": false, + "families": [ + { + "family": "Predictors and the fit package", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py (full, 644 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py (full, 616)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_diagnostic_consumer.py (full, 763)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py (full, 847)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_detail_transfer.py (full, 452)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_detail_transfer.py (full, 780)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py (full, 245)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py (full, 343)", + "packages/microcosm-fit/src/microcosm/fit/model_input.py (full, 140)", + "packages/microcosm-fit/src/microcosm/fit/qrf.py (full, 1826)", + "packages/microcosm-fit/src/microcosm/fit/_graph_legacy_qrf.py (full, 197)", + "packages/microcosm-fit/src/microcosm/fit/graph_legacy_qrf.py (full, 186)", + "packages/microcosm-fit/src/microcosm/fit/graph_legacy_apply.py (full, 40)", + "packages/microcosm-fit/src/microcosm/fit/graph_legacy_apply_matrix.py (full, 107)", + "packages/microcosm-fit/src/microcosm/fit/graph_legacy_train.py (full, 147)", + "packages/microcosm-fit/src/microcosm/fit/_graph_legacy_apply.py (full, 139)", + "no packages/microcosm-fit/src/microcosm/fit/graph_qrf*.py exists at this head (the fit graph modules are the graph_legacy_* files above)", + "packages/microcosm-fit/src/microcosm/fit/qrf_target.py (lines 240-340: to_bytes / from_trusted_bytes; not in the family list, read because graph_legacy_train/apply call it)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py (lines 100-120, 255-380, 405-500: _require, _bounded_json, _canonical_pieces, _segmented_json, _json_sha256, _json_matches, _instruction_document, _allocation_payload)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py (lines 40-70: MAX_PAYLOAD_BYTES, MAX_SEGMENT_BYTES, MAX_ROSTER_BYTES; :78-85 error class)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py (lines 180-380: qualify_current_asec_demographics, the producer of the projections row 7 re-encodes)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py (lines 36-120: health_coverage_seal consumer re-encode)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py (lines 95-130, 840-910, 1280-1470, 1830-1990: reachability)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py (grep: _check_asec_projection :208-215)", + "packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py (lines 230-245: MONEY_FIELDS roster)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_domains.py (lines 27, 52-56: Source, HouseholdKey)", + "packages/microcosm-graph/src/microcosm/graph/store.py (lines 1050-1095: put_bytes/load_bytes have no size cap)", + "experiments/native-byte-transports/census-brief.md (full)", + "experiments/native-byte-transports/byte_census_measure.py (full)", + "experiments/native-byte-transports/byte-census-measurements.json (full)", + "experiments/native-row-ceilings/census.json (this family's rows and the shared-constant rows)", + "experiments/native-scale-transport/harness19_base.py (lines 25-60, 525-560: pilot arguments)", + "docs/us-native-scale-transport.md (section 2d)", + "git diff origin/native-row-ceilings -- current_survey_predictors.py, graph_joint_empirical.py" + ], + "rows": [ + { + "constant": "survey_population_preparation.py:50 MAX_PAYLOAD_BYTES (segment) and :58 MAX_ROSTER_BYTES (maximum), as passed at current_survey_predictors.py:522-523 for the predictor projection", + "value": "segment = 64 * 1024**2 = 67,108,864; maximum = 64 * MAX_SEGMENT_BYTES (:57, = MAX_PAYLOAD_BYTES) = 4,294,967,296", + "enforced_at": [ + "current_survey_predictors.py:512-524 (host.survey_graph._segmented_json(document, segment=source.MAX_PAYLOAD_BYTES, maximum=source.MAX_ROSTER_BYTES))", + "graph_survey_population.py:294 (segment <= 64 MiB) \"TRANSPORT_LIMIT\" SurveyPopulationGraphError(ValueError) (:116, raised by _require :120)", + "graph_survey_population.py:303 and :305 (one token > segment) \"TRANSPORT_LIMIT\" SurveyPopulationGraphError", + "graph_survey_population.py:325 (maximum type/positive) \"TRANSPORT_ROSTER_LIMIT\" SurveyPopulationGraphError", + "graph_survey_population.py:331 (segment accumulation) \"TRANSPORT_LIMIT\" SurveyPopulationGraphError", + "graph_survey_population.py:332 and :337 (running total and joined payload <= maximum) \"TRANSPORT_ROSTER_LIMIT\" SurveyPopulationGraphError", + "graph_survey_population.py:288 / :310 (encoder failure) \"TRANSPORT_ENCODING\" SurveyPopulationGraphError" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT", + "TRANSPORT_ROSTER_LIMIT", + "TRANSPORT_ENCODING" + ], + "encodes": "The predictor source projection: the fixed evidence dict (protocol, digests, money header, split contract, model judgments, optional atomic_geography receipt) plus \"origins\": one [person_id, native_person_id, source] row per stacked person (current_survey_predictors.py:515-517), plus money_fields[*].status_hex/validity_hex/zero_origin_hex, three hex strings of one byte per selected ASEC person for each of the five MONEY_FIELDS (:400-406).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "8,996,644 roster part (356,501 stacked-person origin rows x 24.04 B = 8,570,284; 14,212 ASEC persons x 30 B hex = 426,360) plus the unmeasured fixed evidence preamble", + "bytes_at_full_source": "89,966,662 roster part (3,565,013 x 24.04 B = 85,702,912 origins; 142,125 x 30 B = 4,263,750 hex evidence) plus the fixed preamble", + "measurement_basis": "Measured at this head through graph_survey_population._segmented_json with the module's exact segment/maximum (scratchpad measure_pred.py -> measurements.json: per_stacked_person_origin_row_bytes 24.04, per_asec_person_hex_evidence_bytes 30.0, difference of 1001 vs 1 rows, ids at 7-digit full-source width). The 85,702,912 origins figure matches byte-census-measurements.json predictor_projection.full_source_origins_bytes, which was taken through _bounded_json at the base; segmented bytes equal bounded bytes (measured: segmented_equals_bounded_bytes true). The JSON omitted the per-ASEC-person hex evidence, added here.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "_bounded_json(document, source.MAX_PAYLOAD_BYTES): one accumulation of 67,108,864 (git diff origin/native-row-ceilings: '- projection = host.survey_graph._bounded_json(' / '- source.MAX_PAYLOAD_BYTES,')", + "head_value": "_segmented_json(document, segment=67,108,864, maximum=4,294,967,296) (current_survey_predictors.py:512-524)", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1389) -> current_survey_predictors.qualify_current_survey_predictors -> host.survey_graph._segmented_json (:512); re-derived by every predictor kernel through _Kernel._qualified (graph_current_survey_predictors.py:301, nodes survey_predictors.source_projection/.asec_design_donor/.asec_current_columns/.attach) and by verify_materialized_current_survey_predictors (:578; runner :1841 and :854). The 45-node run reaches it through the same base call (graph_atomic_survey_financial.py:1319-1331). Consumers hold the whole payload: graph store put_bytes (store.py:1057-1074, no cap), projection.payload == result.projection (graph_current_survey_predictors.py:359), projection == qualified.projection (:586), codec.sha(qualified.projection) (:138).", + "classification": "segment", + "notes": "Already took the segmented shape on this branch (brief section 5). At the base value 89,966,662 > 67,108,864 so a full-source run refused with TRANSPORT_LIMIT; a single 64 MiB accumulation admits 2,791,550 stacked persons, i.e. about 78% of the source. At the head the total is 2.1% of the 4 GiB ceiling. The producer's comment at :508-511 states the 85,702,912 figure. Off-path sibling: current_survey_amounts.py:369 still passes predictors.source.MAX_PAYLOAD_BYTES to _bounded_json for its own (digest-only, fixed-shape) projection." + }, + { + "constant": "model_input.py:18 MATRIX_HEADER_MAX_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "model_input.py:86-87 (encode: len(header) > MATRIX_HEADER_MAX_BYTES) ValueError(\"Recipient matrix header exceeds its size bound.\")", + "model_input.py:105-106 (decode: length == 0 or length > MATRIX_HEADER_MAX_BYTES or len(payload) < start + length) ValueError(\"Recipient matrix header is truncated or exceeds its bound.\")" + ], + "refusal_codes": [ + "Recipient matrix header exceeds its size bound.", + "Recipient matrix header is truncated or exceeds its bound." + ], + "encodes": "The JSON header of the recipient-matrix envelope (schema_version, entity, rows, columns, index_name, index_identity = a fixed five-field digest of the id index), behind a 4-byte big-endian length field (:90, :103). The body that follows (entity_ids int64 + float64 features, 8 x (1 + n_features) bytes per row) has no cap.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "header 462 B including the 40-byte magic and 4-byte length (measured at 1 row; grows only by the digits of \"rows\"); the uncapped body is 16,432,339 B for 342,288 ACS persons x 48 B (five features under demographic_conditioning=True)", + "bytes_at_full_source": "header ~462 B; uncapped body 164,319,623 B (3,422,888 ACS persons x 48.006 B measured; byte-census-measurements.json predictor_projection.matrix.full_source_bytes = 164,298,624 body only)", + "measurement_basis": "Measured through model_input.encode_recipient_matrix at this head (measure_pred.py recipient_matrix: bytes_per_acs_person 48.006, header_plus_magic_bytes_at_1_row 462); cited byte-census-measurements.json for the body.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "19", + "reach_chain": "qualify_current_survey_predictors -> model_input.encode_recipient_matrix (current_survey_predictors.py:505); decode at complete_predictor_columns (:590), read_current_survey_draws (graph_current_survey_predictors.py:469), LegacyQRFApplyMatrixKernel.run (graph_legacy_apply_matrix.py:76, nodes survey_predictors.apply.000-002), runner graph_atomic_survey_financial.py:1263. The 45-node run also reaches the codec via current_property_income_sources.py:213/239.", + "classification": "must-not-move", + "notes": "Protects a fixed-shape header behind a fixed-width length field; the header is row-count independent apart from the digits of \"rows\". The matrix BODY has no byte bound anywhere on the path: the encoder caps only the header, store.put_bytes stores whole with no size check, and every _Kernel._qualified plus the verifier re-encodes it (164 MB at full source). Off path, puf_diagnostic_consumer.py:398-406 encodes a 207,692-row PUF donor matrix (~10 MB) through the same codec and puf_detail_transfer.py:452 the diagnostic matrix (row 8 arm b)." + }, + { + "constant": "model_input.py:127 len(body) != rows * 8 + rows * len(columns) * 8", + "value": "exact body width: 8 x (1 + n_columns) bytes per row (48 with the five demographic-conditioning features, 32 with three); an equality, not a ceiling", + "enforced_at": [ + "model_input.py:127-128 ValueError(\"Recipient matrix body length differs from its dimensions.\")" + ], + "refusal_codes": [ + "Recipient matrix body length differs from its dimensions." + ], + "encodes": "The recipient-matrix body: int64 entity ids followed by little-endian float64 features, one row per recipient (ACS persons for the predictor chain).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "16,429,824 body bytes at 342,288 rows x 48 (a memoryview over the loaded payload, np.frombuffer views; no second copy)", + "bytes_at_full_source": "164,298,624 body bytes at 3,422,888 rows x 48", + "measurement_basis": "Width read from model_input.py:88-94 (encoder) and :126-135 (decoder); counts from the brief section 3; product also in byte-census-measurements.json predictor_projection.matrix.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "same expression", + "head_value": "same expression (unchanged)", + "reachable_from": "19", + "reach_chain": "Every decode_recipient_matrix site in the previous row (complete_predictor_columns, read_current_survey_draws, LegacyQRFApplyMatrixKernel.run, runner :1263).", + "classification": "must-not-move", + "notes": "A struct-width identity the decoder needs to slice ids from features (:129, :135); it cannot bind on row count because rows is read from the header and the body is exactly rows wide by construction." + }, + { + "constant": "_graph_legacy_qrf.py:191 len(body) != len(index) * 8 (read_raw_target)", + "value": "exact raw-draw body width: 8 bytes (float64) per recipient row; an equality, not a ceiling. encode_raw_target (:170-175) has no cap.", + "enforced_at": [ + "_graph_legacy_qrf.py:188-193 ValueError(\"Raw legacy target/index/order binding changed.\") (also refuses a missing newline separator or a metadata mismatch)", + "_graph_legacy_qrf.py:172-173 (encoder: values.shape != (len(index),)) ValueError(\"Raw legacy target must be aligned finite float64 draws.\")" + ], + "refusal_codes": [ + "Raw legacy target/index/order binding changed.", + "Raw legacy target must be aligned finite float64 draws." + ], + "encodes": "One raw target draw artifact per apply node: _RAW_MAGIC + canonical JSON {target, index identity} + newline + float64 draws, one per recipient row (ACS persons for survey_predictors.apply.000-002).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "2,738,304 body bytes per raw_draw artifact (342,288 x 8) plus ~250 B metadata; three artifacts on the predictor chain", + "bytes_at_full_source": "27,383,104 body bytes per raw_draw artifact (3,422,888 x 8); three artifacts", + "measurement_basis": "Width read from _graph_legacy_qrf.py:170-175 and :186-194; counts from the brief section 3 (measure_pred.py raw_target restates the product).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "same expression", + "head_value": "same expression (unchanged)", + "reachable_from": "19", + "reach_chain": "LegacyQRFApplyMatrixKernel.run (graph_legacy_apply_matrix.py:95) -> _graph_legacy_apply.run_application -> codec.encode_raw_target (:115) and codec.read_raw_target for each prior (:111-113); CurrentSurveyPredictorAttachKernel.run -> read_current_survey_draws -> codec.read_raw_target (graph_current_survey_predictors.py:508-510); verifier (:589). Also 45 via graph_current_survey_property.py:493 and graph_property_income.py:320 (property family).", + "classification": "must-not-move", + "notes": "payload[len(_RAW_MAGIC):].partition(b\"\\n\") (:186) copies metadata and body once; np.frombuffer then views. No ceiling exists on this artifact anywhere on the path (store.put_bytes uncapped)." + }, + { + "constant": "_graph_legacy_qrf.py:45 encode_json(value) != payload (decode_json canonical re-encode)", + "value": "none (a canonical-form identity: json.loads then json.dumps(sort_keys, separators, allow_nan=False, ensure_ascii default) must reproduce the payload byte for byte)", + "enforced_at": [ + "_graph_legacy_qrf.py:45-46 ValueError(\"Legacy graph checkpoint must be a canonical JSON object.\") (also :36-37 \"Repeated legacy graph checkpoint JSON key.\", :41-42 \"Nonfinite legacy graph checkpoint JSON number.\")", + "callers on path: _graph_legacy_qrf.py:142 read_training, :155 read_application; graph_legacy_apply_matrix.py:28 decode_matrix_apply_state; graph_current_survey_predictors.py:251 (host_pins round trip), :391 (allocation, see its own row), :483-485 read_application(encode_json(packet[\"application\"])); model_input.py:107 (matrix header)" + ], + "refusal_codes": [ + "Legacy graph checkpoint must be a canonical JSON object.", + "Repeated legacy graph checkpoint JSON key.", + "Nonfinite legacy graph checkpoint JSON number." + ], + "encodes": "Every JSON packet the fit graph decodes: training_state and apply_state checkpoints (fixed-shape: a QRFChainState dict whose donor/recipient indices are five-field digests, qrf.py:592-600, plus three model/raw-target history entries), the recipient-matrix header, the host_pins dict. The one roster-shaped document that passes through it on the path is the allocation payload (next row).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "a few KB per packet (fixed-shape); unchanged by the fraction", + "bytes_at_full_source": "a few KB per packet; unchanged", + "measurement_basis": "Read from qrf.py:767-791 (QRFChainState.to_dict carries no row values) and _graph_legacy_qrf.py:123-167; not measured because no field scales with rows.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "same expression", + "head_value": "same expression (unchanged)", + "reachable_from": "19", + "reach_chain": "LegacyQRFTrainKernel.run -> codec.read_training (graph_legacy_train.py:96); run_application -> codec.read_training (:51) and state_decoder (:89-91); LegacyQRFApplyMatrixKernel.read_state -> decode_matrix_apply_state (graph_legacy_apply_matrix.py:83); CurrentSurveyPredictorAttachKernel.run -> read_current_survey_draws (:477, :483-485).", + "classification": "must-not-move", + "notes": "Protects the canonical-form contract every digest in the chain (matrix_sha256, model sha256, raw sha256 histories) depends on; it is not a size ceiling. The whole-document re-encode is harmless for these fixed packets; its one roster-scale use is recorded separately as the allocation consumer row." + }, + { + "constant": "graph_current_survey_predictors.py:391 codec.decode_json(loaded[\"allocation\"]) (consumer-side re-encoding of graph_survey_population's allocation payload)", + "value": "consumer cap: none. Producer caps: graph_survey_population.py:62 ALLOCATION_MAX_BYTES = 67,108,864 per segment, :68 ALLOCATION_ROSTER_BYTES = 64 * ALLOCATION_MAX_BYTES = 4,294,967,296 total", + "enforced_at": [ + "graph_current_survey_predictors.py:386-393 (CurrentSurveyPredictorProjectionKernel.run: codec.sha(loaded[\"frame_context\"]) == codec.decode_json(loaded[\"allocation\"])[\"output_context_sha256\"]) \"HOST_EDGE_BYTES\" ValueError(\"CURRENT_SURVEY_PREDICTOR_HOST_EDGE_BYTES\") on mismatch; the canonical check itself raises at _graph_legacy_qrf.py:45-46", + "off path: graph_puf_diagnostic_consumer.py:612-618 (same decode of allocated.payload) \"SURVEY_HOST_EDGE_BYTES\"", + "producer sites (other family): graph_survey_population.py:481-497 \"ALLOCATION_LIMIT\" / \"ALLOCATION_ROSTER_LIMIT\" SurveyPopulationGraphError" + ], + "refusal_codes": [ + "Legacy graph checkpoint must be a canonical JSON object.", + "HOST_EDGE_BYTES" + ], + "encodes": "The allocation artifact: {\"households\": [one instruction document per selected household: household_id, source, source_year, survey_year, raw_native_id, selected_receiving_household_id, four Fraction pairs, a float64 hex], ...six metadata keys}, streamed by the producer in 64 MiB segments and joined once (graph_survey_population.py:459-497).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "49,684,681 (158,737 selected households x 313.0 B) held three ways at once: the loaded payload, the json.loads tree, and the full re-encoded copy encode_json builds for the comparison", + "bytes_at_full_source": "496,848,688 (1,587,376 x 313.0 B), likewise held as payload + tree + re-encoded copy", + "measurement_basis": "Measured at this head through graph_survey_population._allocation_payload (its own _bounded_json(_instruction_document(row), 4096) per row) on invented rows at full-source id widths, difference of 1001 vs 1 rows (scratchpad measure_allocation.py -> allocation-measurement.json: per_selected_household_bytes 313.0; codec_decode_json_roundtrip_equal true). docs/us-native-scale-transport.md section 2d and the row-ceilings census cite 316.665 B per instruction at 1/1000 widths.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (consumer); producer 67,108,864 per segment / 4,294,967,296 total (the producer was segmented on native-scale-transport, docs section 2d; this consumer is unchanged on this branch)", + "head_value": "none (consumer); producer unchanged", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial -> node survey_predictors.source_projection -> CurrentSurveyPredictorProjectionKernel.run (graph_current_survey_predictors.py:369-397) -> host.shared.artifact(context, \"allocation\", ALLOCATION_TYPE) (:374) -> codec.decode_json(loaded[\"allocation\"]) (:391).", + "classification": "consumer", + "notes": "The decode exists only to read output_context_sha256; decode_json re-encodes the entire ~497 MB tree to prove canonical form, so the consumer accumulates the producer's whole roster in one bytes object (json.dumps builds it whole) with no ceiling of its own; the producer's total is 4 GiB and the full-source document is 11.6% of it. The streaming comparison the lane already uses for this shape is graph_survey_population._json_matches (:356-374; used at survey_origin_budget.py:935). Line 388 also hashes the whole loaded payload (codec.sha) but that streams over an already-materialised artifact." + }, + { + "constant": "current_survey_predictors.py:71 codec.sha(table.reset_index(drop=True).to_json(orient=\"table\").encode()) == document[f\"{entity}_projection_sha256\"] (consumer-side re-encoding of current_asec_demographics' person and household projections)", + "value": "none on either side: the producer (current_asec_demographics.py:359-364) hashes the same pandas to_json text with no cap", + "enforced_at": [ + "current_survey_predictors.py:59-74 _check_demographic_values, loop :68-74, \"DEMOGRAPHIC_PROJECTION_CHANGED\" ValueError(\"CURRENT_SURVEY_PREDICTOR_DEMOGRAPHIC_PROJECTION_CHANGED\"); called at :89 and :170 (_demographic_features) and :573 (final seal), i.e. three re-encodes of both tables per qualify", + "sibling site in another family: current_survey_geography.py:208-215 _check_asec_projection (household table only)" + ], + "refusal_codes": [ + "DEMOGRAPHIC_PROJECTION_CHANGED", + "DEMOGRAPHIC_RECEIPT" + ], + "encodes": "values.person: one row per selected ASEC person (person_id, native_person_id, asec_A_SEX, asec_AXSEX, asec_sex_binding_state, asec_sex_allocation_state, is_female, sex_known, sex_universe, sex_origin; current_asec_demographics.py:311-343) and values.household: one row per selected ASEC household (household_id, native_household_id, H_SEQ_integer, GESTFIPS, state_fips, state_status, state_known; :283-299), each rendered as pandas orient=\"table\" JSON text.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "persons 3,451,995 (14,212 x 242.9 B); households 919,443 (5,576 x 164.9 B)", + "bytes_at_full_source": "persons 34,521,167 (142,125 x 242.9 B); households 9,194,763 (55,762 x 164.9 B)", + "measurement_basis": "Measured at this head through the same expression on invented tables with the producer's exact columns and dtypes, difference of 1001 vs 1 rows at full-source id widths (measure_pred.py asec_demographic_projection_reencode: per_asec_person_bytes 242.893, per_asec_household_bytes 164.893).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none", + "head_value": "none (unchanged)", + "reachable_from": "19", + "reach_chain": "qualify_current_survey_predictors -> _demographic_features (:358-360, only when demographic_conditioning=True, which the pilot sets: experiments/native-scale-transport/harness19_base.py:532 demographic_conditioning=True, n_estimators=2) -> observed_geography.demographics.qualify_current_asec_demographics (:85) -> _check_demographic_values (:89, :170) and again at :573.", + "classification": "stays", + "notes": "No bound exists to meet; recorded because the brief counts consumer-side re-encodings of another module's document. values.person.index must equal the selected ASEC person set (:114-122), so the roster grows with the fraction until the whole ASEC file is selected and then plateaus at the upstream count (142,125 persons). Against the 64 MiB single-accumulation ceiling the neighbouring transports use, the person text is 1.94x under and the household text 7.3x under." + }, + { + "constant": "puf_diagnostic_consumer.py:671 CURRENT_SURVEY_MAX_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "puf_diagnostic_consumer.py:682-685 (0 < len(projection) <= CURRENT_SURVEY_MAX_BYTES) \"SURVEY_HOST_PROJECTION_BOUND\" bare ValueError (require = detail.require, :27; puf_detail_transfer.py:79-82 raises ValueError(reason))", + "puf_diagnostic_consumer.py:833 (len(matrix) <= CURRENT_SURVEY_MAX_BYTES) \"SURVEY_MATRIX_BOUND\" bare ValueError", + "graph_puf_diagnostic_consumer.py:403 (survey_graph._bounded_json(document, consumer.CURRENT_SURVEY_MAX_BYTES)) \"TRANSPORT_LIMIT\" SurveyPopulationGraphError at graph_survey_population.py:272 (limit <= 64 MiB), :281, :283", + "graph_puf_diagnostic_consumer.py:648-652 (kernel re-check of the projection payload) \"SURVEY_HOST_PROJECTION_BOUND\" bare ValueError" + ], + "refusal_codes": [ + "SURVEY_HOST_PROJECTION_BOUND", + "SURVEY_MATRIX_BOUND", + "TRANSPORT_LIMIT" + ], + "encodes": "Two documents. (a) The current-survey host projection: fixed digests plus asec.rows, one 7-element row [stacked_person_id, native_person_id, 2024, 16-hex amount, status, validity, zero_origin] per native ASEC person (:733-742, :754-757). (b) The recipient matrix from puf_detail_transfer.recipient_matrix (:830-832): 48 B per clone-one tax unit (8-byte id + five float64 FEATURES, puf_detail_transfer.py:29, :448-452).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "(a) 682,176 (14,212 native ASEC persons x 48.0 B) plus fixed preamble; (b) 10,236,144 SCALED (213,253 clone-one tax units x 48)", + "bytes_at_full_source": "(a) 6,822,000 (142,125 x 48.0 B); (b) 102,361,680 SCALED (2,132,535 clone-one tax units x 48, ratio 1.343434 tax units per household from the 1/1000 artifact)", + "measurement_basis": "(a) measured at this head through graph._bounded_json(document, CURRENT_SURVEY_MAX_BYTES) on 7-element rows at full-source widths (measure_pred.py diagnostic_host_projection: per_native_asec_person_row_bytes 48.0). (b) cited byte-census-measurements.json diagnostic_matrix (SCALED from the 1/1000 tax-unit ratio); width 8 x (1 + 5) confirmed from puf_detail_transfer.FEATURES and model_input.py:88-94.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "67,108,864", + "head_value": "67,108,864 (unchanged on this branch)", + "reachable_from": "none", + "reach_chain": "Reached only by qualify_current_survey_host / CurrentSurveyHostProjectionKernel.run / CurrentSurveyMatrixKernel.run / current_survey_recipient_matrix, whose sole callers are graph_current_survey_puf_transfer.py:224 and :380 (brief section 2: not on any path). The module itself is on the 19-path only for survey_source/survey_budget/survey_graph aliases, current_survey_host_edges (:421), _current_context_frame (:454) and _CurrentSurveyKernel (:511), none of which enforce this constant.", + "classification": "resource", + "notes": "Arm (b) is a numpy body that exists whole before the check and meets the bound at full source (102,361,680 > 67,108,864; 64 MiB admits 1,398,101 clone-one tax units, about 66% of the source); the row-ceilings census verdict (binds at full source) is confirmed at this head, SCALED. Arm (a) is capped by the real ASEC file at 142,125 rows and never binds. The encode side cannot be lifted in place: _bounded_json refuses any limit above 64 MiB at graph_survey_population.py:272." + }, + { + "constant": "puf_detail_transfer.py:34 MAX_BYTES", + "value": "128 * 1024 * 1024 = 134,217,728", + "enforced_at": [ + "puf_detail_transfer.py:114-119 (isinstance(payload, bytes) and len(payload) <= MAX_BYTES and payload.startswith(MAGIC)) \"PRICE_ENVELOPE\" bare ValueError (require :79-82)", + "graph_puf_diagnostic_consumer.py:104-107 (pinned size: 0 < size <= detail.MAX_BYTES) \"PRICE_TRANSPORT_SIZE\" bare ValueError", + "graph_puf_diagnostic_consumer.py:108-109 (handle.read(size + 1) read cap on each raw-bytes-v1 source, then consumer.check_pin :110-112 \"PRICE_TRANSPORT_PIN:\")" + ], + "refusal_codes": [ + "PRICE_ENVELOPE", + "PRICE_TRANSPORT_SIZE", + "PRICE_TRANSPORT_PIN" + ], + "encodes": "The local PUF price-restatement arrays envelope: MAGIC (38 B) + 8-byte big-endian header length + JSON header (<= 65,536) + 22 column bodies at 155 B per ordinary PUF return (ARRAY_DTYPES :50-61: 13 x f8, RECID i8, provenance recid i8, source AGI f8, S006 f8, FLPDYR i4, demographic_status i4, S006_delivered i8, FLPDYR_delivered i2, demographic_status_delivered i1).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "~32,192,260 + header (207,692 ordinary PUF returns x 155 B); invariant to the survey fraction", + "bytes_at_full_source": "~32,192,260 + header (same)", + "measurement_basis": "Width summed from ARRAY_DTYPES at this head (155 B); the 207,692-row count is the row-ceilings census's PUF figure (207,696 delivered minus four aggregate RECIDs), not re-measured here.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "134,217,728", + "head_value": "134,217,728 (unchanged)", + "reachable_from": "none", + "reach_chain": "decode_price_arrays is called only by graph_puf_detail_transfer.py:290 (FixturePriceExportKernel.run), :339 (FixtureDonorKernel.run) and puf_diagnostic_consumer.py:351 (qualify_price_transport via DiagnosticDonorKernel.run, graph_puf_diagnostic_consumer.py:113); none of those kernels is declared by the 19-, 45- or age runners.", + "classification": "must-not-move", + "notes": "Protects an upstream file's real size (4.2x headroom); the operative row ceiling on the same envelope is the inline 250_000 at :113 and MAX_DONORS, both row counts. The read cap at graph_puf_diagnostic_consumer.py:109 is the only read(N + 1) in this family." + }, + { + "constant": "puf_detail_transfer.py:128 0 < size <= 65536 (PRICE_HEADER_BOUND)", + "value": "65,536 on the envelope's JSON header, whose length is written as an 8-byte big-endian field (graph_puf_detail_transfer.py:271, read at puf_detail_transfer.py:126)", + "enforced_at": [ + "puf_detail_transfer.py:128 (0 < size <= 65536 and len(payload) >= start + size) \"PRICE_HEADER_BOUND\" bare ValueError" + ], + "refusal_codes": [ + "PRICE_HEADER_BOUND" + ], + "encodes": "The envelope header: six identity fields plus 22 column descriptors {dtype, rows, sha256} (graph_puf_detail_transfer.py:264-268), about 2.5 KB.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "~2.5 KB (fixed: 22 descriptors x ~100 B + six header fields); invariant", + "bytes_at_full_source": "~2.5 KB (same)", + "measurement_basis": "Shape read from graph_puf_detail_transfer.py:256-268 and puf_detail_transfer.py:149-166; not measured (row-count independent apart from the digits of \"rows\").", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "none", + "reach_chain": "Same callers as the MAX_BYTES row (fixture and diagnostic donor kernels only).", + "classification": "must-not-move", + "notes": "Protects a fixed-width length field's admitted range; the header carries no per-row content." + }, + { + "constant": "puf_detail_transfer.py:169-170 end = cursor + rows * np.dtype(entry[\"dtype\"]).itemsize; require(end <= len(payload)) (PRICE_BODY_TRUNCATED)", + "value": "struct width x declared rows per column, with rows bounded by max_rows (:161-166, 0 < rows <= max_rows; :113 0 < max_rows <= 250_000; default MAX_ROWS = 4096 at :33): at most 250,000 x 155 B = 38,750,000 B of column bodies", + "enforced_at": [ + "puf_detail_transfer.py:169-170 \"PRICE_BODY_TRUNCATED\" bare ValueError", + "puf_detail_transfer.py:177 (cursor == len(payload)) \"PRICE_TRAILING_BYTES\" bare ValueError", + "puf_detail_transfer.py:161-166 \"PRICE_COLUMN_TYPE_ROWS\", :168 \"PRICE_ROW_ALIGNMENT\", :113 \"PRICE_ROW_BOUND\" (the row-count arms)" + ], + "refusal_codes": [ + "PRICE_BODY_TRUNCATED", + "PRICE_TRAILING_BYTES", + "PRICE_COLUMN_TYPE_ROWS", + "PRICE_ROW_ALIGNMENT", + "PRICE_ROW_BOUND" + ], + "encodes": "Each column body of the price-restatement envelope, sliced at rows x itemsize and viewed with np.frombuffer (:171-173).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "207,692 x 155 = 32,192,260 B across the 22 columns; invariant to the survey fraction", + "bytes_at_full_source": "32,192,260 B (same)", + "measurement_basis": "Width from ARRAY_DTYPES (:50-61); PUF row count from the row-ceilings census.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "same expression; max_rows domain 250,000; MAX_ROWS default 4,096", + "head_value": "unchanged", + "reachable_from": "none", + "reach_chain": "Same callers as the MAX_BYTES row.", + "classification": "must-not-move", + "notes": "A struct-width product whose row factor is a PUF (upstream) count: 207,692 rows sit under the 250,000 max_rows the qualified route passes (puf_diagnostic_consumer.py:355 max_rows=rows) but above the 4,096 default, so a direct caller using the default refuses with PRICE_COLUMN_TYPE_ROWS. MAX_ROWS (:33), the inline 250_000 (:113) and puf_diagnostic_consumer.py:32 MAX_DONORS (250_000, enforced :331-341 \"PRICE_ORDINARY_ROSTER\") are row counts, not byte transports (row-count rule: the PUF ordinary-return count is fixed at 207,692 by the delivery)." + }, + { + "constant": "puf_diagnostic_consumer.py:77 0 < len(payload) <= 16_000_000 (parse_document)", + "value": "16,000,000", + "enforced_at": [ + "puf_diagnostic_consumer.py:76-79 \"DIAGNOSTIC_DOCUMENT_BOUND\" bare ValueError; callers :125 (the six DEA receipts acceptance/proposal/config/completion/cold/required), :287 and :293 (definition), :289 and :298 (projection_definition)" + ], + "refusal_codes": [ + "DIAGNOSTIC_DOCUMENT_BOUND", + "DIAGNOSTIC_DUPLICATE_KEY", + "DIAGNOSTIC_NONFINITE_JSON", + "DIAGNOSTIC_DOCUMENT_TYPE" + ], + "encodes": "Eight upstream JSON documents supplied as raw-bytes-v1 sources with reviewed content pins (graph_puf_diagnostic_consumer.py:56-57, :99-112): six DEA helper receipts, the PUF raw-source definition and the AGI projection definition.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "in-tree packaged twins: puf_2015_raw_source_definition.json 9,707 B, puf_2015_monetary_agi_source_projection.json 285,396 B; the six DEA receipts are not in this tree (sizes unknown here); invariant to the survey fraction", + "bytes_at_full_source": "same (upstream files)", + "measurement_basis": "wc -c of the packaged resources under packages/microcosm-build/src/microcosm/build/us_runtime/ at this head; the DEA receipts are fixture/packaged inputs the brief's read-only rule keeps me from fetching.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "16,000,000", + "head_value": "16,000,000 (unchanged)", + "reachable_from": "none", + "reach_chain": "qualify_price_transport is called only from DiagnosticDonorKernel.run (graph_puf_diagnostic_consumer.py:113), declared only by diagnostic_nodes (:242-351), which no runner calls.", + "classification": "must-not-move", + "notes": "Protects upstream files' real sizes; the packaged definition and projection are 0.06% and 1.8% of the bound." + }, + { + "constant": "graph_puf_diagnostic_consumer.py:583 and :703 len(pins_bytes) <= 4096 / len(encoded_pins) <= 4096 (SURVEY_HOST_EDGE_PIN_BOUND)", + "value": "4,096", + "enforced_at": [ + "graph_puf_diagnostic_consumer.py:582-583 (CurrentSurveyHostProjectionKernel.run: context.params[\"host_edges\"].encode()) \"SURVEY_HOST_EDGE_PIN_BOUND\" bare ValueError", + "graph_puf_diagnostic_consumer.py:702-703 (current_survey_host_nodes: codec.encode_json(host_pins)) \"SURVEY_HOST_EDGE_PIN_BOUND\" bare ValueError" + ], + "refusal_codes": [ + "SURVEY_HOST_EDGE_PIN_BOUND", + "SURVEY_HOST_EDGE_PIN_ROSTER", + "SURVEY_HOST_EDGE_PIN" + ], + "encodes": "The three-edge host pin document (preparation, allocation, frame_context; :421-441), each pin {producer_key, artifact_key, payload_sha256} of 64-hex strings (:696-701).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "about 700 B (3 pins x ~230 B); invariant", + "bytes_at_full_source": "about 700 B (same)", + "measurement_basis": "Shape read from :696-701 (three 64-hex digests per pin, codec._hash) and :421-441 (three edges); not measured.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4,096", + "head_value": "4,096 (unchanged)", + "reachable_from": "none", + "reach_chain": "current_survey_host_nodes and CurrentSurveyHostProjectionKernel.run are called only from graph_current_survey_puf_transfer.py (brief section 2: not on any path). The on-path twin, graph_current_survey_predictors._params (:139 \"host_edges\": codec.encode_json(host_pins).decode()), carries the same pins with no byte bound.", + "classification": "must-not-move", + "notes": "A fixed-shape token document; a fourth edge (the geography validation pin the predictor kernels add, graph_current_survey_predictors.py:123-124) would still sit near 900 B." + }, + { + "constant": "current_survey_health_source.py:316 projection = raw.reset_index().to_json(orient=\"table\", index=False).encode() (no cap; consumer re-encode at graph_current_survey_health.py:66-68)", + "value": "none on either side", + "enforced_at": [ + "graph_current_survey_health.py:64-70 health_coverage_seal: qualified.projection == qualified.raw.reset_index().to_json(orient=\"table\", index=False).encode() and evidence[\"projection_sha256\"] == sha256(projection) \"PROJECTION_BINDING\" ValueError(\"CURRENT_SURVEY_HEALTH_PROJECTION_BINDING\") (require = health.require, current_survey_health_coverage.py:98-100)", + "current_survey_health_source.py:326 (projection_sha256 = sha256(projection)) records the digest of the whole materialised text" + ], + "refusal_codes": [ + "PROJECTION_BINDING" + ], + "encodes": "The health raw-literal projection: one row per person of the preparation frame (origins = document[\"origins\"][\"persons\"], :108-127, checked equal to state.frame.person :128-140), carrying source, source_year, survey_year and the 35 RAW_COLUMNS literal tokens (ASEC NOW_* values and I_* flags, ACS HINS*/FHINS*P/FHINS*C), the other survey's columns null (:183-191).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "232,053,630 (356,501 stacked persons x 650.9 B), and a second whole copy in the consumer's re-encode", + "bytes_at_full_source": "2,320,538,261 (3,565,013 x 650.9 B), twice", + "measurement_basis": "Measured at this head through the same pandas expression on invented rows with the module's RAW_COLUMNS, one-character tokens on the row's own survey and None elsewhere, difference of 1001 vs 1 rows at full-source id widths (measure_pred.py health_projection: per_stacked_person_bytes 650.92).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none", + "head_value": "none (unchanged)", + "reachable_from": "none", + "reach_chain": "qualify_current_survey_health is called only through graph_current_survey_health.qualify_health_coverage (:43), imported by graph_us_survey_enrichment.py:51 (brief section 2: not on any path); current_survey_housing.py:31 and current_survey_person_status_source.py:25 import this module for _scan (housing is off path; person status is the 49/51-node variant only).", + "classification": "stays", + "notes": "There is no bound to stay under; the full-source text is 34.6x a 64 MiB accumulation and the seal holds it twice. While scanning, this module also enforces other families' bounds it does not define: asec_coverage_authentication._BODY_MAX (268,435,456) through asec._capture(..., budget=[asec._BODY_MAX]) at :265-271, acs_person_coverage_authentication.MAX_RECORD_BYTES (400,000) and MAX_CSV_HEADER_BYTES through records._records/_decode_record at :83-84, and acs_housing_universe_source._copy(..., exact_size=acs_size) at :287-293; those owners' rows should list these sites." + }, + { + "constant": "current_survey_health_coverage.py:124 len(x) <= 64 (LITERAL_CONTRACT) and current_survey_health_source.py:100 len(row[c]) <= 64 (SOURCE_TOKEN_BOUND)", + "value": "64 characters per source token", + "enforced_at": [ + "current_survey_health_coverage.py:122-126 (all(type(x) is str and len(x) <= 64 for x in (*values, *flags))) \"LITERAL_CONTRACT\" ValueError(\"CURRENT_SURVEY_HEALTH_LITERAL_CONTRACT\")", + "current_survey_health_source.py:100 (all(len(row[c]) <= 64 for c in columns)) \"SOURCE_TOKEN_BOUND\" ValueError(\"CURRENT_SURVEY_HEALTH_SOURCE_TOKEN_BOUND\")" + ], + "refusal_codes": [ + "LITERAL_CONTRACT", + "SOURCE_TOKEN_BOUND" + ], + "encodes": "Single CSV tokens: ASEC NOW_*/I_* codes and ACS HINS*/FHINS* codes (one character in the dictionaries) plus the key columns PERIDNUM (22 digits), PH_SEQ, A_LINENO, SERIALNO (13 chars), SPORDER.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "per token, at most 22 characters in the real files (PERIDNUM); invariant", + "bytes_at_full_source": "same", + "measurement_basis": "Token shapes from _key (:41-67: PERIDNUM [0-9]{22}, SERIALNO 2024(HU|GQ)[0-9]{7}) and the recode dictionaries (:127-134); not measured.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64", + "head_value": "64 (unchanged)", + "reachable_from": "none", + "reach_chain": "recode is reached from recode_field (graph_current_survey_health._completed, :103) and _scan from qualify_current_survey_health (:273, :300), both off path as in the previous row.", + "classification": "must-not-move", + "notes": "Protects a token width; checked per row as the CSV streams, no accumulation. The row-count bound beside it, :97 count <= maximum (maximum = the pinned ASEC member rows :278 or the ACS catalogue people count :305, \"SOURCE_ROW_SHAPE\"), and the 65,536-byte drain chunk at :308 (while stream.read(65536): pass, exhausting auxiliary zip members for their CRC) are not byte transports: the first is an exact upstream row pin, the second a chunk size with no ceiling and no document." + }, + { + "constant": "qrf.py:407 _PREDICT_CHUNK_ROWS", + "value": "10_000 rows per draw chunk (chunk = max(1, min(_PREDICT_CHUNK_ROWS, -(-n // (4 * workers)))), :353); bounds the (rows x 203-point _QUANTILE_GRID) float64 prediction matrix to about 16 MB per chunk", + "enforced_at": [ + "qrf.py:349-363 (_Forest.draw slices rows into chunks; no refusal, no exception)" + ], + "refusal_codes": [], + "encodes": "No document: an in-memory prediction workspace during the per-target draw over the recipient rows (ACS persons for the predictor chain).", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "about 16.2 MB per chunk (10,000 x 203 x 8) regardless of the 342,288 recipient rows", + "bytes_at_full_source": "about 16.2 MB per chunk regardless of the 3,422,888 rows", + "measurement_basis": "Product of the constant and _QUANTILE_GRID (203 points, :394-400) read from the module; not measured.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "10_000", + "head_value": "10_000 (unchanged)", + "reachable_from": "19", + "reach_chain": "LegacyQRFApplyMatrixKernel.run -> run_application -> qrf_target.apply_target -> qrf._draw_target_with_rng (:1003-1041) -> _Forest.draw (:305-382).", + "classification": "not-a-transport", + "notes": "A memory ceiling with no document behind it. The module's other ceilings are likewise not byte transports: _DISCRETE_Y_LEAF_BOUND = 4_096 (:432, applied at :527-528 as max_samples_leaf for near-discrete targets) and _DISCRETE_Y_UNIQUE_MAX = 32 (:424) bound the forest's per-leaf value store, a model hyperparameter, not an artifact." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/pred/measure_pred.py (predictor projection through _segmented_json at the module's segment/maximum; recipient matrix; ASEC demographic projection re-encode; diagnostic host projection; health projection; raw target width) -> measurements.json", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/pred/measure_allocation.py (allocation payload per selected household through graph_survey_population._allocation_payload; codec.decode_json round trip) -> allocation-measurement.json", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/pred/RECEIPT.md (head, interpreter, sys.path and counts used)" + ], + "open_questions": [ + "The brief's _recovered/pilot-runs/... inputs (the 1/1000 preparation.json, graph.json and the recovered projection/matrix artifacts) do not exist in this worktree, and the read-only rule kept me inside it; the projection's fixed evidence preamble (including the atomic_geography validation receipt) is therefore unmeasured and every roster figure above is the roster part only. Reachability was traced from graph_atomic_survey_financial.py and experiments/native-scale-transport/harness19_base.py:532 (demographic_conditioning=True, n_estimators=2) rather than from the recovered graph.json.", + "The recipient-matrix body (48 B per ACS person under demographic conditioning: 164.3 MB at full source, 16.4 MB at 1/10) has no byte bound anywhere on the 19-path: model_input caps only the header, store.put_bytes has no size check, and the body is re-encoded whole by every _Kernel._qualified (seven kernels), the runner and the verifier. Whether the lane wants a row-count or byte ceiling on it, and whether the qualify-per-kernel re-encoding is a cost the lane accepts, are design questions for the owner.", + "origins rows were measured with native_person_id at the 7-digit stacked-person width, as byte-census-measurements.json did; if full-source ACS native person ids are wider, each extra digit adds 1 B per stacked-person row (3.6 MB per digit at full source). The full-source id widths of the spine_source_id_column were not verified from a full-source artifact.", + "qrf_target.py (not in the family list; read at :240-340 because graph_legacy_train/apply call it) carries the fitted-model envelope: _MAGIC + 8-byte header length + JSON header + pickle body with no byte bound on either side (from_trusted_bytes only checks 0 < size < len(data) - offset). The pickle grows with donor rows (selected ASEC persons, 142,125 at full source) x n_estimators (2 in the pilot, 100 default) x leaf retention (max_samples_leaf=None keeps every leaf sample). Whichever family owns qrf_target should record it.", + "Other families' constants are enforced from this family's modules and should appear as enforcement sites on the owners' rows: asec_coverage_authentication.py:39 _BODY_MAX at current_survey_health_source.py:265-271; acs_person_coverage_authentication.py:48 MAX_RECORD_BYTES and MAX_CSV_HEADER_BYTES via records._records/_decode_record at current_survey_health_source.py:83-84; acs_housing_universe_source._copy exact_size at current_survey_health_source.py:287-293; graph_survey_population.py:62/:68 ALLOCATION_MAX_BYTES/ALLOCATION_ROSTER_BYTES consumed whole at graph_current_survey_predictors.py:391 (row above) and graph_puf_diagnostic_consumer.py:616; survey_population_preparation.py:50/:58 MAX_PAYLOAD_BYTES/MAX_ROSTER_BYTES at current_survey_predictors.py:522-523 (row above) and, off path, current_survey_amounts.py:369. current_survey_geography.py:208-215 _check_asec_projection is the household half of the demographic re-encode row in another family's module.", + "The preparation receipt (about 1.2 GB at full source per the row-ceilings census) is json.loads'ed whole at current_survey_predictors.py:105 and :336 (CheckedSurveyPopulationView) and current_survey_health_source.py:218; a decode, not a bound, so no row, but the source family may want the sites.", + "graph_survey_completion_host passes state.demographic_conditioning through to the base financial run (:895), so the 45-node pilot's setting is whatever native45-v5's harness passed; that harness is not in this tree, so the demographic re-encode row's 45-node reachability is inferred from the 19-node harness only.", + "The diagnostic matrix figure (row CURRENT_SURVEY_MAX_BYTES arm b) is SCALED from the 1/1000 tax-unit ratio (1.343434 per household), per the brief's rule, and the row is off path; a verifier wanting a measured count needs a tax-unit catalogue that does not exist in the counts." + ] + }, + { + "family": "ASEC current-money donor columns codec", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py (full, 143 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py (full, 1099 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py (full, 549 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py (full, 607 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py (full, 411 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py (full, 114 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py (full, 34 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py (full, 625 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py (full, 377 lines)", + "current_survey_predictors.py (lines 330-500; def index; :600 call)", + "graph_current_survey_predictors.py (lines 270-305; :301, :540, :556-578, :606 calls)", + "asec_person_income_source.py (lines 150-232, 300-330, 380-420, 436-490)", + "asec_2024_native_population.py (lines 660-745; _ISSUED/parent greps)", + "graph_asec_prepared.py (lines 520-575; selection/receipt greps)", + "survey_population_preparation.py (:1965 in prepare_authenticated_survey_population, :2167 in _current_survey_wage_projection; graph_sources uses)", + "graph_survey_population.py (:746, :1012 prepare_authenticated_survey_population)", + "graph_atomic_survey_financial.py (lines 1090-1115; :1786)", + "graph_property_tax_leaves.py (:39, :68-69, :384 cps_carried constants)", + "asec_coverage_authentication.py (:349, :356 money uses)", + "education_assistance_source.py (:117, :139, :161 rows pins; :346)", + "native_household_origin.py (:147, :157, :167 hhpub rows pins)", + "asec_income_observations.py (lines 400-412)", + "graph_composed_asec_binding.py (lines 77-92 imports; grep of :593, :781, :799, :956-1061)", + "graph_implementation.py (:109-117 loader registry via grep)", + "experiments/native-byte-transports/census-brief.md (full)", + "experiments/native-byte-transports/byte_census_measure.py (full)", + "experiments/native-byte-transports/byte-census-measurements.json (full dump; no money-family entry)", + "experiments/native-row-ceilings/census.json (the 21 rows matching this family)" + ], + "rows": [ + { + "constant": "asec_current_money.py:46 RESOURCE_MAX_BYTES", + "value": "512 * 1024 = 524,288", + "enforced_at": [ + "asec_current_money.py:84 (`_parse` default `limit`: `_require(type(value) is bytes and 0 < len(value) <= limit, \"JSON_SIZE_OR_TYPE\")`) \u2014 \"JSON_SIZE_OR_TYPE\", MoneyRefusalError (ValueError subclass, asec_current_money.py:59-70)", + "in-family callers inheriting the default: asec_current_money.py:223 (_restored_source), :401 (three packaged resources in compile_asec_current_money_spec), :482 (source identity), :794 (MoneyBindings, authenticated evidence), :923-925 (classify_asec_money scope/index identities), :1029; asec_current_money_source.py:295-297 (_input_binding), :504 (validate); asec_current_money_units.py:269, :289, :391, :396, :403; asec_current_money_graph_resources.py:25 (`_parse(payload)` of the 3,953 B graph consumers file)", + "out-of-family callers inheriting the default: asec_person_income_source.py:232, :465; asec_student_controls.py:81, :180, :318, :329, :424, :453; asec_income_observations.py:147, :179, :307, :477, :487; asec_prepared_source.py:417, :506; graph_asec_income.py:120, :164, :217, :359, :467, :477, :585 (prepared receipt payload), :586 (ASEC household selection payload)" + ], + "refusal_codes": [ + "JSON_SIZE_OR_TYPE" + ], + "encodes": "Upstream packaged JSON resources (asec_current_money_domains_v1.json 136,308 B; asec_current_money_consumers_v1.json 17,494 B; asec_current_money_price_basis_v1.json 689 B; asec_current_money_graph_consumers_v1.json 3,953 B), all sha256-pinned (RESOURCE_PINS :39-43, GRAPH_CONSUMERS_SHA256), plus fixed-shape identity documents: the source evidence (synthetic 1,236 B; authenticated schema-2 evidence adds three keys), the spec identity (2,217 B), the scope identity (684 B: eight digests), index descriptors, the money header when parsed at units:403 (1,094 B), the tax receipt and implementation identity. Off-path only, graph_asec_income.py:585-586 also parses the prepared receipt and the per-selected-ASEC-household selection payload under this default (see open questions).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "136,308 (largest document under this cap: asec_current_money_domains_v1.json); identical at every fraction", + "bytes_at_full_source": "136,308; identical at every fraction (packaged files and fixed-key identity documents; 3.85x headroom on the largest)", + "measurement_basis": "ls -l of the four packaged resource files at this head; identity document sizes from scratchpad receipt.json fixed_documents (synthetic_source_identity 1,236; spec_identity 2,217; execution_identity 475; scope_identity 684) via measure_money_bodies.py", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "524,288 (git diff origin/native-row-ceilings 0319b1c73 is empty for this module)", + "head_value": "524,288", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial \u2192 graph_survey_population (:746/:1012 owner.prepare_authenticated_survey_population) \u2192 survey_population_preparation.prepare_authenticated_survey_population (:1965 asec_native.load_authenticated_asec_2024_native_population) \u2192 asec_2024_native_population (:702 restoration.load_authenticated_restored_current_money_source) \u2192 asec_person_income_source (:455 verify \u2192 :316 legacy.load_authenticated_current_money_source; :483 compile_asec_current_money_spec(load_current_money_resources(), authority)) \u2192 asec_current_money_source._load (:604 compile_asec_current_money_spec) \u2192 asec_current_money.compile_asec_current_money_spec :401 `_parse(v)` for domains/price/consumers; also survey_population_preparation.py:2167 `current_money._origin_code(parent.spec, \"WSAL_VAL\")` \u2192 :223 `_parse(source.identity)`; every `_spec` re-check (:493-498) recompiles and re-parses", + "classification": "must-not-move", + "notes": "The three resources are pinned bit-for-bit (`tuple(_sha(v) for v in (resources.domains, resources.price, resources.consumers)) == RESOURCE_PINS`, :405-411), so the cap is a parse guard in front of already-fixed bytes. No document under this cap on the 19/45 paths is roster-shaped. graph_asec_income.py:586 `_parse(values[\"selection\"].payload)` is the one roster-shaped consumer (selected ASEC households, produced by graph_asec_prepared.py:507-518 `\"selection\": _json(selection)`), reachable only through the PUF diagnostic consumer (puf_diagnostic_consumer.py:19, graph_puf_diagnostic_consumer.py:36), which the brief lists as off every path; it belongs to the graph_asec_income family and is flagged, not measured." + }, + { + "constant": "asec_current_money.py:47 HEADER_MAX_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "asec_current_money.py:84 via explicit limit at :147 (`_parse(self.evidence, HEADER_MAX_BYTES)` in AuthenticatedAsecSource._validate_content), :417 (execution identity), :739 (MoneyBindings.__post_init__ header), :868 (_validate_money), :1050 (restate_asec_current_money) \u2014 \"JSON_SIZE_OR_TYPE\", MoneyRefusalError", + "_asec_current_money_codec.py:115-118 (decode_current_money: `0 < size <= HEADER_MAX_BYTES and start + size <= len(payload) - 32`) \u2014 \"HEADER_SIZE\", MoneyRefusalError; :120 (`_parse(header, HEADER_MAX_BYTES)`)", + "asec_current_money_selection.py:193, :221 (header_data), :275-278 (parse_current_money_body) \u2014 \"HEADER_SIZE\"; :281; :399 (_validate_selected); :503-506 (decode_selected_current_money) \u2014 \"HEADER_SIZE\"; :508", + "out-of-family: graph_composed_asec_binding.py:1016 (`len(header) <= HEADER_MAX_BYTES`) \u2014 \"ARM_ROWS_HEADER_SIZE\", PreparedGraphError; :1057 \u2014 \"ARM_ROWS_HEADER_SIZE\"; :1061 (`_parse(header, HEADER_MAX_BYTES)`)", + "additive term of _asec_current_money_codec.py:27 PAYLOAD_MAX_BYTES, asec_current_money_selection.py:86 SELECTED_PAYLOAD_MAX_BYTES and graph_composed_asec_binding.py:959 ARM_ROWS_PAYLOAD_MAX_BYTES" + ], + "refusal_codes": [ + "JSON_SIZE_OR_TYPE", + "HEADER_SIZE", + "ARM_ROWS_HEADER_SIZE" + ], + "encodes": "Fixed-shape headers: the money header (closed key set :740-758 plus provenance: digests, two row counts, the 32-name field list) = 1,094 B at three-cohort digit widths; the selected header (_SELECTED_HEADER_KEYS :58-81 incl. 32 field_entities pairs) = 2,043 B; the source evidence (:158-175, schema 2 adds three keys) \u2248 1.2-1.5 KB; the execution identity 475 B. The header travels as a uint32 length prefix (`struct.pack(\" 64", + "value": "len(value) == 64 (hex sha256 width) = 64", + "enforced_at": [ + "asec_current_money.py:106-111 `_digest` (`type(value) is str and len(value) == 64 and all(c in \"0123456789abcdef\" ...)`), a predicate consumed by: asec_current_money.py:191-204 \u2014 \"SOURCE_PIN\"; :205-211 \u2014 \"RESTORED_SOURCE_SCHEMA\"; :266-276 \u2014 \"SOURCE_PIN\"; :281-287 \u2014 \"COHORT_PINS\"; :296-302 \u2014 \"SIDECAR_PINS\"; :431-433 \u2014 \"EXECUTION_MODULES\"; :781-792 \u2014 \"HEADER_DIGEST\" (all MoneyRefusalError)", + "asec_current_money_selection.py:269 \u2014 \"EXPECTED_HEADER_DIGEST\"; :270 \u2014 \"EXPECTED_CONTENT_DIGEST\"; :333-344 \u2014 \"SELECTION_BINDING_DIGEST\"; :415-428 \u2014 \"SELECTED_HEADER_DIGEST\"; :488-499 \u2014 \"SELECTION_BINDING_DIGEST\" (MoneyRefusalError)" + ], + "refusal_codes": [ + "SOURCE_PIN", + "RESTORED_SOURCE_SCHEMA", + "COHORT_PINS", + "SIDECAR_PINS", + "EXECUTION_MODULES", + "HEADER_DIGEST", + "EXPECTED_HEADER_DIGEST", + "EXPECTED_CONTENT_DIGEST", + "SELECTION_BINDING_DIGEST", + "SELECTED_HEADER_DIGEST" + ], + "encodes": "A single token: one 64-character lowercase-hex sha256 string (pins, digests inside headers and evidence).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 (fixed token width); does not move with any roster", + "bytes_at_full_source": "64", + "measurement_basis": "code: sha256().hexdigest() is 64 hex characters; every digest checked is produced by _sha (:73-74)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 (no diff vs origin/native-row-ceilings)", + "head_value": "64", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_current_money_source._load :603 AuthenticatedAsecSource \u2192 _validate_content :191-204; predictor chain \u2192 classify :1036 MoneyBindings \u2192 :781-792", + "classification": "must-not-move", + "notes": "An exact width, not a ceiling; changing it would reject every real digest. Listed so the `len(...)` check is not silently dropped from the census." + }, + { + "constant": "asec_current_money.py:447 128", + "value": "0 < len(v) <= 128 \u2192 128 characters per execution-identity string", + "enforced_at": [ + "asec_current_money.py:445-452 (compile_asec_current_money_spec: `all(type(v) is str and 0 < len(v) <= 128 for group in (execution[\"dependencies\"], execution[\"platform\"]) for v in group.values())`) \u2014 \"EXECUTION_IDENTITY\", MoneyRefusalError" + ], + "refusal_codes": [ + "EXECUTION_IDENTITY" + ], + "encodes": "Single scalar tokens: the seven dependency/platform strings of the execution identity built at asec_current_money_resources.py:46-63 (python version, numpy, pandas versions; system, machine, byteorder, python_implementation).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "\u2264 7 characters each in this venv ('3.14.4', '2.4.6', '3.0.3', 'Darwin', 'arm64', 'little', 'CPython'); whole execution identity 475 B; fixed", + "bytes_at_full_source": "same; no roster behind it", + "measurement_basis": "receipt.json fixed_documents.execution_identity_bytes = 475 via packaged_current_money_resources at this head; venv versions printed by the sanity check", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "128 (no diff vs origin/native-row-ceilings)", + "head_value": "128", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_current_money_source._load :604 / asec_person_income_source.py:483 `compile_asec_current_money_spec(load_current_money_resources(), authority)` \u2192 :445-452; re-run on every `_spec` check (:493-498) from _validate_money/classify on the predictor chain", + "classification": "must-not-move", + "notes": "Protects a fixed-width field of a fixed-shape identity document; not a transport." + }, + { + "constant": "asec_current_money.py:573 256", + "value": "0 < len(k) <= 256 \u2192 256 characters per native key", + "enforced_at": [ + "asec_current_money.py:565-575 (AsecMoneyScope.__post_init__: `all(type(k) is str and 0 < len(k) <= 256 for k in keys)` for person_native_keys and household_native_keys) \u2014 \"NATIVE_KEYS\", MoneyRefusalError" + ], + "refusal_codes": [ + "NATIVE_KEYS" + ], + "encodes": "A per-element width on a per-row roster: one native key string per ASEC person (PERIDNUM, decoded at asec_current_money_source.py:272-275; an exact 22-digit string per education_assistance_source.py:346) and one per ASEC household (`str(int(v)) for v in keys.H_SEQ`, :276, \u2264 6 digits). The roster length is bounded by MAX_PERSONS/MAX_HOUSEHOLDS, not by this literal.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "22 characters per person key, \u2264 6 per household key (fixed widths); the whole tuple is 432,523 + H3 strings at every fraction", + "bytes_at_full_source": "same; 11.6x headroom on the 22-character PERIDNUM", + "measurement_basis": "code widths (education_assistance_source.py:346 PERIDNUM 22 digits; H_SEQ str(int)); measurement scripts used 22-character keys and passed the check", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "256 (no diff vs origin/native-row-ceilings)", + "head_value": "256", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_current_money_source._load :584 `_scope(frame)` \u2192 :265-277 AsecMoneyScope(...) \u2192 __post_init__ :570-575; predictor chain \u2192 validate :512 `_scope(self.frame)`", + "classification": "must-not-move", + "notes": "Fixed-width field. The same tuples are also JSON-encoded whole and uncapped by `identity` (:583-585); see open questions." + }, + { + "constant": "_asec_current_money_codec.py:26 PAYLOAD_MAX_BYTES", + "value": "len(MAGIC) + 4 + HEADER_MAX_BYTES + 11 * (32 * MAX_PERSONS + MAX_HOUSEHOLDS) + 32 = 9 + 4 + 65,536 + 11 * (32,000,000 + 400,000) + 32 = 356,465,581", + "enforced_at": [ + "_asec_current_money_codec.py:56 (_ready_parts, shared by encode_current_money :79 and current_money_content_sha256 :65: `_require(sum(map(len, parts)) + 32 <= PAYLOAD_MAX_BYTES, \"PAYLOAD_SIZE\")`) \u2014 \"PAYLOAD_SIZE\", MoneyRefusalError", + "_asec_current_money_codec.py:97-101 (decode_current_money: `len(MAGIC) + 4 + 32 < len(payload) <= PAYLOAD_MAX_BYTES`) \u2014 \"PAYLOAD_SIZE\"; :122-125 exact length `start + size + 11 * (32 * data[\"person_rows\"] + data[\"household_rows\"]) + 32` \u2014 \"PAYLOAD_LENGTH\"", + "asec_current_money_selection.py:264-268 (parse_current_money_body, imported as BODY_PAYLOAD_MAX_BYTES :28) \u2014 \"PAYLOAD_SIZE\"; :298-301 \u2014 \"PAYLOAD_LENGTH\"", + "consumer-side re-encodings of the whole body through _ready_parts (cap :56): asec_current_money_units.py:299-305 (`current_money_content_sha256(self.money) == receipt[\"money_content_sha256\"]`) \u2014 \"TAX_RESULT_CHANGED\"; asec_student_controls.py:476; current_survey_person_status_source.py:516; graph_current_survey_person_status.py:279; and byte comparisons _asec_current_money_codec.py:137 (`tuple(fields) == expected.fields`) \u2014 \"REPLAY_CONTENT\", asec_current_money_selection.py:305-307 (content re-hash) \u2014 \"BODY_CONTENT_BINDING\"" + ], + "refusal_codes": [ + "PAYLOAD_SIZE", + "PAYLOAD_LENGTH" + ], + "encodes": "The numpy money body of the whole authenticated ASEC current-money source: MAGIC (9) + uint32 header length + header (1,094 B) + for each of 32 person fields four buffers of 8+1+1+1 bytes per ASEC person (`field.amount_bytes, status_bytes, validity_bytes, zero_origin_bytes`, :47-55) + 11 bytes per ASEC household for HTOTVAL + 32-byte sha256. The source carries all three income years (asec_current_money_source.py:264), so the body is 432,523 persons plus the three-cohort households at every selection fraction.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "154,115,924 (432,523 persons pinned; households 169,699 ESTIMATE) \u2014 at most 155,190,448 (households \u2264 267,383); identical at every fraction because the source is the whole pinned file set. A 2024-cohort-only body would be 50,642,520 (measured) but the loader refuses a source without all three years.", + "bytes_at_full_source": "154,115,924 (ESTIMATE households) / \u2264 155,190,448 (hard bound); headroom 2.31x / 2.30x against 356,465,581", + "measurement_basis": "measure_three_cohort_body.py through _asec_current_money_codec.encode_current_money on invented zero amounts (asserted == 9 + 4 + len(header) + 11 * (32P + H) + 32); marginals 352 B/person and 11 B/household from measure_money_bodies.py at (200,40)/(1200,40)/(200,140); person pins education_assistance_source.py:117/139/161 enforced at asec_person_income_source.py:171; receipts receipt_three_cohort.json and receipt.json", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "356,465,581 (no diff vs origin/native-row-ceilings)", + "head_value": "356,465,581", + "reachable_from": "none", + "reach_chain": "none from 19/45: the predictor lane slices the in-process ready object (current_survey_predictors.py:391-394 `field = ready.field(name)`, `amounts = field.amounts[take]`) and never calls the codec. Functions that would reach it: asec_prepared_source.py:471 `encode_current_money(ready)` / :503 (load_graph_asec_prepared, registered at graph_implementation.py:117); asec_income_observations.py:490; asec_student_controls.py:332/:476/:487; current_survey_person_status_source.py:346/:516 (49/51-node person status); asec_current_money_units.py:301/:390 (reconstruct_current_money_tax_units \u2190 asec_prepared_source.py:440); parse_current_money_body only at graph_asec_prepared.py:422 and graph_composed_asec_binding.py:593 (PUF diagnostic consumer path)", + "classification": "stays", + "notes": "Derived: it moves only with MAX_PERSONS/MAX_HOUSEHOLDS/HEADER_MAX_BYTES; the row widths (8,1,1,1) are fixed at :132 and :46-55 and the decoder recomputes the exact length from the header rows (:122-125), so the cap is the pre-parse budget, not the wire format. The bytes exist whole: `b\"\".join((*parts, checksum.digest()))` (:83); the content digest streams over parts that already exist as MoneyField bytes. The row-ceilings census's ~154.1 MB agrees; its `count_at_full_source` for this row is fraction-independent, as here." + }, + { + "constant": "asec_current_money_selection.py:83 SELECTED_PAYLOAD_MAX_BYTES", + "value": "len(SELECTED_MAGIC) + 4 + HEADER_MAX_BYTES + 11 * (32 * MAX_PERSONS + MAX_HOUSEHOLDS) + 32 = 9 + 4 + 65,536 + 356,400,000 + 32 = 356,465,581", + "enforced_at": [ + "asec_current_money_selection.py:461 (_selected_parts, used by encode_selected_current_money :467: `_require(sum(map(len, parts)) + 32 <= SELECTED_PAYLOAD_MAX_BYTES, \"PAYLOAD_SIZE\")`) \u2014 \"PAYLOAD_SIZE\", MoneyRefusalError", + "asec_current_money_selection.py:483-487 (decode_selected_current_money: `len(SELECTED_MAGIC) + 4 + 32 < len(payload) <= SELECTED_PAYLOAD_MAX_BYTES`) \u2014 \"PAYLOAD_SIZE\"; :523-524 exact length \u2014 \"PAYLOAD_LENGTH\"", + "consumer-side re-encoding of the whole selected body under the :461 cap: graph_asec_income.py:380 (`_sha(encode_selected_current_money(selected)) == selected_money_sha256`); producers graph_asec_prepared.py:563/:569, graph_composed_asec_binding.py:799; decoders graph_asec_income.py:607, graph_asec_prepared.py:691, graph_composed_asec_measures.py:202" + ], + "refusal_codes": [ + "PAYLOAD_SIZE", + "PAYLOAD_LENGTH" + ], + "encodes": "The selected-subset money body: whole selected ASEC households and their persons, a strictly increasing position slice of the body (`_positions` :122-132; `field.amounts[index].tobytes()` :360-366): SELECTED_MAGIC + uint32 + selected header (2,043 B) + 352 B per selected ASEC person + 11 B per selected ASEC household + 32.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "5,066,046 (14,212 selected ASEC persons, 5,576 households; header 2,041 B)", + "bytes_at_full_source": "50,643,470 (142,125 persons, 55,762 households; header 2,043 B); 7.04x headroom", + "measurement_basis": "measure_money_bodies.py: parse_current_money_body(encode_current_money(ready), \u2026) \u2192 select_current_money(positions 0..P-1, 0..H-1) \u2192 encode_selected_current_money, asserted == 9 + 4 + len(header) + 11 * (32P + H) + 32 and round-tripped through decode_selected_current_money; measured at 1/10 and full 2024-cohort counts from the brief \u00a73 (receipt.json full_source_body_selected_tenth / full_source_body_selected_all)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "356,465,581 (no diff vs origin/native-row-ceilings)", + "head_value": "356,465,581", + "reachable_from": "none", + "reach_chain": "none from 19/45: select_current_money / encode_selected_current_money are called only from graph_asec_prepared.py:543/:563/:569 (load_graph_asec_prepared lane) and graph_composed_asec_binding.py:781/:799 (PUF diagnostic consumer via puf_diagnostic_consumer.py:498); decode only from graph_asec_income.py:607, graph_asec_prepared.py:691, graph_composed_asec_measures.py:202; the on-path predictor lane takes its ASEC slice directly from the ready object (current_survey_predictors.py:381-399)", + "classification": "stays", + "notes": "Bounded above by the parent body's rows (selection is a subset), so it can never exceed the PAYLOAD_MAX_BYTES row's document. The row-ceilings census recorded '~52 MB' at full source; the encoder gives 50,643,470 B." + }, + { + "constant": "asec_current_money_source.py:72 1024 * 1024", + "value": "1024 * 1024 = 1,048,576 (a read chunk, not a cap)", + "enforced_at": [ + "asec_current_money_source.py:72 (_stage_verified: `while chunk := src.read(1024 * 1024): digest.update(chunk); dst.write(chunk)`) \u2014 no size refusal; the loop digests and copies the whole file. Refusals on this path are identity refusals only: \"SOURCE_BYTES_CHANGED\" (:76-92, fstat before/after), \"SOURCE_BYTES_MISMATCH\" (:93), \"SOURCE_BYTES_UNAVAILABLE\" (:94-95); MoneyRefusalError" + ], + "refusal_codes": [], + "encodes": "Upstream files streamed whole into a private staged copy: the pinned parent checkpoint (parent.h5, _SOURCE_PINS[0]) and the household-observation attachment (attachment.h5, _SOURCE_PINS[1]) at :549/:555, and the three ASEC person member CSVs via asec_person_income_source.py:167.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "the input files' own sizes, identical at every fraction; not read here (gated data; their byte caps are asec_2024_native_population._MAX_CHECKPOINT_BYTES :720 and asec_person_income_source._ATTACHMENT_MAX_BYTES :39, other families)", + "bytes_at_full_source": "same files; the chunk size does not bound them", + "measurement_basis": "code only (no data read, per brief \u00a74)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,048,576 (no diff vs origin/native-row-ceilings)", + "head_value": "1,048,576", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_person_income_source :316 legacy.load_authenticated_current_money_source \u2192 asec_current_money_source._load :549 `_stage_verified(parent_path, _SOURCE_PINS[0], parent_copy)`, :555; also asec_person_income_source.py:167 `legacy._stage_verified(member_paths[year], pin, staged)`", + "classification": "must-not-move", + "notes": "Recorded because it is an inline byte literal on a read; it bounds nothing. The whole file is digested and copied; the only size-shaped observation is `before.st_size == after.st_size` (:81/:88)." + }, + { + "constant": "asec_current_money_source.py:126 1024 / 262144", + "value": "len(string_tokens) < 1024 and token_bytes + len(token) <= 262144 \u2192 1,024 cache entries / 262,144 bytes", + "enforced_at": [ + "asec_current_money_source.py:126-128 (_series_digest cache admission: `if len(string_tokens) < 1024 and token_bytes + len(token) <= 262144: string_tokens[value] = token`) \u2014 no refusal, nothing raised; a token past either ceiling is still digested (:129 `digest.update(token)`) but not memoised" + ], + "refusal_codes": [], + "encodes": "Nothing on the wire: a per-column memo of length-prefixed `checkpoint._encode_object_scalar` string tokens while `_frame_signature` (:136-165) streams every column of the whole source frame into sha256. The digest itself is unbounded.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "\u2264 262,144 retained per object column by construction; the digested stream is the whole three-cohort frame at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "code only", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,024 / 262,144 (no diff vs origin/native-row-ceilings)", + "head_value": "1,024 / 262,144", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_current_money_source._load :560 `_validate_attachment` \u2192 :394 `_same_table` \u2192 :334-335 `_series_digest`; :599 `_frame_signature(frame)`; asec_person_income_source.py:238/:239/:474; asec_coverage_authentication.py:356 `money._frame_signature(source.frame)`; predictor chain \u2192 AuthenticatedCurrentMoneySource.validate :511", + "classification": "not-a-transport", + "notes": "A memory ceiling with no document behind it; the comment at :114-117 states the intent (\"Both entry and payload bounds keep distinct or large strings bounded\"). Not a cap on the digested bytes." + }, + { + "constant": "asec_current_money_resources.py:104 1024 * 1024", + "value": "1024 * 1024 = 1,048,576", + "enforced_at": [ + "asec_current_money_resources.py:104 (load_current_money_resources: `_require(path.stat().st_size <= 1024 * 1024, \"MICROUNIT_MODULE_SIZE\")`) \u2014 \"MICROUNIT_MODULE_SIZE\", MoneyRefusalError", + "asec_current_money_resources.py:106-109 (`len(module) <= 1024 * 1024 and _sha(module) == MICROUNIT_SHA256`) \u2014 \"MICROUNIT_MODULE_SHA256\", MoneyRefusalError; OSError/PackageNotFoundError \u2192 \"MICROUNIT_UNAVAILABLE\" (:110-111)" + ], + "refusal_codes": [ + "MICROUNIT_MODULE_SIZE", + "MICROUNIT_MODULE_SHA256" + ], + "encodes": "Upstream file: the installed microunit/tax_unit_construction.py, whose sha256 must equal MICROUNIT_SHA256 (asec_current_money.py:45).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "31,259 (the venv's microunit 0.1.0 tax_unit_construction.py; sha256 79007065\u2026 matches the pin); identical at every fraction; 33.5x headroom", + "bytes_at_full_source": "31,259", + "measurement_basis": "importlib.metadata.distribution('microunit').locate_file(...).stat().st_size in the worktree venv, sha256 verified against MICROUNIT_SHA256", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,048,576 (no diff vs origin/native-row-ceilings)", + "head_value": "1,048,576", + "reachable_from": "19", + "reach_chain": "admission chain \u2192 asec_current_money_source._load :604 `compile_asec_current_money_spec(load_current_money_resources(), authority)`; asec_person_income_source.py:483 (restored source) \u2192 asec_current_money_resources.load_current_money_resources :72-114", + "classification": "must-not-move", + "notes": "Protects an upstream file whose exact bytes are pinned; the stat check precedes the read so the whole read is bounded before it materialises." + }, + { + "constant": "asec_current_money_units.py:114 1024 * 1024", + "value": "1024 * 1024 = 1,048,576", + "enforced_at": [ + "asec_current_money_units.py:114 (_runtime_file_bytes: `_require(path.stat().st_size <= 1024 * 1024, \"MICROUNIT_EXECUTION_SIZE\")`) and :116 (`_require(len(value) <= 1024 * 1024, \"MICROUNIT_EXECUTION_SIZE\")`), called at :129 for each of the seven _MICROUNIT_EXECUTION_PINS \u2014 \"MICROUNIT_EXECUTION_SIZE\", MoneyRefusalError (sha mismatch \u2192 \"MICROUNIT_EXECUTION_PIN\" :130)" + ], + "refusal_codes": [ + "MICROUNIT_EXECUTION_SIZE" + ], + "encodes": "Upstream files: the seven pinned microunit runtime files (__init__.py 1,799 B; core.py 6,439; diagnostics.py 2,655; registry.py 1,267; rule_helpers.py 4,638; tax_unit_construction.py 31,259; data/dependent_gross_income_limit.yaml 2,769 \u2014 venv, all sha-pinned at :102-110).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "31,259 (largest of the seven); identical at every fraction; 33.5x headroom", + "bytes_at_full_source": "31,259", + "measurement_basis": "sizes and sha256 of each pinned file from the worktree venv's microunit 0.1.0 distribution", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,048,576 (no diff vs origin/native-row-ceilings)", + "head_value": "1,048,576", + "reachable_from": "none", + "reach_chain": "none from 19/45: `_execution` (:120) is called only by `_reconstruct` (:324, :364) and `CurrentMoneyTaxUnitResult.validate` (:307); `reconstruct_current_money_tax_units` is called only from asec_prepared_source.py:440 (load_graph_asec_prepared lane); `CurrentMoneyTaxUnitResult` is consumed by asec_housing_universe_source.py:17 and asec_housing_status_source.py:24, the ASEC housing sources, not the on-path acs_housing_universe_source", + "classification": "must-not-move", + "notes": "Same shape as the resources-module cap; protects upstream pinned files. The units module also re-derives the whole money body without any byte cap (`expected = source.ready()`, `ready.fields == expected.fields`, :189-193) and JSON-encodes the person-by-tax-unit partition whole (`_sha(_json(_partitions(original)))`, :401-402); both off-path, see open questions." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/money/measure_money_bodies.py (encode_current_money / parse_current_money_body / select_current_money / encode_selected_current_money / decode_selected_current_money on invented zero amounts at full-source id widths; receipt: .../census/money/receipt.json)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/money/measure_three_cohort_body.py (encode_current_money at the three-cohort source counts 432,523 persons with 169,699 and 267,383 households; receipt: .../census/money/receipt_three_cohort.json)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/money/README.md (index; both scripts assert every imported module resolves inside the worktree, per the venv-shadowing memory note)" + ], + "open_questions": [ + "Three-cohort household count of the authenticated source (H3) is not pinned in code: only the 2024 cohort (55,762) is a catalogue count; 169,699 is the row-ceilings lane's estimate and 267,383 (all hhpub rows, native_household_origin.py:147/157/167) is the hard upper bound. The PAYLOAD_MAX_BYTES verdict does not depend on it (headroom 2.31x vs 2.30x), but the body's exact byte figure does.", + "Uncapped materialised JSON on the 19 path: AsecMoneyScope.identity (asec_current_money.py:583-585) encodes each roster vector whole with `_json(getattr(self, f.name))` before hashing \u2014 at the three-cohort source person_native_keys is 10,813,076 B and person_ids 3,027,662 B (measured). There is no bound and no re-encoding check, so no row; flag for the transport lane if whole-roster JSON materialisation is in scope.", + "Off-path roster-shaped consumer of RESOURCE_MAX_BYTES: graph_asec_income.py:586 `_parse(values[\"selection\"].payload)` parses the ASEC household selection payload (graph_asec_prepared.py:507-518, `\"selection\": _json(selection)`) under the 512 KiB default; that document is per selected ASEC household and belongs to the graph_asec_income family (PUF diagnostic consumer path, off 19/45). Not measured here; its owner should decide whether it binds at 55,762 households.", + "Off-path uncapped re-derivation: asec_current_money_units.current_money_microunit_view (:189-193) recomputes the whole money body via `source.ready()` and compares MoneyField bytes with no byte cap; `_partitions` JSON (:401-402) is a whole-roster JSON with no cap. Reached only from asec_prepared_source.py:440.", + "cps_carried.py and cps_carried_current.py carry no byte bound at all; cps_carried.py:79 US_REPORTED_COVERAGE_VINTAGE_GATE_MIN_ROWS = 5_000 is a row-count minimum enforced in support_provenance (not a byte bound) and gets no row. cps_carried is reached from the 45-node graph only as constants (graph_property_tax_leaves.py:68-69; graph_atomic_survey_financial.py:1100-1113 `tax.cps_carried`); cps_carried_current is reached from 19 via current_survey_predictors.py:479/:600 with no bound behind it.", + "asec_current_money_graph_resources.py defines no bound of its own; its `_parse(payload)` (:25) is an enforcement site of RESOURCE_MAX_BYTES on a 3,953 B pinned file, reached only from graph_asec_prepared.py:317.", + "The row-ceilings census (experiments/native-row-ceilings/census.json) lists SELECTED_PAYLOAD_MAX_BYTES count_at_full_source as '~52 MB'; the module's own encoder gives 50,643,470 B for 142,125 persons + 55,762 households. Its ~154.1 MB for the body matches 154,115,924 B here.", + "No bound in this family moved on this branch: `git diff origin/native-row-ceilings (0319b1c73)` is empty for all nine modules, so base_value == head_value throughout." + ] + }, + { + "family": "Geography and the graph package", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py (full, 639 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py (full, 320)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py (full, 277)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_geography.py (full, 395)", + "packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_geography.py (full, 384; no byte bound found)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py (full, 1141; no byte bound found)", + "packages/microcosm-build/src/microcosm/build/us_runtime/geography_ladder.py (full, 760; no byte bound found)", + "packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py (full, 276; no byte bound found)", + "packages/microcosm-graph/src/microcosm/graph/codecs.py (full, 611)", + "packages/microcosm-graph/src/microcosm/graph/store.py (full, 1451)", + "packages/microcosm-graph/src/microcosm/graph/executor.py (full, 3072)", + "packages/microcosm-graph/src/microcosm/graph/manifest.py (full, 1460)", + "packages/microcosm-graph/src/microcosm/graph/canonical.py (full, 106; no byte bound; canonical_json has no cap)", + "packages/microcosm-graph/src/microcosm/graph/serialize.py (full, 378; no byte bound; graph declaration is fixed-shape, 73,156 B at 1/1000)", + "packages/microcosm-graph/src/microcosm/graph/keys.py (full, 301)", + "supporting excerpts for enforcement chains and document shapes: survey_population_preparation.py:40-175,775-795 (_encode/_roster_segments); survey_origin_budget.py:1-60,180-260,440-470 (geography_contract attestation, reconstruct call); graph_atomic_survey_population.py:79-125,140-330; graph_atomic_survey_financial.py:1275-1440; survey_age_calibration.py:78-100,440-475; graph_survey_completion_host.py (def/call grep); graph_survey_population.py:292-375,881-925; current_asec_demographics.py:57-72,245-375; microcosm/graph/population.py:366-449,1656-1712 (expand_lineage_receipt); graph_combined_clone.py:226-266,612-657; microcosm/build/graph_atomic_geography.py:40-120; microcosm/build/atomic_geography.py:541-591; graph_context.py:104-178; puf_support.py:2324-2364 (clone id remap); atomic_block_sources.py:47-49, atomic_block_api_sources.py:30-32, puf55_survey_recipients.py:51-53 (_require types)", + "inputs: experiments/native-byte-transports/census-brief.md; experiments/native-byte-transports/byte_census_measure.py; experiments/native-byte-transports/byte-census-measurements.json (no geography document measured there); experiments/native-row-ceilings/census.json (6 rows for this family, stale on MAX_HOUSEHOLDS); docs/us-native-row-ceilings.md:100-140,205-235; docs/us-native-scale-transport.md (grep); _recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/{graph.json (node/kernel roster), preparation.json (id and key widths), manifest.json (receipt sizes and lineage row counts only)}; stat + zip central directory of _recovered/microcosm-native-sources-20260909/national-atomic-support.npz" + ], + "rows": [ + { + "constant": "packages/microcosm-graph/src/microcosm/graph/codecs.py:78 RAW_BYTES_MAX_BYTES", + "value": "64 * 1024 * 1024 = 67,108,864", + "enforced_at": [ + "codecs.py:571 `payload = handle.read(RAW_BYTES_MAX_BYTES + 1)` (read cap, no refusal)", + "codecs.py:576-580 `if len(payload) > RAW_BYTES_MAX_BYTES: raise ValueError(f\"Raw source at {source} is larger than the {RAW_BYTES_MAX_BYTES}-byte raw-bytes-v1 limit.\")` (message, no code literal; plain ValueError)", + "survey_atomic_geography.py:63 alias `MAX_SUPPORT_BYTES = RAW_BYTES_MAX_BYTES` enforced at :152 \"SUPPORT_SIZE\" ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_SIZE\") (own row below)", + "atomic_block_sources.py:365 `_require(len(payload) <= RAW_BYTES_MAX_BYTES, \"SUPPORT_BYTES\")` ValueError(\"ATOMIC_BLOCK_SOURCES_SUPPORT_BYTES\") (producer of the same support file; not on any runner path)", + "atomic_block_api_sources.py:264 `_require(len(payload) <= RAW_BYTES_MAX_BYTES, \"SUPPORT_BYTES\")` ValueError(\"ATOMIC_BLOCK_API_SOURCES_SUPPORT_BYTES\") (not on path)", + "puf55_survey_recipients.py:422 and :431 \"MATRIX_SIZE\" ValueError(\"PUF55_SURVEY_RECIPIENTS_MATRIX_SIZE\") (recipient matrix; reachable_from none per brief \u00a72)", + "call sites that reach codecs.py:571/:576: microcosm/build/graph_atomic_geography.py:72 `load_source_bytes(\"raw-bytes-v1\", ...)` (geography.support.0, on the 19/45/age paths); graph_geography.py:124-129 (us_puma_ladder_2020, us_cd_crosswalk_117_119; reachable_from none); registry binding graph_atomic_survey_population.py:199 `store.codecs.register_bytes(\"raw-bytes-v1\", load_raw_bytes)`", + "attested, not enforced: survey_origin_budget.py:213 folds geography.RAW_BYTES_MAX_BYTES into `geography_contract`; tests pin it at test_us_atomic_block_sources.py:243" + ], + "refusal_codes": [ + "(codecs) ValueError message \"Raw source at ... is larger than the 67108864-byte raw-bytes-v1 limit.\"", + "SUPPORT_SIZE", + "SUPPORT_BYTES", + "MATRIX_SIZE" + ], + "encodes": "an upstream file: one raw-bytes-v1 source read whole. On the 19/45/age paths the only raw-bytes source is `us_atomic_block_support` (graph.json sources: survey_population_source/us-survey-population-source-v1, us_atomic_block_support/raw-bytes-v1), the national 2020 block support NPZ; on the unreachable graph_geography path the PUMA-ladder NPZ and CD crosswalk CSV.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "28,862,508 (the support file's on-disk size; national block geography, identical at every fraction; stat verified this session)", + "bytes_at_full_source": "28,862,508 (unchanged; 43.0% of the cap)", + "measurement_basis": "ls -l of national-atomic-support.npz (sha256 pinned by the pilot configs; row-ceilings census.json measured the same 28,862,508); the file does not carry a survey roster (atomic_block_support.py:21-22 SYSTEM/SOURCE, :105-205 block arrays only)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "67,108,864 (origin/native-row-ceilings 0319b1c73; git diff empty)", + "head_value": "67,108,864", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1275) \u2192 :1355 atomic.run_atomic_survey_population \u2192 :231 run_graph \u2192 executor.py:2784 kernel.run for node geography.support.0 (kernel geography.support_import@1) \u2192 graph_atomic_geography.AtomicSupportImportKernel.run:72 load_source_bytes(\"raw-bytes-v1\") \u2192 codecs.load_source_bytes:602 \u2192 SourceCodecRegistry.load_bytes:198 \u2192 load_raw_bytes:571/:576. Same chain from 45 (run_atomic_survey_financial:1320 base run, then completion_host.extend) and age (survey_age_calibration.py:447 run_atomic_survey_population).", + "classification": "must-not-move", + "notes": "codecs.py:79 \"The most ``raw-bytes-v1`` will read from one source file (64 MiB).\"; :541-547 \"The read is bounded rather than trusted to ``st_size``, because a file may grow after it is measured ... this bound only keeps the codec from reading an unbounded amount first.\" Protects an upstream file's real size; the sha256 pin (survey_atomic_geography.py:162) is what asserts the file's content. Shared constant: moving it moves atomic_block_sources:365, atomic_block_api_sources:264 and the puf55 recipient-matrix cap at once. The recovered 1/1000 manifest shows geography.support.0's receipt at 1,190 B (support_sha256/areas/metadata), so the receipt side is fixed; only the artifact bytes (28.9 MB) are whole. No geography document is measured in byte-census-measurements.json." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:63 MAX_SUPPORT_BYTES", + "value": "RAW_BYTES_MAX_BYTES = 64 * 1024 * 1024 = 67,108,864 (alias of codecs.py:78)", + "enforced_at": [ + "survey_atomic_geography.py:151-154 `_require(stat.S_ISREG(before.st_mode) and 0 < before.st_size <= MAX_SUPPORT_BYTES, \"SUPPORT_SIZE\")` \u2192 ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_SIZE\") (_require at :68-70)", + "survey_atomic_geography.py:155 `payload = stream.read(before.st_size + 1)` read cap derived from the checked st_size; :158-164 `len(payload) == before.st_size and _sha(payload) == config.support_sha256` \"SUPPORT_CHANGED\" ValueError", + "survey_atomic_geography.py:420 first `_read_support(config)` and :594 second read `_read_support(config) == (support_payload, support_identity)` \"FINAL_SUPPORT\" (the file is read whole twice per reconstruct call)", + "attested, not enforced: survey_origin_budget.py:210 folds geography.MAX_SUPPORT_BYTES into `geography_contract`" + ], + "refusal_codes": [ + "SUPPORT_SIZE", + "SUPPORT_CHANGED", + "FINAL_SUPPORT" + ], + "encodes": "an upstream file: the national atomic block support NPZ named by AtomicSurveyReconstruction.support_path and pinned by support_sha256 (:81-84, :106)", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "28,862,508 (identical at every fraction)", + "bytes_at_full_source": "28,862,508 (43.0% of the cap)", + "measurement_basis": "stat of national-atomic-support.npz this session; national block geography carries no survey roster", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "67,108,864 (identical on origin/native-row-ceilings)", + "head_value": "67,108,864", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 \u2192 graph_atomic_survey_population.run_atomic_survey_population:169 reconstruction.reconstruct_atomic_survey_geography (again at :293, and a third call at graph_atomic_survey_financial.py:1408) \u2192 survey_atomic_geography.py:420/:594 _read_support \u2192 :152. Age: survey_age_calibration.py:447 (prefix run) and :457 (direct call). 45: same base run via run_atomic_survey_financial:1320.", + "classification": "must-not-move", + "notes": "Same number as codecs.py:78 by alias; recorded separately because it is a module-level definition with its own refusal literal. The row-ceilings verifier's correction stands at this head: the size ceiling is a bounded-read resource ceiling, the sha256 at :162 is the content claim. The _require helper is at :68-70 (census.json cited :67-69). Reconstruct is called three times per 19-node run and each call reads the file twice, so six whole 28.9 MB reads before the executor's own read at graph_atomic_geography.py:72." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:64 MAX_SUPPORT_EXPANDED_BYTES", + "value": "2 * 1024**3 = 2,147,483,648", + "enforced_at": [ + "survey_atomic_geography.py:173-178 `_require(0 < len(members) <= MAX_SUPPORT_MEMBERS and sum(member.file_size for member in members) <= MAX_SUPPORT_EXPANDED_BYTES, \"SUPPORT_EXPANDED_SIZE\")` \u2192 ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_EXPANDED_SIZE\")", + "attested, not enforced: survey_origin_budget.py:211" + ], + "refusal_codes": [ + "SUPPORT_EXPANDED_SIZE" + ], + "encodes": "an upstream file's declared decompressed size: the sum of ZipInfo.file_size over the support archive's members, checked before numpy opens any array (:168-169 comment)", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "1,061,673,840 (zip central directory: area 346,196,648 + tract 253,877,576 + puma 161,558,504 + county 115,398,968 + district 92,319,200 + population 46,159,664 + state 46,159,664 + metadata_json 3,616; identical at every fraction)", + "bytes_at_full_source": "1,061,673,840 (49.4% of the cap)", + "measurement_basis": "python zipfile.infolist() over the pinned support archive this session; agrees with the row-ceilings census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "2,147,483,648 (identical on origin/native-row-ceilings)", + "head_value": "2,147,483,648", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 \u2192 run_atomic_survey_population:169/:293 (and financial:1408) \u2192 reconstruct_atomic_survey_geography:421 _decode_support \u2192 :173-178. Also age (:447/:457) and 45.", + "classification": "must-not-move", + "notes": "Tightest relative headroom in the family (2.02\u00d7), but the quantity is national block geography (5.77M blocks at U15 = 60 B/row for area.npy) and cannot move with the survey fraction. At the archive's 36.78\u00d7 expansion ratio this ceiling would refuse at about 55.7 MiB compressed, before MAX_SUPPORT_BYTES does." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:65 MAX_SUPPORT_MEMBERS", + "value": "32", + "enforced_at": [ + "survey_atomic_geography.py:173-178 (same _require as MAX_SUPPORT_EXPANDED_BYTES) \"SUPPORT_EXPANDED_SIZE\" ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_EXPANDED_SIZE\")", + "attested, not enforced: survey_origin_budget.py:212" + ], + "refusal_codes": [ + "SUPPORT_EXPANDED_SIZE" + ], + "encodes": "a record cap on the upstream archive: the number of ZIP members the support NPZ declares (a fixed eight-array schema: area, county, district, metadata_json, population, puma, state, tract)", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "8 members (not bytes; identical at every fraction)", + "bytes_at_full_source": "8 members", + "measurement_basis": "zip central directory of the pinned archive this session", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "32 (identical on origin/native-row-ceilings)", + "head_value": "32", + "reachable_from": "19", + "reach_chain": "same as MAX_SUPPORT_EXPANDED_BYTES: reconstruct_atomic_survey_geography:421 _decode_support \u2192 :173-178 (19 via financial:1355; age via :447/:457; 45 via financial:1320).", + "classification": "must-not-move", + "notes": "A member count, not a byte count; listed because the brief counts record caps and it shares the byte bound's refusal. It bounds the per-member NPY header loop at :179-196. 4\u00d7 headroom over a fixed schema." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:190-195 SUPPORT_ARRAY_SIZE (math.prod(shape) * dtype.itemsize == member.file_size - stream.tell())", + "value": "an equality, no fixed integer: declared array bytes must equal the physical member payload length (per member); plus :181-187 NPY version 1.0/2.0 only (\"SUPPORT_NPY_VERSION\")", + "enforced_at": [ + "survey_atomic_geography.py:190-195 `_require(not dtype.hasobject and all(type(size) is int and size >= 0 for size in shape) and math.prod(shape) * dtype.itemsize == member.file_size - stream.tell(), \"SUPPORT_ARRAY_SIZE\")` \u2192 ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_ARRAY_SIZE\")", + "survey_atomic_geography.py:187 `raise ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_NPY_VERSION\")`" + ], + "refusal_codes": [ + "SUPPORT_ARRAY_SIZE", + "SUPPORT_NPY_VERSION" + ], + "encodes": "a struct-width product on the upstream file: each NPY member's header-declared shape \u00d7 itemsize against its physical bytes (:188-189 comment: \"A small ZIP member must not claim a huge allocation in its NPY header\")", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "largest member area.npy 346,196,648 bytes = 5,769,944 blocks \u00d7 U15 (60 B); identical at every fraction", + "bytes_at_full_source": "346,196,648 (unchanged)", + "measurement_basis": "zip central directory this session; row-ceilings verifier's area.npy row width", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "unchanged (identical on origin/native-row-ceilings)", + "head_value": "unchanged", + "reachable_from": "19", + "reach_chain": "reconstruct_atomic_survey_geography:421 _decode_support \u2192 :179-196 per member (19 via financial:1355 \u2192 run_atomic_survey_population:169; age via survey_age_calibration:447/:457; 45 via financial:1320).", + "classification": "must-not-move", + "notes": "Not a ceiling but an exact-length check that refuses a header over-claim; recorded because the brief lists struct-width products. The shared decoder atomic.decode_atomic_support (:197) owns every schema/value check after it." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:98 0 < len(config.support_path) <= 4096", + "value": "4096 characters", + "enforced_at": [ + "survey_atomic_geography.py:96-103 `_require(type(config.support_path) is str and 0 < len(config.support_path) <= 4096 and \"\\0\" not in ... and Path(...).is_absolute() and \"..\" not in parts, \"SUPPORT_PATH\")` \u2192 ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_PATH\"); called from __post_init__ :87, to_bytes :91, reconstruct :407 and :602" + ], + "refusal_codes": [ + "SUPPORT_PATH" + ], + "encodes": "a single scalar: the support file path literal carried into the config document (`canonical_json({... \"support_path\": ...})` :125-133)", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "the pilot path length (tens of characters); does not move with the fraction", + "bytes_at_full_source": "same", + "measurement_basis": "scalar length check; no roster behind it", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4096 (identical on origin/native-row-ceilings)", + "head_value": "4096", + "reachable_from": "19", + "reach_chain": "AtomicSurveyReconstruction(...) __post_init__ :86-87 \u2192 _config_bytes :94; reconstruct_atomic_survey_geography:407 (19 via financial:1355 \u2192 atomic:169; age :447/:457 and :442-446 to_bytes; 45 via financial:1320).", + "classification": "must-not-move", + "notes": "Protects a token/scalar. The seed range check at :124 (`0 <= config.seed < 2**63`, \"SEED\") is a value range, not a byte bound, and is not a row." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:116 0 < len(pair[1]) <= 4096", + "value": "4096 characters per source identity string (three strings: district, population, puma)", + "enforced_at": [ + "survey_atomic_geography.py:109-123 `_require(type(config.source_ids) is tuple and len(config.source_ids) == 3 and all(... 0 < len(pair[1]) <= 4096 and pair[1].strip() == pair[1] ...) and tuple(pair[0] ...) == (\"district\", \"population\", \"puma\"), \"SOURCE_IDENTITIES\")` \u2192 ValueError(\"ATOMIC_SURVEY_RECONSTRUCTION_SOURCE_IDENTITIES\")" + ], + "refusal_codes": [ + "SOURCE_IDENTITIES" + ], + "encodes": "three scalars: the publisher source identities written into the config document (:130) and the assignment definition (:427 `source_ids=dict(config.source_ids)` \u2192 atomic_block_support.assignment_definition:223 `_sources`)", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "three short identity strings; does not move with the fraction", + "bytes_at_full_source": "same", + "measurement_basis": "scalar length check", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4096 (identical on origin/native-row-ceilings)", + "head_value": "4096", + "reachable_from": "19", + "reach_chain": "same as :98 (_config_bytes from __post_init__ and reconstruct:407/:602).", + "classification": "must-not-move", + "notes": "Protects a token/scalar. atomic_block_support._sources (:93-102) re-checks non-emptiness only, no length." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:263 len(part).to_bytes(8, \"little\")", + "value": "8-byte little-endian length prefix: 2**64 - 1 = 18,446,744,073,709,551,615 per storage part", + "enforced_at": [ + "survey_atomic_geography.py:262-264 in _population_stamp: `for part in _storage_parts(series, slice(None)): digest.update(len(part).to_bytes(8, \"little\")); digest.update(part)` (OverflowError from int.to_bytes if exceeded; no refusal code)", + ":265-267 `digest.update(canonical_json((entity, str(values.dtype), values.shape))); digest.update(values.tobytes())` for design weights (no length prefix; tobytes() copies the array)" + ], + "refusal_codes": [ + "(none; OverflowError from int.to_bytes)" + ], + "encodes": "an encoder width in a streaming seal: every column's raw storage parts of the population frame at each stage (stacked rows for create/allocate/observed_geography, clone rows for expand/claim/assign/derive/gate), plus strata and design weights, streamed into one sha256", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "largest single part at 1/10: an 8-byte column at 713,002 clone persons = 5,704,016 bytes (the width admits 2**64-1)", + "bytes_at_full_source": "largest single part: an 8-byte column at 7,130,026 clone persons = 57,040,208 bytes; design-weight tobytes() copy at 1,587,376 households = 12,699,008 bytes", + "measurement_basis": "rows \u00d7 dtype width from brief \u00a73 counts; no whole copy of the frame is made (parts are the live buffers, only tobytes() at :267 copies)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "8 bytes (identical on origin/native-row-ceilings)", + "head_value": "8 bytes", + "reachable_from": "19", + "reach_chain": "_population_stamp is called at reconstruct:418, :542-ish via stages (:562-564, :585, :590, :613-616) and by every runner: graph_atomic_survey_population.py:215/:218/:229/:320/:329-330, graph_atomic_survey_financial.py:152/:1379/:1645/:1809/:1928/:1943, survey_age_calibration via atomic_source, graph_survey_completion_host:255/:511/:514.", + "classification": "must-not-move", + "notes": "No accumulation: digest.update per part. The docstring (:231-238) says the seal folds physical storage parts and is never persisted. Recorded as a width row because the brief lists struct widths; it cannot bind at any survey scale." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py:542 canonical_json(receipt) (stage receipt; the EXPAND stage's carries receipt[\"expand\"] = expand_lineage_receipt(lineage) from :342)", + "value": "none (canonical_json has no cap: canonical.py:43-58 json.dumps whole)", + "enforced_at": [ + "survey_atomic_geography.py:540-544 `stages.append(AtomicSurveyGeographyStage(node, current, canonical_json(receipt), artifacts))` for every ordered node; the EXPAND stage's receipt is built at :339-350 by _clone_expectations: `receipt = clone.USCombinedSurveyCloneExpandKernel._receipt(...)`; :342 `receipt[\"expand\"] = expand_lineage_receipt(lineage)`; :344-350 `receipt[\"expand_writes\"] = expand_writes_receipt(...)`", + "consumer decode of the same bytes: graph_atomic_survey_population.py:212 `{stage.node.id: json.loads(stage.receipt) for stage in geography.stages}` \u2192 _states:105 `dict(raw_receipts[node_id])` \u2192 graph_survey_population._check_node_states:922 `node.receipt == state[\"receipt\"]` (dict equality, no re-encode)", + "refusal on encode: canonical.py:25 ValueError(\"non-finite numbers are not canonical JSON\"), :33 TypeError(\"canonical mappings require string keys\"), :38-40 TypeError (unsupported type) \u2014 none size-based" + ], + "refusal_codes": [ + "(none; no cap)" + ], + "encodes": "a per-row roster document: the combined-clone EXPAND receipt whose `expand` key lists one [new_id, copied_id] pair per row of every entity of the expanded frame (population.py:424-449 `payload[entity] = [[target, source] ...]`, entities from survey_atomic_geography:332-335 `for entity in US_SCHEMA.entities`): clone households, clone persons, clone tax units, clone families, clone marital units, clone SPM units; plus the fixed clone facts (:626-657)", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "45,052,091 bytes for the lineage part at 1/10 widths (2,650,123 pairs \u00d7 17 B: households 317,475 + persons 713,003 + tax units 426,507 SCALED + families 319,480 SCALED + marital units 555,982 SCALED + SPM units 317,676 SCALED)", + "bytes_at_full_source": "503,523,275 bytes for the lineage part (26,501,225 pairs \u00d7 19 B at 8-digit clone ids over 7-digit native ids: households 3,174,752 + persons 7,130,026 + tax units 4,265,071 SCALED + families 3,194,795 SCALED + marital units 5,559,825 SCALED + SPM units 3,176,756 SCALED); the fixed remainder is under 5 KB", + "measurement_basis": "scratchpad geo_byte_census_measure.py M4: canonical_json on invented pairs, difference of 1,000 vs 2,000 rows = 19.0 B/pair (8/7 digits), 17.0 (7/6 digits, 1/10), 15.0 at the 1/1000 observed widths; corroborated by the recovered 1/1000 manifest: combined_survey_puf_support_clone receipt 198,514 B holding 13,129 pairs (family 1594, household 1584, marital_unit 2774, person 3464, spm_unit 1585, tax_unit 2128), sample [[1000001, 1], ...]; clone id width from puf_support.py:2324-2345 (id + clone_index * 10**digits(max_id))", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (identical on origin/native-row-ceilings)", + "head_value": "none", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 \u2192 run_atomic_survey_population:169 (and :293; financial:1408) \u2192 reconstruct_atomic_survey_geography \u2192 :483 _clone_expectations \u2192 :342 expand_lineage_receipt \u2192 :542 canonical_json(receipt) \u2192 graph_atomic_survey_population:212 json.loads(stage.receipt). Age: survey_age_calibration:447/:457. 45: financial:1320.", + "classification": "stays", + "notes": "No bound to meet, so it cannot bind; but it is a whole-roster canonical document held whole (about 0.50 GB at full source) three times per 19-node run in this module alone, then decoded whole again at graph_atomic_survey_population:212. The document is another family's (graph_combined_clone._receipt + population.expand_lineage_receipt); this row is the consumer-side re-encoding the brief asks for, with the encoder's cap recorded as none. The same document is also materialised by store.put_json and RunManifest (rows below) and by graph_combined_clone's own lineage digest (:217 canonical_json(payload)) and survey_age_calibration._node_identity:80-98 (includes node.receipt) \u2014 both outside this family. Whether an uncapped materialisation of this size should take the segmented shape is an open question for the lane, not a refusal at any fraction." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/store.py:1099 json.loads(_canonical_json(dict(payload))) in ContentStore.put_json (with :1102 _write_json \u2192 :121 _canonical_json again)", + "value": "none (_canonical_json :99-111 is json.dumps whole; put_json/put_bytes/put_frame carry no byte cap)", + "enforced_at": [ + "store.py:1097-1105 put_json: `normalized = json.loads(_canonical_json(dict(payload)))` then `_write_json(root / \"payload.json\", normalized)` \u2192 :120-121 `_write_bytes(path, _canonical_json(value))`; refusal only :108-111 TypeError(\"Store metadata must be finite, JSON-compatible data.\")", + "executor.py:1681-1708 _write_node builds `record = {... \"receipt\": dict(receipt), ...}` and :1703 `store.put_json(_cache_record_key(key), record, node_key=key, ...)` \u2014 for the EXPAND node the receipt carries the lineage appended at executor.py:1355 `receipt[\"expand\"] = expand_lineage_receipt(result.expand)` and :2910 `normalized_receipt[\"expand_writes\"]`", + "cache-hit read of the same bytes: executor.py:1860 _load_record \u2192 store.load_json:1107-1115 \u2192 _load_json_file:212-225 json.loads whole (no cap); executor.py:2323 `record[\"receipt\"] != receipt` (dict compare)", + "store.py:141-153 _payload_table records sha256 + size per file and :228-292 _verified_meta re-hashes every payload on each read (1 MiB chunks at :136, streaming)" + ], + "refusal_codes": [ + "(none size-based; TypeError \"Store metadata must be finite, JSON-compatible data.\")" + ], + "encodes": "every node's cache record (fixed for 18 of the 19 nodes; the recovered 1/1000 receipts are 471 B to 29,387 B) and, for combined_survey_puf_support_clone, the per-row EXPAND lineage roster (clone households, persons, tax units, families, marital units, SPM units) \u2014 the same document as the survey_atomic_geography:542 row", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "45,052,091 (lineage part; held as one bytes object twice: the normalisation at :1099 and the write at :1102)", + "bytes_at_full_source": "503,523,275 (lineage part), plus the same again on every cache-hit load", + "measurement_basis": "same canonical encoder and pair widths as the survey_atomic_geography:542 row (store._canonical_json uses the identical json.dumps parameters: sort_keys, separators (\",\", \":\"), ensure_ascii=False, allow_nan=False); 1/1000 manifest receipt 198,514 B for 13,129 pairs", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (identical on origin/native-row-ceilings)", + "head_value": "none", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 \u2192 run_atomic_survey_population:231 run_graph \u2192 executor._execute_graph:2784 kernel.run (combined_survey_puf_support_clone) \u2192 :1355 lineage appended in _validate_result \u2192 :2943 _write_node \u2192 :1703 store.put_json \u2192 store.py:1099/:1102. Same for age (survey_age_calibration:447) and 45 (financial:1320).", + "classification": "stays", + "notes": "docs/us-native-scale-transport.md:79 already records that put_bytes and put_frame carry no byte cap; put_json is the same. The store's own integrity table (:141-153) is a per-file sha256/size, not a bound. Executor.py:111 `_normal_json_mapping` (`json.loads(canonical_json(value))`) re-encodes the kernel's receipt at :331 before the lineage is appended at :1355, so on this path that earlier re-encode is fixed-shape (largest kernel receipt at 1/1000: survey_predictors.attach 29,387 B); see open questions for the asec_design_donor `selection` key." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/manifest.py:661 canonical_json(self.content_addressed) (RunManifest.key), with :731 canonical_json(payload) in to_json and :751 json.loads in from_json", + "value": "none (no cap anywhere in manifest.py; canonical_json and json.loads whole)", + "enforced_at": [ + "manifest.py:659-661 `key` property: `sha256_domain(\"manifest\", canonical_json(self.content_addressed))` \u2014 content_addressed (:578-591) holds every node's `_content_payload()` including `receipt` (:387, :417-428)", + "manifest.py:709-731 to_json: payload carries `\"key\": self.key` (:718, one full encode), `\"content_addressed\": self.content_addressed` (:721) and `\"nodes\": {... receipt._payload()}` (:723-725), then :731 `canonical_json(payload)` \u2014 the lineage roster appears twice inside one document and a third time for the key", + "manifest.py:744-830 from_json: :751 json.loads whole; :805-807 `recomputed_body = manifest.content_addressed` and `sha256_domain(\"manifest\", canonical_json(body))`; :976-978 `canonical_json(body_nodes[node_id]) != canonical_json(expected_nodes[node_id])` per node (re-encodes each receipt twice more); :833-893 load reads the file whole (:838) and :851-856 re-canonicalises raw_body", + "manifest.py:277 `_freeze_json(self.receipt)` in NodeReceipt.__post_init__ copies the receipt into MappingProxyType/tuples (in-memory copy of the roster, not bytes)", + "refusals: canonical.py:25/:33/:38-40 (type/finite only); :114 ValueError(\"manifest values must be finite\"), :120/:125 TypeError \u2014 none size-based" + ], + "refusal_codes": [ + "(none; no cap)" + ], + "encodes": "the run manifest: every node receipt, including the combined_survey_puf_support_clone EXPAND receipt with its per-row lineage roster (clone households, persons, tax units, families, marital units, SPM units)", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "45,052,091 per copy of the lineage part; to_json holds it twice in one document plus one more encode for key = about 135 MB of lineage bytes in flight, plus the str\u2192bytes copy at to_json_bytes:734", + "bytes_at_full_source": "503,523,275 per copy; to_json_bytes \u2248 1.01 GB of lineage in the document string plus its UTF-8 copy, plus the key encode", + "measurement_basis": "same canonical_json pair cost (19.0 B at full widths, 17.0 at 1/10) \u00d7 the \u00a73 clone counts (tax unit/family/marital/SPM SCALED); recovered 1/1000 manifest.json is 950,156 B of which the clone receipt is 198,514 B, consistent with 2\u00d7 (nodes + content_addressed) plus fixed nodes", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (identical on origin/native-row-ceilings)", + "head_value": "none", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1374 `prefix_manifest_bytes = prefix.manifest.to_json_bytes()` \u2192 manifest.py:733-734 \u2192 to_json:709-731 \u2192 key:661 (19 and 45 through the same base run). The age runner reaches manifest.py only through RunManifest/NodeReceipt construction (executor.py:3054, _freeze_json copy) and node.key; it does not call key/to_json (survey_age_calibration greps: node identities only, :78-98, :275, :546, :713, :723).", + "classification": "stays", + "notes": "RunManifest docstring (:435-445): the content-addressed body is every complete node receipt less run-level fields. Because the executor authors the lineage into the receipt (executor.py:1355) rather than into an artifact, the portable manifest and its key are roster-shaped by construction; a manifest saved with RunManifest.save (:736-741) writes the whole string. RunManifest.load (:833) re-reads it whole and from_json re-canonicalises each node twice (:976-978). No refusal at any fraction; memory only." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/executor.py:111 json.loads(canonical_json(value)) in _normal_json_mapping", + "value": "none (canonical_json whole; no cap)", + "enforced_at": [ + "executor.py:107-118 `_normal_json_mapping(value, label)`: `restored = json.loads(canonical_json(value))`; refusal :113-115 NodeRejected(f\"{label} must contain canonical JSON values: {error}\") on TypeError/ValueError only", + "executor.py:331 `receipt = _normal_json_mapping(result.receipt, f\"Node {node.id!r} receipt\")` in _validate_result, for every executed node's kernel-authored receipt (before :1355 appends the EXPAND lineage)" + ], + "refusal_codes": [ + "(none size-based; NodeRejected \"... must contain canonical JSON values: ...\")" + ], + "encodes": "every kernel-authored receipt on the path, re-encoded to detach it. On the 19-node path these are fixed-shape at the executor's re-encode point: the recovered 1/1000 kernel receipts are 471 B (create) to 29,387 B (survey_predictors.attach); the clone kernel's own receipt (graph_combined_clone._receipt :612-657: counts, digests, provenance counts) is fixed, the roster lineage is appended only afterwards at :1355", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "under 30 KB per receipt for the geography kernels (observed_geography 2,071 B; support 1,190; assign 761; derive 1,519; gate 1,307 at 1/1000; only integer counts and digests change)", + "bytes_at_full_source": "same order (digits change only)", + "measurement_basis": "recovered 1/1000 manifest.json receipt sizes; receipt shapes read at graph_current_survey_geography.py:226-233, graph_atomic_geography.py:80-84, survey_atomic_geography.py:497-528", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (identical on origin/native-row-ceilings)", + "head_value": "none", + "reachable_from": "19", + "reach_chain": "run_graph \u2192 _execute_graph:2830 _validate_result \u2192 :331 _normal_json_mapping for every node (19, 45, age all pass through run_graph: graph_atomic_survey_population:231, graph_atomic_survey_financial:1647, graph_survey_completion_host:786, survey_age_calibration via atomic_source:447).", + "classification": "stays", + "notes": "Recorded as the family's consumer-side re-encoding of kernel receipts. It is fixed-shape for every geography receipt; the only roster-shaped receipt on the path (the EXPAND lineage) is authored after this point. One receipt outside this family may be roster-shaped at this site: survey_predictors.asec_design_donor carries a `selection` key (12,876 B at 1/1000) \u2014 see open questions." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/executor.py:436 len(payload).to_bytes(8, \"little\") (with :457-458 np.full(rows, width, dtype=\" len(relative).to_bytes(8, \"little\") and :48 len(content).to_bytes(8, \"little\") in _directory_identity", + "value": "8-byte little-endian length prefixes: 2**64 - 1 per member name and per member content", + "enforced_at": [ + "keys.py:31-51 _directory_identity: `content = candidate.read_bytes()` (:45, whole file, no cap) then :46-49 prefix + name, prefix + content into sha256; OverflowError only", + "keys.py:54-71 source_content_key: regular file `content = source_path.read_bytes()` (:64, whole, no cap) \u2192 sha256; directory \u2192 _directory_identity", + "executor.py:2100-2104 _source_paths_and_keys \u2192 source_content_key/identities.key; :2824 per-node recheck for nodes declaring sources; :3030-3031 run-end `source_identities.derive` (cache bypassed) and :2511 _settle_failed_run" + ], + "refusal_codes": [ + "(none; OverflowError from int.to_bytes)" + ], + "encodes": "an encoder width over upstream files: the raw-bytes source `us_atomic_block_support` (28,862,508 B, read whole at :64) and every member of the `survey_population_source` directory (the ACS zips, ASEC h5/csv files listed at survey_population_preparation.py:62-73, each read whole at :45)", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "28,862,508 B for the support file (one whole read_bytes per derivation); the survey source directory members are the input archives' own sizes, identical at every fraction (not measured here; the coverage lane measured the uncompressed person archive at 2,405,464,349 B)", + "bytes_at_full_source": "same (upstream sizes do not move with the fraction)", + "measurement_basis": "stat of the support file; keys.py:45/:64 read_bytes has no cap; brief \u00a74 forbids reading the gated survey archive here", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "8 bytes (identical on origin/native-row-ceilings)", + "head_value": "8 bytes", + "reachable_from": "19", + "reach_chain": "run_graph \u2192 _execute_graph:2638 _source_paths_and_keys \u2192 keys.source_content_key:54 (run start), :2824 (after each source-declaring node: survey_population.create, geography.support.0), :3031 run end (19: graph_atomic_survey_population:221 also calls _source_paths_and_keys directly; 45 and age through the same runs).", + "classification": "must-not-move", + "notes": "The width never binds. What a reviewer should see is that source identity reads each upstream member whole with no cap, at least three times per run (start, per-node, end) \u2014 an upstream materialisation rather than a roster transport; the sizes are the input files' own." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/store.py:68 _KEY = re.compile(r\"[0-9a-f]{64}\\Z\")", + "value": "exactly 64 lowercase hex characters (a fixed-width field)", + "enforced_at": [ + "store.py:193-196 _require_key \u2192 ValueError(\"Content-store keys must be 64 lowercase hexadecimal digits.\") (object_path :706, evict :770, _put :800)", + "store.py:243-244 _verified_meta `metadata.get(\"key\") != path.name or _KEY.fullmatch(path.name) is None` \u2192 StoreCorrupt(\"Stored object ... does not bind its directory key.\")", + "store.py:261-262 payload table `_KEY.fullmatch(expected_hash) is None` \u2192 StoreCorrupt(f\"Payload {name!r} has an invalid SHA-256.\")" + ], + "refusal_codes": [ + "\"Content-store keys must be 64 lowercase hexadecimal digits.\" (ValueError)", + "\"... does not bind its directory key.\" (StoreCorrupt)", + "\"Payload ... has an invalid SHA-256.\" (StoreCorrupt)" + ], + "encodes": "a single token: every content-store object key and payload digest (sha256 hexdigests from keys.py)", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 characters, always", + "bytes_at_full_source": "64 characters", + "measurement_basis": "regex width", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 hex (identical on origin/native-row-ceilings)", + "head_value": "64 hex", + "reachable_from": "19", + "reach_chain": "every store put/load from executor._write_node (:1614/:1628/:1656/:1673/:1703) and _load_record/_load_cached_result (19, 45, age all through run_graph).", + "classification": "must-not-move", + "notes": "Protects a fixed-width field; listed for completeness." + }, + { + "constant": "packages/microcosm-graph/src/microcosm/graph/store.py:338 offsets = np.empty(len(chunks) + 1, dtype=np.int64) in _encode_offsets (utf8-offsets-v1 / object-scalars-v1)", + "value": "int64 offsets: 2**63 - 1 = 9,223,372,036,854,775,807 bytes of payload per string/object column", + "enforced_at": [ + "store.py:337-344 _encode_offsets: `payload = bytearray(); for chunk in chunks: payload.extend(chunk); offsets[index + 1] = len(payload)` then `np.frombuffer(bytes(payload), dtype=np.uint8).copy()` (the whole column payload is built twice: bytearray, then bytes, then a numpy copy)", + "store.py:461-480 _write_series StringDtype branch: `chunks.append(value.encode(\"utf-8\"))` per row, `_encode_offsets(chunks)`, values.npy/offsets.npy/mask.npy", + "store.py:481-486 object branch `_encode_object_scalar` per value \u2192 _encode_offsets", + "read side store.py:415-431 _validate_offsets requires dtype int64, monotone, last == payload length \u2192 StoreCorrupt(\"Stored offsets for ... are malformed.\"); :532-559 decodes each string from `raw = payload.tobytes()`" + ], + "refusal_codes": [ + "(no size refusal; StoreCorrupt \"Stored offsets for ... are malformed.\" on read shape only)" + ], + "encodes": "a numpy body per string column: put_column (executor.py:1614) writes survey_population.observed_geography's three string columns at stacked-household rows (survey_geography_origin_key, survey_observed_state, survey_observed_puma) and geography.assign/derive's string columns; put_frame (executor.py:1628, store.py:1194-1250 _write_frame) writes every string column of a structural version at clone rows (combined_survey_puf_support_clone) or filtered rows (survey_predictors.asec_design_donor)", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "survey_geography_origin_key column body 6,627,930 B at 158,737 stacked households (values + int64 offsets + mask), 13,255,852 B at 317,475 clone households", + "bytes_at_full_source": "66,279,466 B at 1,587,376 stacked households (ACS 33-byte keys \u00d7 1,531,614 + ASEC 26-byte keys \u00d7 55,762, + 8 B offset + 1 B mask per row); 132,558,924 B at 3,174,752 clone households; the width admits 2**63 - 1", + "measurement_basis": "scratchpad geo_byte_census_measure.py M5: store._encode_offsets on invented keys at the 1/1000-observed key widths (13-char SERIALNO, 5-char H_SEQ), difference of 1,000 vs 2,000 rows = 42.0 B/ACS row, 35.0 B/ASEC row", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "int64 (identical on origin/native-row-ceilings)", + "head_value": "int64", + "reachable_from": "19", + "reach_chain": "run_graph \u2192 _execute_graph:2943 _write_node \u2192 :1614 store.put_column (observed_geography, assign, derive) and :1628 store.put_frame (clone EXPAND, donor FILTER) \u2192 store.py:920/:1207 _write_series \u2192 :471/:483 _encode_offsets (19: graph_atomic_survey_population:231; 45 and age likewise).", + "classification": "must-not-move", + "notes": "Protects an encoder width (docstring :8-12: strings use one UTF-8 byte buffer plus int64 offsets and a null mask). No cap in front of it; put_frame at clone rows materialises each string column whole (bytearray + bytes + numpy copy). The largest string column on the path is the origin key measured here; other string columns (state 2 B, puma 7 B, provenance channel/source ids) are narrower." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:36 MAX_HOUSEHOLDS", + "value": "7_000_000 = 7,000,000 (a row count; comment :29-35 \"Four times [1,587,376], rounded up to the next whole million ... there is no byte transport behind this bound and it is an explicit row ceiling\")", + "enforced_at": [ + "current_survey_geography.py:73-78 `_require(type(origins) is list and 0 < len(origins) <= MAX_HOUSEHOLDS and len(origins) == len(households), \"HOUSEHOLD_COUNT\")` \u2192 ValueError(\"CURRENT_SURVEY_GEOGRAPHY_HOUSEHOLD_COUNT\") (_require :40-42)", + "current_survey_geography.py:190-198 `_require(... and 0 < len(household) <= MAX_HOUSEHOLDS and ..., \"PROJECTION_STORAGE\")` \u2192 ValueError(\"CURRENT_SURVEY_GEOGRAPHY_PROJECTION_STORAGE\")", + "attested, not enforced: survey_origin_budget.py:222 folds geography.observed.MAX_HOUSEHOLDS into `geography_contract`" + ], + "refusal_codes": [ + "HOUSEHOLD_COUNT", + "PROJECTION_STORAGE" + ], + "encodes": "a row count of the selected/stacked household roster (origins[\"households\"] from the preparation payload :235 and state.frame.table(\"household\") :246), pre-clone", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "n/a \u2014 row count: 158,737 selected households (2.3% of the ceiling)", + "bytes_at_full_source": "n/a \u2014 row count: 1,587,376 selected households (22.7%; headroom 4.41\u00d7)", + "measurement_basis": "brief \u00a73 counts; docs/us-native-row-ceilings.md:112 rule (4\u00d7 full source rounded up to the next million) and :228 (streams verdict)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "7,000,000 on origin/native-row-ceilings 0319b1c73 (git show confirms); experiments/native-row-ceilings/census.json still records the pre-lift 64 * 1024**2 // 128 = 524,288 and binds_at_full_source true \u2014 stale, superseded by its own verifier verdict", + "head_value": "7,000,000", + "reachable_from": "19", + "reach_chain": "reconstruct_atomic_survey_geography:431 observed.qualify_current_survey_geography \u2192 :244 _project (:75) and :261 _projection_digest (:195), again at :309/:317; and CurrentSurveyGeographyKernel.run:217 inside run_graph (registered graph_atomic_survey_population:200) \u2192 qualify \u2192 same sites; _check_projection:149 and run:274 call _projection_digest again. 19 via financial:1355 \u2192 atomic:169; age via :447/:457; 45 via financial:1320.", + "classification": "not-a-transport", + "notes": "A row count with no byte name; the row-count rule at docs/us-native-row-ceilings.md:105-116 and :214-228 applies (\"streams \u2014 moved in \u00a73\"). Listed because the brief counts record caps and because the row-ceilings census row for it is stale at this head." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:37 MAX_RECEIPT_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "current_survey_geography.py:262-290 `receipt = source._encode({...}, maximum=MAX_RECEIPT_BYTES)` \u2192 survey_population_preparation.py:116-122 _encode: `_require(len(result) + len(encoded) <= maximum, \"PAYLOAD_LIMIT\")` \u2192 SurveyPopulationPreparationError(\"PAYLOAD_LIMIT\") (ValueError subclass, :78; _require :85-87)", + "graph_current_survey_geography.py:133-141 _check_projection: `_require(... and type(projection.receipt) is bytes and 0 < len(projection.receipt) <= qualifier.MAX_RECEIPT_BYTES and source_graph._sha(projection.receipt) == receipt_sha256, \"PROJECTION_RECEIPT\")` \u2192 ValueError(\"SURVEY_GEOGRAPHY_PROJECTION_PROJECTION_RECEIPT\") (_require :41-43)", + "attested, not enforced: survey_origin_budget.py:223" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT", + "PROJECTION_RECEIPT" + ], + "encodes": "a fixed-shape receipt: 26 keys of protocol strings, four 64-hex digests, the three column names and five integer counts (households, acs_households, asec_households, state_unknown_households, puma_observed_households); no per-household value (:263-288)", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "1,223 bytes (through source._encode at 1/10 counts)", + "bytes_at_full_source": "1,228 bytes (1.87% of the cap; headroom 53.4\u00d7); 1,213 at 1/1000", + "measurement_basis": "scratchpad geo_byte_census_measure.py M1: the exact key roster of :263-288 encoded by source._encode(doc, maximum=MAX_RECEIPT_BYTES) at the \u00a73 counts; only integer digits move", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536 (identical on origin/native-row-ceilings)", + "head_value": "65,536", + "reachable_from": "19", + "reach_chain": "reconstruct_atomic_survey_geography:431 qualify_current_survey_geography:262 (encode) and :433/:619 observed_graph._check_projection:138 (len check); CurrentSurveyGeographyKernel.run:217/:218/:259 inside run_graph. 19 via financial:1355 \u2192 atomic:169/:293 and financial:1408; age via survey_age_calibration:447/:457; 45 via financial:1320.", + "classification": "stays", + "notes": "Fixed-shape, so it cannot bind at any survey scale; the consumer-side check at graph_current_survey_geography:138 is a plain len() on the issued bytes, not a re-encode. The kernel's own receipt (:226-234, canonical_json at :234 and :267 `canonical_json(result.receipt) == receipt_bytes`) embeds this document as `source_projection` and is likewise fixed (2,071 B at 1/1000); that :267 re-encode is not a bound and is not a row." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:60 0 < len(raw) <= 128 (NATIVE_KEY)", + "value": "128 characters of raw_native_id per origin row", + "enforced_at": [ + "current_survey_geography.py:59-60 `raw = row[\"raw_native_id\"]; _require(type(raw) is str and 0 < len(raw) <= 128, \"NATIVE_KEY\")` \u2192 ValueError(\"CURRENT_SURVEY_GEOGRAPHY_NATIVE_KEY\"), in _origin_key called per origin at :121 (_project), i.e. once per selected household per qualify" + ], + "refusal_codes": [ + "NATIVE_KEY" + ], + "encodes": "a single token per row: the source literal id inside the draw key `json.dumps([channel, source_year, survey_year, raw])` (:63-68); observed widths 13 (ACS SERIALNO) and 4-5 (ASEC H_SEQ) at 1/1000", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13 characters max observed (ACS); does not move with the fraction (ACS SERIALNO is a fixed 13-char literal per acs_household_archive serialno_widths {13: 1,631,969}; ASEC H_SEQ grows to at most 6 digits)", + "bytes_at_full_source": "13 (ACS), \u22646 (ASEC)", + "measurement_basis": "preparation.json raw_native_id lengths (acs 13/13, asec 4/5) and byte-census-measurements.json acs_household_archive.serialno_widths", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "128 (identical on origin/native-row-ceilings)", + "head_value": "128", + "reachable_from": "19", + "reach_chain": "reconstruct:431 qualify_current_survey_geography:244/:309 _project:121 _origin_key:60; and CurrentSurveyGeographyKernel.run:217 inside run_graph (19 via financial:1355; age :447/:457; 45 financial:1320).", + "classification": "must-not-move", + "notes": "Protects a token; it is what keeps the :202 per-row encode (next row) under 4096 even in the worst escaping case (960 B)." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:202 source._encode(values, maximum=4096)", + "value": "4096 bytes per encoded row [household_id, origin_key, state, puma]", + "enforced_at": [ + "current_survey_geography.py:200-204 _projection_digest: `for row in household.itertuples(...): values = [int(row[0]), *(None if pd.isna(v) else v ...)]; encoded = source._encode(values, maximum=4096); digest.update(len(encoded).to_bytes(4, \"big\")); digest.update(encoded)` \u2192 survey_population_preparation.py:120 \"PAYLOAD_LIMIT\" SurveyPopulationPreparationError", + "callers: current_survey_geography.py:261 and :317 (qualify, twice); graph_current_survey_geography.py:148-149 (_check_projection, `document[\"projection_sha256\"] == qualifier._projection_digest(projection.household)`) and :274 (FINAL_OUTPUT)" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT" + ], + "encodes": "one row at a time of the stacked-household projection (survey_geography_origin_key, survey_observed_state, survey_observed_puma keyed by household_id) streamed into a sha256; the roster is stacked households (ACS 1,531,614 + ASEC 55,762 at full source) but the bound is per row", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "per row 68 B (ACS, 7-digit household_id) / 56 B (ASEC); worst case 960 B for a 128-char raw id of control characters; streamed total per digest pass 10,727,204 B at 1/10", + "bytes_at_full_source": "same row widths (69 B with an 8-digit id); streamed total per pass 107,272,424 B; no whole copy exists", + "measurement_basis": "scratchpad geo_byte_census_measure.py M2: source._encode([hid, key, state, puma], maximum=4096) + 4-byte prefix per row on rows shaped from preparation.json (13-char SERIALNO, 5-char H_SEQ), and geo._projection_digest run on a 2-row table to prove the path", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4096 (identical on origin/native-row-ceilings)", + "head_value": "4096", + "reachable_from": "19", + "reach_chain": "reconstruct:431 qualify:261/:317 \u2192 _projection_digest:202; reconstruct:433/:619 observed_graph._check_projection:149; CurrentSurveyGeographyKernel.run:218/:259 (_check_projection) and :274 inside run_graph. 19 via financial:1355 \u2192 atomic:169/:293 and financial:1408; age :447/:457; 45 financial:1320.", + "classification": "must-not-move", + "notes": "docs/us-native-row-ceilings.md:228 already recorded this site as streaming; measured here: the widest possible row is 960 B (128 control chars escaped to \u0001 then re-escaped), 4.3\u00d7 under the cap. Each reconstruct chain runs the digest at least six times (two per qualify, one per _check_projection \u00d72, plus the kernel's three), each pass streaming ~107 MB at full source." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:203 len(encoded).to_bytes(4, \"big\")", + "value": "4-byte big-endian length prefix: 2**32 - 1 = 4,294,967,295 per row", + "enforced_at": [ + "current_survey_geography.py:203 in _projection_digest (OverflowError from int.to_bytes if exceeded; unreachable because :202 caps the row at 4096)" + ], + "refusal_codes": [ + "(none; OverflowError, unreachable behind the 4096 cap)" + ], + "encodes": "an encoder width on the per-row length prefix of the streamed projection digest", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "960 B maximum row (see the :202 row); the width admits 4,294,967,295", + "bytes_at_full_source": "same", + "measurement_basis": "scratchpad M2 widest row", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4 bytes (identical on origin/native-row-ceilings)", + "head_value": "4 bytes", + "reachable_from": "19", + "reach_chain": "same as the :202 row (every _projection_digest call).", + "classification": "must-not-move", + "notes": "Protects an encoder width; moving the 4096 cap above 2**32 - 1 would be the only way to reach it." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py:213 _sha(values.household.reset_index(drop=True).to_json(orient=\"table\").encode()) == document.get(\"household_projection_sha256\")", + "value": "none (pandas to_json has no cap; the producer's encode at current_asec_demographics.py:362-364 is the same expression under source._encode's 64 MiB receipt cap for the surrounding receipt only)", + "enforced_at": [ + "current_survey_geography.py:208-216 _check_asec_projection: `_require(type(receipt) is bytes and values.receipt == receipt, \"ASEC_RECEIPT\")`; `_require(_sha(values.household.reset_index(drop=True).to_json(orient=\"table\").encode()) == document.get(\"household_projection_sha256\"), \"ASEC_PROJECTION_DIGEST\")` \u2192 ValueError(\"CURRENT_SURVEY_GEOGRAPHY_ASEC_PROJECTION_DIGEST\")", + "called at current_survey_geography.py:243 and :307 (twice per qualify)", + "producer: current_asec_demographics.py:269-281 builds htable (household_id, native_household_id, H_SEQ_integer, GESTFIPS, state_fips Int64, state_status, state_known) and :362-364 digests the identical to_json(orient=\"table\") bytes" + ], + "refusal_codes": [ + "ASEC_PROJECTION_DIGEST", + "ASEC_RECEIPT" + ], + "encodes": "a per-row roster document: the ASEC household demographic projection (ASEC households), re-encoded whole by pandas to_json(orient=\"table\") to recompute its digest", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "969,567 bytes at 5,576 ASEC households (174.0 B/row)", + "bytes_at_full_source": "9,747,693 bytes at 55,762 ASEC households; 9,903 at 1/1000", + "measurement_basis": "scratchpad geo_byte_census_measure.py M3: htable built exactly as current_asec_demographics.py:269-281 on invented rows (7-digit ids, 5-digit H_SEQ, state 06/in_printed_range), pandas to_json(orient=\"table\").encode(); per-row from 1,000 vs 2,000 rows and direct totals at the \u00a73 counts", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (identical on origin/native-row-ceilings)", + "head_value": "none", + "reachable_from": "19", + "reach_chain": "reconstruct:431 qualify_current_survey_geography:241 demographics.qualify_current_asec_demographics \u2192 :243/:307 _check_asec_projection:213; and CurrentSurveyGeographyKernel.run:217 inside run_graph. 19 via financial:1355 \u2192 atomic:169/:293 and financial:1408; age :447/:457; 45 financial:1320.", + "classification": "stays", + "notes": "A consumer-side re-encoding with no cap, held whole (9.7 MB at full source) twice per qualify; the roster is the ASEC channel only (3.5% of stacked households), so it is small. The producer (another family) materialises the same document; its receipt (which carries selected_household_ids and per-household state tokens, :339-358) is that family's transport." + }, + { + "constant": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py:63 0 < len(population) <= 256 (and :65 0 < len(node_id) <= 256)", + "value": "256 characters per node/population name", + "enforced_at": [ + "graph_current_survey_geography.py:61-68 current_survey_geography_node: `_require(type(population) is str and 0 < len(population) <= 256 and type(node_id) is str and 0 < len(node_id) <= 256 and population != node_id, \"NODE_NAMES\")` \u2192 ValueError(\"SURVEY_GEOGRAPHY_PROJECTION_NODE_NAMES\") (_require :41-43)" + ], + "refusal_codes": [ + "NODE_NAMES" + ], + "encodes": "two scalars: the declared population version name and node id (\"survey_population.allocate\" = 26 chars, \"survey_population.observed_geography\" = 36 chars)", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "36 characters; does not move with the fraction", + "bytes_at_full_source": "36", + "measurement_basis": "the literals at graph_current_survey_geography.py:31 and graph_survey_population.ALLOCATION_NODE as used at survey_atomic_geography.py:444", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "256 (identical on origin/native-row-ceilings)", + "head_value": "256", + "reachable_from": "19", + "reach_chain": "reconstruct_atomic_survey_geography:441 observed_graph.current_survey_geography_node; graph_current_survey_geography._check_context:95 (called from run:210/:253) inside run_graph. 19 via financial:1355; age :447/:457; 45 financial:1320.", + "classification": "must-not-move", + "notes": "Protects tokens; listed for completeness. graph_geography.py (PUMA-ladder kernels) defines no byte bound of its own: its only byte site is load_source_bytes(\"raw-bytes-v1\") at :124-129 (recorded under codecs.py:78) and its canonical_json(document) at :249 re-encodes the frame_context, which graph_context._row_identity (:104-120) keeps fixed-shape (rows, id_dtype, ordered_ids_sha256); reachable_from none (only graph_composed_population:711-727 and graph_composed_asec_measures:614-619 use us_geography_nodes)." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/geo/geo_byte_census_measure.py (run with /Users/maxghenis/PolicyEngine/_worktrees/microcosm-native-byte-transports/.venv/bin/python 3.14.4, sys.path = packages/*/src; every imported module resolved inside the worktree)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/geo/geo-byte-census-measurements.json (receipt: receipt 1,213/1,223/1,228 B; projection rows 68/56/960 B, streamed 10,727,204 / 107,272,424; ASEC projection 174.0 B/row, 969,567 / 9,747,693; lineage 17.0/19.0 B per pair, 45,052,091 / 503,523,275; origin-key store column 42.0/35.0 B per row, 6,627,930 / 66,279,466 stacked and 13,255,852 / 132,558,924 clone)" + ], + "open_questions": [ + "Uncapped whole-roster materialisation of the combined-clone EXPAND lineage receipt (about 0.50 GB at full source, 26.5M pairs incl. four SCALED entities) at four sites in this family with encoder cap none \u2014 survey_atomic_geography.py:542 (three reconstruct calls per 19-node run), store.put_json:1099/:1102 (every write and every cache-hit load_json), RunManifest.key:661 and to_json:731 (held twice in one document; graph_atomic_survey_financial:1374 calls to_json_bytes on the 19/45 path) \u2014 plus two outside it (graph_combined_clone lineage digest :217; survey_age_calibration._node_identity:80-98 on the age path). No refusal at any fraction, so no `binds`; whether the lane wants the segmented shape for an uncapped document this size, or an artifact instead of a receipt, is its call. The 1/1000 manifest corroborates the shape (198,514 B clone receipt, 13,129 pairs) \u2014 I read manifest.json's receipt sizes and lineage counts from the recovered run directory although \u00a74 lists only preparation.json there.", + "Lineage byte cost assumes puf_support's decimal remap (puf_support.py:2324-2345, id + clone_index * 10**digits(max_id)) yields 8-digit clone targets over 7-digit natives at full source and 7 over 6 at 1/10; the stacked household id scheme (ASEC receiving ids at 55,002 in the 1/1000 artifact) may widen ids by a digit, adding at most 1 B per pair.", + "executor.py:111 _normal_json_mapping re-encodes every kernel receipt with no cap; on the 19-node path the only receipt that may be roster-shaped at that point is survey_predictors.asec_design_donor (FILTER) whose `selection` key is 12,876 B at 1/1000 \u2014 owned by the predictors family, not measured here.", + "keys.py source identity (source_content_key:64, _directory_identity:45) reads each upstream survey-source member whole with read_bytes and no cap, at run start, after each source-declaring node, and at run end (executor.py:2638/:2824/:3031); the ACS/ASEC archive sizes were not measured (brief \u00a74 forbids the gated data) and the transport is upstream-sized, not roster-sized.", + "graph_geography.py (assign_us_puma_ladder.* nodes) is reachable from none of the three runners (only graph_composed_population and graph_composed_asec_measures declare us_geography_nodes); its two raw-bytes sources (us_puma_ladder_2020 NPZ, us_cd_crosswalk_117_119 CSV) were not sized. congressional_district_geography, puma_ladder, geography_ladder, atomic_block_support, canonical, serialize contain no byte bound (value-range checks such as `< 2**53`, `< 2**63`, block/puma/district integer ranges and NPZ np.load were read and excluded as not byte bounds).", + "experiments/native-row-ceilings/census.json is stale for current_survey_geography.py:36 MAX_HOUSEHOLDS (records 524,288 and binds_at_full_source true; base and head are both 7,000,000, docs/us-native-row-ceilings.md:112) and cites the survey_atomic_geography _require helper at :67-69 (it is :68-70); no geography document appears in byte-census-measurements.json, so every byte figure here is from the scratchpad script.", + "store.put_frame (executor.py:1628) writes the whole clone-version frame with no byte cap (docs/us-native-scale-transport.md:79); only its string-column int64-offset width was measured (origin key 132.6 MB at clone households); numeric columns are exact numpy buffers (rows \u00d7 dtype width) and were not enumerated column by column." + ] + }, + { + "family": "Preparation and population graph", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py (2195 lines, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py (1399 lines, full; working tree differs from HEAD 77bafbb3a: _json_matches added, _json_sha256 signature changed)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_domains.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_sources.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/support_provenance.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py (14231 lines, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py (full)", + "partial, for reachability: graph_atomic_survey_financial.py:1275-1420 and grep; graph_survey_completion_host.py grep; survey_age_calibration.py:394-560; graph_puf_diagnostic_consumer.py:370-410; current_survey_predictors.py:500-530; current_survey_amounts.py:355-375", + "partial, for encoder caps: packages/microcosm-graph/src/microcosm/graph/canonical.py (full); graph/store.py grep (put_bytes/load_bytes carry no byte cap); operator_column_contracts.py:55-70; puf_support.py:2316-2328", + "inputs: experiments/native-byte-transports/census-brief.md, byte-census-measurements.json, byte_census_measure.py; experiments/native-row-ceilings/census.json, roster-census.json; experiments/native-scale-transport/ceiling-receipt.json; docs/us-native-scale-transport.md section 2; docs/us-native-row-ceilings.md; git diff origin/native-row-ceilings and git diff HEAD for the family" + ], + "rows": [ + { + "constant": "survey_population_preparation.py:49 MAX_REQUEST_BYTES", + "value": "4096 (literal) = 4,096", + "enforced_at": [ + "survey_population_preparation.py:456 REQUEST_FILE SurveyPopulationPreparationError(ValueError) (lstat st_size <= 4096 before open)", + "survey_population_preparation.py:458-459 read cap: _regular_reader(path, MAX_REQUEST_BYTES) then stream.read(MAX_REQUEST_BYTES + 1); _regular_reader refuses st_size > maximum at :426 SOURCE_REGULAR_FILE SurveyPopulationPreparationError", + "survey_population_preparation.py:460 REQUEST_LIMIT SurveyPopulationPreparationError", + "survey_population_preparation.py:487 REQUEST_CANONICAL SurveyPopulationPreparationError: consumer re-encoding _encode(document, MAX_REQUEST_BYTES) == payload, whose own refusal is :120 PAYLOAD_LIMIT at the same 4,096", + "survey_population_preparation.py:512 as the _file_limits entry for selection-request.json, enforced by _source_files at :561 SOURCE_FILE_LIMIT and through _regular_reader at :426 SOURCE_REGULAR_FILE" + ], + "refusal_codes": [ + "REQUEST_FILE", + "REQUEST_LIMIT", + "REQUEST_CANONICAL", + "PAYLOAD_LIMIT", + "SOURCE_REGULAR_FILE", + "SOURCE_FILE_LIMIT" + ], + "encodes": "an upstream file: /selection-request.json, a four-field JSON document {protocol, declaration, fraction:[num,den], seed}; also its canonical re-encoding", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "~168 B (the recovered artifact's source_files[0] records 168 B per the row-ceilings census; the artifact is absent from this worktree so not re-read here). Only the fraction pair's digits change with the fraction.", + "bytes_at_full_source": "~168 B (fraction [1,1] is shorter than [1,10])", + "measurement_basis": "row-ceilings census reading of _recovered preparation.json source_files[0]; document shape read from _request :461-486 (four fields, fraction pair of two ints, seed < 2**64)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 -> graph_atomic_survey_population.run_atomic_survey_population:157 -> graph_survey_population.run_authenticated_survey_population:1012 -> survey_population_preparation.prepare_authenticated_survey_population:1901 -> _request:451; also survey_population_source_codecs.load:745 read_survey_population_request:491 -> _request. Same prefix serves 45 (graph_survey_completion_host.extend over run_atomic_survey_financial:1319) and age (survey_age_calibration:427/:444).", + "classification": "must-not-move", + "notes": "Protects a fixed-shape upstream file with >24x headroom. Quoted refusal: `_require(len(payload) <= MAX_REQUEST_BYTES, \"REQUEST_LIMIT\")` (:460). Also sealed into the producer identity at :685 (identity only, not an enforcement). Base value = head value (unchanged on this branch)." + }, + { + "constant": "survey_population_preparation.py:50 MAX_PAYLOAD_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "survey_population_preparation.py:116/:120 PAYLOAD_LIMIT SurveyPopulationPreparationError: default `maximum` of _encode, reached for every non-roster document and per-row encode: _producer() (:670, :1920, :1795, :1734), _authority() (:700), spill header.json (:237), each _bounded_append row (:1151), _frame_identity header lists (:986-1044), _nested_seals snapshot list (:1650), _request canonical (:487 with 4096)", + "survey_population_preparation.py:751/:787 PAYLOAD_LIMIT SurveyPopulationPreparationError: _frame_cell_encode, one normalized Frame cell (fast path bound; slow path :769 is _encode(value, maximum))", + "current_survey_predictors.py:522 (other family, 19-node node survey_predictors.source_projection): passed as `segment=source.MAX_PAYLOAD_BYTES` to host.survey_graph._segmented_json, i.e. the per-segment ceiling of the predictor projection; refuses TRANSPORT_LIMIT in graph_survey_population.py:331 SurveyPopulationGraphError", + "current_survey_amounts.py:369 (not on either path): the _bounded_json limit of the amounts projection" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT", + "TRANSPORT_LIMIT" + ], + "encodes": "in this module: single fixed documents (the ~21 KB _producer() record, the ~2 KB spill header, the 168 B request), single rows (24-247 B) and single cells (<= 6*4096+2 B); out of family: the per-segment width of the predictor projection whose roster is stacked persons (24.04 B per origin row per byte-census-measurements.json predictor_projection)", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "largest document this module encodes under it: _producer() ~21,135 B (row-ceilings census); as the predictor projection's segment: 8,570,284 B total at 1/10 (byte-census-measurements.json predictor_projection.tenth_origins_bytes), one segment", + "bytes_at_full_source": "same fixed documents; as the predictor projection's segment: 85,702,912 B total at full source (byte-census-measurements.json), i.e. two segments; no single accumulation exceeds 64 MiB", + "measurement_basis": "byte-census-measurements.json predictor_projection (per_person_origin_row_bytes 24.04 through graph._bounded_json); row-ceilings census for _producer(); prep-measurements.json per-row encodes (24-247 B)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1892 -> _encode(_producer()):1920 etc.; _frame_identity:980 -> _cells_blob:949/_frame_cell_encode:751 from :2012 and _pure_final:1760; out of family: run_atomic_survey_financial -> current_survey_predictors.qualify_current_survey_predictors -> _segmented_json(segment=source.MAX_PAYLOAD_BYTES) :512-523", + "classification": "must-not-move", + "notes": "This is the encoder's accumulation width; the whole-roster preparation receipt no longer passes through _encode (it goes through _roster_segments under MAX_SEGMENT_BYTES, the alias at :57). Sealed at :686. The predictor projection that meets 64 MiB at full source is that family's segment row; this row records the enforcement site. Base value = head value." + }, + { + "constant": "survey_population_preparation.py:57 MAX_SEGMENT_BYTES", + "value": "MAX_SEGMENT_BYTES = MAX_PAYLOAD_BYTES = 67,108,864", + "enforced_at": [ + "survey_population_preparation.py:163-164 segment close (`if current and len(current) + len(encoded) > segment: close()`), then :165 PAYLOAD_LIMIT SurveyPopulationPreparationError on a single token larger than one whole segment; reached from _roster_payload:241-253 at :2013 (preparation receipt, spill=snapshots) and :2180 (wage projection)" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT" + ], + "encodes": "one segment of the whole-roster preparation receipt: rosters selection.selected (stacked households), origins.households (stacked households), origins.persons.rows (stacked persons), origins.entities (household, person, and SCALED tax_unit/family/marital_unit/spm_unit rows); fixed blocks selection.excluded (all catalogue exclusions), selection.cells, storage_transitions, producer, catalogues, native, source_files, request, digests. Also one segment of the ASEC wage projection (rows = ASEC persons).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "receipt 126,660,764 B at full-source id widths (this lane; 2 segments) = 1.89x one segment; transport lane's replicated 1/1000-width artifact 109,804,304 B (ceiling-receipt.json, 2 segments). Wage projection 667,964 B (14,212 rows x 47 B), one segment.", + "bytes_at_full_source": "receipt 1,251,912,520 B (this lane; 18.65x, 19 segments); transport lane 1,099,892,722 B (17 segments). Per selected household 788.6 B at full widths (selected 238/235 B, households 179/174 B, persons 76/86 B x 2.19, entity rows 24 B x 6.9). Wage projection 6,679,875 B (142,125 x 47 B).", + "measurement_basis": "scratchpad/census/prep/prep-measurements.json: per-row bytes through prep._encode and prep._plan_document on invented rows at full-source widths (7-digit stacked/native ids, 13-char SERIALNO, 22-char PERIDNUM, 4-digit WGTP, unreduced 1/10 inclusion fractions), marginal check by difference of two row counts (matches row+comma), roster totals at the roster-census counts, plus the row-ceilings census's 45,922 B fixed preamble; _roster_segments run on a 20,000-row synthetic document with segment=1 MiB: stream total == len(_encode), digest == sha256(_encode) == _digest, joined == single, every segment under the segment size. Cross-checked against ceiling-receipt.json.", + "binds_at_tenth": true, + "binds_at_full_source": true, + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 -> run_atomic_survey_population:157 -> run_authenticated_survey_population:1012 -> prepare_authenticated_survey_population:2013 -> _roster_payload:241 -> _roster_segments:132 (:165). Wage projection: only graph_puf_diagnostic_consumer.qualify_current_survey_host:393 (callers graph_current_survey_puf_transfer:224/:380 and graph_puf_diagnostic_consumer:569/:751), reachable_from none.", + "classification": "segment", + "notes": "'Binds' here means the document exceeds one segment at both scales (the brief's definition); the site closes a segment rather than refusing, so PAYLOAD_LIMIT at :165 can only fire on one token larger than 64 MiB, which _check_scalars (:97, 1 MiB per string) makes unreachable. Quoted: `_require(len(current) + len(encoded) <= segment, \"PAYLOAD_LIMIT\")`. The segmented shape already shipped on the base branch (base :129 _roster_segments), so base value = head value. Sealed at :687." + }, + { + "constant": "survey_population_preparation.py:58 MAX_ROSTER_BYTES", + "value": "64 * MAX_SEGMENT_BYTES = 4,294,967,296", + "enforced_at": [ + "survey_population_preparation.py:166 ROSTER_LIMIT SurveyPopulationPreparationError (running stream total in _roster_segments)", + "survey_population_preparation.py:262 ROSTER_LIMIT SurveyPopulationPreparationError (`len(payload) == total <= maximum` on the joined payload in _roster_payload)", + "survey_population_preparation.py:1152 ORIGIN_LIMIT SurveyPopulationPreparationError (_bounded_append: three independent budgets, each `budget[0] + len(_encode(row)) + 1`: _plan_document :387-398 selected/excluded/cells; _origins :1157 entities :1204, households :1280, persons :1299; _current_survey_wage_projection :2148)", + "survey_population_preparation.py:1899 CANDIDATE_LIMIT SurveyPopulationPreparationError (caller-supplied candidate copy of the receipt)", + "current_survey_predictors.py:523 (other family, 19-node path): `maximum=source.MAX_ROSTER_BYTES` of the segmented predictor projection; refuses TRANSPORT_ROSTER_LIMIT in graph_survey_population.py:332/:337 SurveyPopulationGraphError" + ], + "refusal_codes": [ + "ROSTER_LIMIT", + "ORIGIN_LIMIT", + "CANDIDATE_LIMIT", + "TRANSPORT_ROSTER_LIMIT" + ], + "encodes": "the whole preparation receipt (same rosters as MAX_SEGMENT_BYTES); the Python row lists behind it (ORIGIN_LIMIT); a candidate copy; out of family, the predictor projection total (stacked persons)", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "receipt 126,660,764 B (0.0295 x); ORIGIN_LIMIT budgets: _origins 87,423,531 B, _plan_document 39,191,311 B, wage rows 667,964 B; predictor projection 8,570,284 B", + "bytes_at_full_source": "receipt 1,251,912,520 B (0.291 x; transport lane 1,099,892,722 B = 0.256 x); ORIGIN_LIMIT budgets: _origins 874,238,396 B, _plan_document 377,628,202 B, wage rows 6,679,875 B; predictor projection 85,702,912 B. The 4 GiB roster admits ~5,445,991 selected households (3.43 x source) at 788.6 B each.", + "measurement_basis": "prep-measurements.json (this lane, full widths, module encoders); ceiling-receipt.json (transport lane); byte-census-measurements.json predictor_projection", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1892 -> _plan_document:2032/:2061, _origins:1986, _roster_payload:2013 (:166, :262), candidate :1899; also survey_population_source_codecs.load:746. 45 and age via the same prefix.", + "classification": "stays", + "notes": "Headroom 3.4x at full source (3.9x on the transport lane's narrower-id figure). Quoted: `_require(budget[0] + size <= MAX_ROSTER_BYTES, \"ORIGIN_LIMIT\")` (:1152) with the comment at :1148-1150 calling it 'A bound on the Python row list, not on a transport'. Base value = head value (already 64 x 64 MiB on origin/native-row-ceilings). Sealed at :688." + }, + { + "constant": "survey_population_preparation.py:60 _MAX_SCALAR_BYTES", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "survey_population_preparation.py:97 SCALAR_LIMIT SurveyPopulationPreparationError in _check_scalars, called before every encode path: _chunks :102, _catalogue_chunks :307, _frame_cell_encode :778" + ], + "refusal_codes": [ + "SCALAR_LIMIT" + ], + "encodes": "one string scalar anywhere in an encoded document (a token), checked in characters before encoding", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "<= 128 characters for every source literal (survey_population_domains.MAX_TOKEN_CHARS); real maxima 64-hex digests, 22-char PERIDNUM, 13-char SERIALNO; identical at every fraction", + "bytes_at_full_source": "same; widths do not scale with the roster", + "measurement_basis": "code: domains._token :187 bounds source literals at 128; _check_scalars walks every string (:96-97)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "every _encode/_digest/_roster_segments call in prepare_authenticated_survey_population (via _chunks:99-102), and _frame_identity via _frame_cell_encode:778", + "classification": "must-not-move", + "notes": "It is what makes the segment refusal at :165 unreachable: no token can exceed 1 MiB, 64x below a segment (comment :159-162). Sealed at :689." + }, + { + "constant": "survey_population_preparation.py:61 _FRAME_FAST_STRING_CHARS", + "value": "4096 (literal) = 4,096", + "enforced_at": [ + "survey_population_preparation.py:765 (fast-path eligibility test `kind is str and len(value) <= _FRAME_FAST_STRING_CHARS` in _frame_cell_encode; an ineligible string takes _encode(value, maximum) at :769 under MAX_PAYLOAD_BYTES; no refusal code of its own)" + ], + "refusal_codes": [], + "encodes": "one string Frame cell's canonical spelling (comment :781-783: at most 6 * 4096 + 2 encoded bytes on the fast path)", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "per cell: string columns of the stacked frame are SERIALNO (13), channel labels, PERIDNUM (22); no scaling with rows", + "bytes_at_full_source": "same", + "measurement_basis": "code reading; no roster document behind it", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:2012 _frame_identity -> _cells_blob:949 -> _string_cells (encodes each distinct value once via _frame_cell_encode) / cell-at-a-time fallback :976; again in _pure_final:1760 and verify_materialized_survey_population:1886", + "classification": "must-not-move", + "notes": "A width threshold, not a ceiling: exceeding it selects the slow path, which is still bounded by MAX_PAYLOAD_BYTES per cell. Sealed at :691. Unchanged on this branch." + }, + { + "constant": "survey_population_preparation.py:793 _FLOAT_WIDTH", + "value": "37 (literal) = 37 bytes per padded float64 cell spelling", + "enforced_at": [ + "survey_population_preparation.py:849-850 (`cells = np.zeros((count, _FLOAT_WIDTH), dtype=np.uint8)`; `keep = np.zeros((count, _FLOAT_WIDTH), dtype=bool)` in _float_cells; no refusal: the only refusal in the function is :834 FRAME_NONFINITE SurveyPopulationPreparationError, a value check)" + ], + "refusal_codes": [ + "FRAME_NONFINITE" + ], + "encodes": "a fixed-width encoder matrix: one float64 column of a Frame (person, household, group or strata/index column) laid out at 37 byte slots per cell plus a 37-slot keep mask, cut to the exact bytes a cell-at-a-time walk would digest", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "transient per float column: 37+37 B x rows -> 26.4 MB for a stacked-person column at 356,501 rows (52.8 MB for a clone-person column at 713,002); the resulting bytes (<= 37 x rows) are then digested. No cap.", + "bytes_at_full_source": "transient per float column: 263.8 MB for a stacked-person column at 3,565,013 rows (527.6 MB at clone persons 7,130,026, reached only from graph_puf_diagnostic_consumer:399, off-path). No cap.", + "measurement_basis": "arithmetic from the code (:849-850, :898 `cells[keep].tobytes()`) at the roster-census counts", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:2012 _frame_identity:980 -> _cells_blob:949 -> _float_cells:814; _pure_final:1760; verify_materialized_survey_population:1886", + "classification": "must-not-move", + "notes": "An encoder width: moving it changes the digest every sealed identity depends on. Recorded because the brief lists struct widths; there is no byte ceiling on the column blob it produces (see open questions)." + }, + { + "constant": "survey_population_preparation.py:289 characters > 65_536 (with :278 len(people) > 20, :274 len(record) != 7, :280 len(person) != 9)", + "value": "65_536 ASCII characters per ACS catalogue record (fast-path eligibility); comment :296-297: 'At most 186 leaves and 65,536 ASCII characters bound this individual JSON record below 400 KiB'", + "enforced_at": [ + "survey_population_preparation.py:289 (returns False from _catalogue_fast_path; the value then takes the generic _chunks path at :302-304; no refusal code)" + ], + "refusal_codes": [], + "encodes": "one raw ACS catalogue record (a 7-tuple with <= 20 person 9-tuples) inside the (records, vacancies) tuple whose digest _nested_seals carries; the tuple covers the whole ACS catalogue (1,631,969 records, 374,174,650 canonical bytes per roster-census.json), fraction-invariant", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "per record a few hundred bytes (<= 20 persons x 9 short literals); the whole catalogue digest is 374,174,650 B at every fraction, streamed record by record into sha256", + "bytes_at_full_source": "same (the catalogue is the whole upstream archive)", + "measurement_basis": "roster-census.json catalogues.acs.canonical_record_bytes; code :267-318", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:2040 _nested_seals -> :1650-1655 _memoized_catalogue_digest -> _catalogue_digest -> _catalogue_chunks:299 -> _catalogue_fast_path:267; again in _pure_final:1762 and _catalogue_memo:2043", + "classification": "must-not-move", + "notes": "Never refuses; failing the test only removes the fast path. Included for completeness as a per-record character cap." + }, + { + "constant": "acs_population_catalogue.py:25 _ARCHIVE_BYTES (defined in the ACS source family; enforced here)", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "survey_population_preparation.py:513-514 (_file_limits entry for acs/csv_hus.zip and acs/csv_pus.zip) -> _source_files:558 _regular_reader(path, limits[name]) -> :426 SOURCE_REGULAR_FILE SurveyPopulationPreparationError (st_size <= maximum) and :561 SOURCE_FILE_LIMIT SurveyPopulationPreparationError (running size while hashing in 1 MiB blocks :559)", + "survey_population_preparation.py:547-552 ARCHIVE_LIMIT SurveyPopulationPreparationError (sum of both archives' lstat sizes <= _ARCHIVE_BYTES)", + "acs_population_catalogue.py:481 ARCHIVE_LIMIT (its own family) and acs_native_coverage_binding.py:568 MAX_ARCHIVE_BYTES (same value, same quantity)" + ], + "refusal_codes": [ + "SOURCE_REGULAR_FILE", + "SOURCE_FILE_LIMIT", + "ARCHIVE_LIMIT" + ], + "encodes": "two upstream files: the captured ACS PUMS archives csv_hus.zip and csv_pus.zip", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "251,500,587 + 602,847,146 = 854,347,733 B (acs_2024_1yr_sources.json, confirmed present in this tree), identical at every fraction", + "bytes_at_full_source": "854,347,733 B (9.9% of the ceiling)", + "measurement_basis": "acs_2024_1yr_sources.json sizes; row-ceilings census confirmed them on the pilot's captured files", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1919 _source_files(root) and _validate:1791 _source_files(state.root) (every full validation, including verification-epoch closes)", + "classification": "must-not-move", + "notes": "Recorded here for its enforcement sites in this family; keyed by its definition in the ACS family. Quoted: `_require(size <= limits[name], \"SOURCE_FILE_LIMIT\")` (:561)." + }, + { + "constant": "asec_2024_native_population.py:42 _MAX_CHECKPOINT_BYTES (defined in the ASEC family; enforced here)", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "survey_population_preparation.py:506 (_file_limits default for every _SOURCE_ROSTER entry; it remains the limit for asec/parent.h5, asec/household-attachment.h5 and asec/person-income-attachment.h5) -> :426 SOURCE_REGULAR_FILE and :561 SOURCE_FILE_LIMIT SurveyPopulationPreparationError", + "asec_2024_native_population.py:163 SOURCE_FILE_BOUNDS (its own family)" + ], + "refusal_codes": [ + "SOURCE_REGULAR_FILE", + "SOURCE_FILE_LIMIT" + ], + "encodes": "three upstream HDF5 files: the ASEC parent and its two attachments", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "person-income-attachment.h5 666,333,922 B (row-ceilings census); parent.h5 and household-attachment.h5 sizes not established in either census; identical at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "row-ceilings census (asec_2024_native_population.py:42 row); no size pin for the other two files exists in this tree", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1919 / _validate:1791 -> _source_files:518 -> _regular_reader:420", + "classification": "must-not-move", + "notes": "An upstream file's real size; fraction-invariant. The two unestablished sizes are an open question." + }, + { + "constant": "asec_person_coverage_source.py:33 MAX_MEMBER_BYTES (defined in the ASEC family; enforced here as asec_native.coverage_owner.literal.MAX_MEMBER_BYTES)", + "value": "1_000_000_000", + "enforced_at": [ + "survey_population_preparation.py:507-509 (_file_limits entries for asec/pppub23.csv, pppub24.csv, pppub25.csv) -> :426 SOURCE_REGULAR_FILE and :561 SOURCE_FILE_LIMIT SurveyPopulationPreparationError", + "asec_person_coverage_source.py:168/:181/:191 MEMBER_BYTES and asec_coverage_authentication.py:315-318 COVERAGE_MEMBER_BYTES (own family)" + ], + "refusal_codes": [ + "SOURCE_REGULAR_FILE", + "SOURCE_FILE_LIMIT" + ], + "encodes": "three upstream CSV files: the pinned ASEC person members", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "largest member 281,065,733 B (row-ceilings census; the figure appears in packages/microcosm-build/src/microcosm/build/us/source_stages.json), identical at every fraction", + "bytes_at_full_source": "same (28% of the ceiling)", + "measurement_basis": "row-ceilings census; source_stages.json", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1919 / _validate:1791 -> _source_files:518", + "classification": "must-not-move", + "notes": "asec_coverage_authentication.py:28 binds `literal` to asec_person_coverage_source, so :508 resolves to this constant." + }, + { + "constant": "asec_original_household_weights.py:41 _MAX_MEMBER_BYTES (defined in the ASEC family; enforced here as asec_native.anchor_owner._MAX_MEMBER_BYTES)", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "survey_population_preparation.py:511 (_file_limits entry for asec/hhpub25.csv) -> :426 SOURCE_REGULAR_FILE and :561 SOURCE_FILE_LIMIT SurveyPopulationPreparationError", + "asec_original_household_weights.py:174 (own family registry pin check)" + ], + "refusal_codes": [ + "SOURCE_REGULAR_FILE", + "SOURCE_FILE_LIMIT" + ], + "encodes": "one upstream CSV file: the pinned ASEC 2025 household member hhpub25.csv", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "33,298,479 B (native_household_origin.py:164 pin `size_bytes=33_298_479`), identical at every fraction", + "bytes_at_full_source": "33,298,479 B (2.0x headroom)", + "measurement_basis": "the pin in native_household_origin.py:158-166 (this tree); row-ceilings census agrees", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1919 / _validate:1791 -> _source_files:518", + "classification": "must-not-move", + "notes": "asec_2024_native_population.py:34 binds anchor_owner to asec_original_household_weights." + }, + { + "constant": "graph_survey_population.py:61 PREPARATION_MAX_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "graph_survey_population.py:403-404 CONTEXT_BYTES SurveyPopulationGraphError (`0 < len(context) <= PREPARATION_MAX_BYTES` on view.context in _checked_preparation)", + "graph_survey_population.py:564 TRANSPORT_LIMIT SurveyPopulationGraphError (`new_context = _bounded_json(context, PREPARATION_MAX_BYTES)` in _allocation_output, the outgoing allocation frame context)", + "graph_survey_population.py:676-677 CONTEXT_BYTES SurveyPopulationGraphError (SurveyPopulationAllocationKernel.__init__ on context_bytes)", + "BASE ONLY (origin/native-row-ceilings): base:305 and base:577 PREPARATION_BYTES on the preparation payload itself; on this branch those two sites moved to PREPARATION_ROSTER_BYTES (head :399-400, :671-672)" + ], + "refusal_codes": [ + "CONTEXT_BYTES", + "TRANSPORT_LIMIT", + "PREPARATION_BYTES (base only)" + ], + "encodes": "the US frame-context document (graph_context.encode_us_frame_context: per entity the column list, row count, id dtype and one 64-hex ordered-id digest; normative metadata; mass_log; weight_sources), a fixed-shape header that does not grow with rows", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "5,442 B measured through graph_context.encode_us_frame_context on an invented six-entity Frame with 40 columns per entity (prep-measurements.json frame_context); the real inventory's width differs but no term grows with rows", + "bytes_at_full_source": "same order (a few KB); at the BASE value the payload sites also saw the receipt: 126,660,764 B and 1,251,912,520 B, both over 64 MiB", + "measurement_basis": "prep-measurements.json frame_context; code graph_context.py:127-160 (only columns lists and one digest per entity vary)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "67,108,864, and at base also enforced on the preparation payload (base:305 PREPARATION_BYTES, base:577), where the receipt exceeded it at 1/10 (1.64-1.89x) and full source (16-19x)", + "head_value": "67,108,864, frame-context sites only (:403-404, :564, :676-677)", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial:1355 -> run_atomic_survey_population:157 -> run_authenticated_survey_population:1015 _checked_preparation:390 (:403) and :1028 _allocation_output:543 (:564) and :1034 SurveyPopulationAllocationKernel.__init__:662 (:676); run_graph -> kernel.run:726; graph_atomic_survey_population:172/:176; survey_age_calibration:467/:473", + "classification": "must-not-move", + "notes": "The bound now sits only in front of a fixed-shape document at the encoder's own 64 MiB. The binds flags are false for the document it bounds at head; the receipt's binding at the base payload sites is carried by the PREPARATION_ROSTER_BYTES row. graph_context itself has no constant: encode_us_frame_context is canonical_json (no cap) and _row_identity :113-115 digests `ids.to_numpy(' run_atomic_survey_population:157 -> run_authenticated_survey_population:1015 -> _checked_preparation:390 (:399); :1034 -> SurveyPopulationAllocationKernel.__init__:662 (:671); :1284 final _checked_preparation; graph_atomic_survey_population:172; survey_age_calibration:467. 45 via the same prefix under graph_survey_completion_host.extend.", + "classification": "consumer", + "notes": "The binds flags follow the brief: evaluated at the base value (64 MiB) the receipt exceeded the cap at both scales; at the head value it is 0.29x of the ceiling and equals the producer's own MAX_ROSTER_BYTES total. Quoted head comment :69-75: 'this consumer re-checked the same bytes against PREPARATION_MAX_BYTES ... The consumer's ceiling on the receipt is the producer's own.' Two further whole copies of the receipt exist without any bound: _final_artifact:899-900 (`actual = store.load_bytes(key); _require(... actual == payload, \"FINAL_ARTIFACT_BYTES\")`) and graph_atomic_survey_population:264 (ATOMIC_PREFIX_ARTIFACT_BYTES)." + }, + { + "constant": "graph_survey_population.py:62 ALLOCATION_MAX_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "graph_survey_population.py:478-483 segment close (`if (segments or output) and len(output) + len(separator) + len(raw) + len(tail) > (ALLOCATION_MAX_BYTES): segments.append(bytes(output))`)", + "graph_survey_population.py:484-487 ALLOCATION_LIMIT SurveyPopulationGraphError (one instruction row plus the reserved tail larger than one segment)", + "graph_survey_population.py:495 ALLOCATION_LIMIT SurveyPopulationGraphError (last segment plus tail)" + ], + "refusal_codes": [ + "ALLOCATION_LIMIT" + ], + "encodes": "one segment of the allocation payload `{\"households\":[],\"input_context_sha256\":...,\"output_context_sha256\":...,\"preparation_sha256\":...,\"protocol\":...,\"release_eligible\":false,\"weight_transition\":[...]}`; roster = selected households = stacked households", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "51,879,119 B (158,737 households; 1 segment; 0.77x): 327 B per ACS row and 322 B per ASEC row at 1/10 widths (unreduced inclusion fraction)", + "bytes_at_full_source": "490,164,612 B (8 segments; 7.3x): 309 B per ACS row, 303 B per ASEC row (inclusion [1,1]); docs/us-native-scale-transport.md cites 316.665 B/row -> 502.6 MB. One 64 MiB accumulation admits ~217,180 households (13.7% of source).", + "measurement_basis": "prep-measurements.json allocation_payload: graph._allocation_payload on invented AllocationInstruction tuples at full widths, marginal by difference (2 vs 1002 rows); per-row instruction document via graph._bounded_json(graph._instruction_document(row), 4096); payload bytes proved equal to the canonical whole-tree _bounded_json of the decoded document (payload_equals_canonical_whole_tree true)", + "binds_at_tenth": false, + "binds_at_full_source": true, + "reachable_from": "19", + "reach_chain": "run_authenticated_survey_population:1028 -> _allocation_output:543 -> _allocation_payload:565 (:447); again in SurveyPopulationAllocationKernel.run:726 during run_graph; graph_atomic_survey_population:176; survey_age_calibration:473", + "classification": "segment", + "notes": "Already segmented on the base branch (base:388/:404), so base value = head value; at full source it closes seven segments and never refuses. 'Binds at full source' means the roster exceeds one segment. Quoted: `_require(len(output) + len(separator) + len(raw) + len(tail) <= ALLOCATION_MAX_BYTES, \"ALLOCATION_LIMIT\")` (:484-487)." + }, + { + "constant": "graph_survey_population.py:68 ALLOCATION_ROSTER_BYTES", + "value": "64 * ALLOCATION_MAX_BYTES = 4,294,967,296", + "enforced_at": [ + "graph_survey_population.py:170 ALLOCATION_LIMIT SurveyPopulationGraphError as a row count: `len(plan.selected) <= ALLOCATION_ROSTER_BYTES // 128` = 33,554,432 selected households", + "graph_survey_population.py:488-491 ALLOCATION_ROSTER_LIMIT SurveyPopulationGraphError (running total in _allocation_payload)", + "graph_survey_population.py:499 ALLOCATION_ROSTER_LIMIT (total after the tail)", + "graph_survey_population.py:501 ALLOCATION_ROSTER_LIMIT (`len(payload) == total` on the joined payload)" + ], + "refusal_codes": [ + "ALLOCATION_LIMIT", + "ALLOCATION_ROSTER_LIMIT" + ], + "encodes": "the whole allocation payload (selected households); the :170 arm bounds the count of selected households before the two maps are built", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "51,879,119 B (0.012x); 158,737 households against 33,554,432", + "bytes_at_full_source": "490,164,612 B (0.114x); the byte total admits ~13,899,570 households (8.8x source) at 309 B; 1,587,376 against the 33,554,432 row pre-check (21x)", + "measurement_basis": "prep-measurements.json allocation_payload", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "run_authenticated_survey_population:1020 allocation_instructions:154 (:170); :1028 -> _allocation_payload (:488-501); graph_atomic_survey_population:173/:176; survey_age_calibration:470/:473", + "classification": "stays", + "notes": "Headroom 8.8x on bytes. The :170 arm is a row count wearing a byte name (rule: ALLOCATION_ROSTER_BYTES // 128; the comment at :165-169 says the implied 128-byte row budget is below the payload's structural overhead so it never binds the payload) and is not-a-transport by itself. Base value = head value." + }, + { + "constant": "graph_survey_population.py:472 4096 (tail) and :477 4096 (per-instruction document)", + "value": "4096 = 4,096 bytes per _bounded_json call", + "enforced_at": [ + "graph_survey_population.py:472 TRANSPORT_LIMIT SurveyPopulationGraphError via `_bounded_json(metadata, 4096)` (the six-field metadata tail with three 64-hex digests)", + "graph_survey_population.py:477 TRANSPORT_LIMIT SurveyPopulationGraphError via `_bounded_json(_instruction_document(row), 4096)` (one instruction document per selected household)" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT" + ], + "encodes": "one instruction document (11 fields: ids, key, four fraction pairs, a float64 hex) or the payload tail", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "instruction document 326 B (ACS) / 321 B (ASEC) at 1/10 widths; tail ~300 B", + "bytes_at_full_source": "308 B / 302 B (inclusion [1,1]); tail unchanged. 13x headroom.", + "measurement_basis": "prep-measurements.json allocation_payload.rows.instruction_document_bytes through graph._bounded_json(graph._instruction_document(row), 4096)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "as ALLOCATION_MAX_BYTES: _allocation_output:565 -> _allocation_payload:447", + "classification": "must-not-move", + "notes": "The only variable-width field is raw_native_id, bounded at 128 chars by :184 SELECTED_KEY; every other member is an integer or a two-integer pair." + }, + { + "constant": "graph_survey_population.py:184 0 < len(row.key.native_id) <= 128", + "value": "128 characters", + "enforced_at": [ + "graph_survey_population.py:184-185 SELECTED_KEY SurveyPopulationGraphError (allocation_instructions, per selected household)" + ], + "refusal_codes": [ + "SELECTED_KEY" + ], + "encodes": "one raw native household id token (ACS SERIALNO, 13 chars by survey_population_domains.py:208; ASEC H_SEQ <= 5 digits by :219)", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13 / <= 5 characters; identical at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "code: domains regexes :208 and :219 fix the widths", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "run_authenticated_survey_population:1020 -> allocation_instructions:154; graph_atomic_survey_population:173; survey_age_calibration:470", + "classification": "must-not-move", + "notes": "Same value as survey_population_domains.MAX_TOKEN_CHARS; the comment at :279-280 relies on it to justify the pre-encoding TRANSPORT_LIMIT check." + }, + { + "constant": "graph_survey_population.py:272 0 < limit <= 64 * 1024**2 (_bounded_json) and :294 (_canonical_pieces segment)", + "value": "64 * 1024**2 = 67,108,864: the shared encoder's ceiling on any limit/segment argument", + "enforced_at": [ + "graph_survey_population.py:272 TRANSPORT_LIMIT SurveyPopulationGraphError (refuses a limit above 64 MiB before encoding a byte); per piece :281 (characters) and :283 (bytes) TRANSPORT_LIMIT", + "graph_survey_population.py:294 TRANSPORT_LIMIT (segment argument of _canonical_pieces); per piece :303/:305 TRANSPORT_LIMIT; used by _segmented_json:313 (:331 per segment), _json_sha256:341, _json_matches:356", + "callers passing a limit <= 64 MiB: this module :472/:477 (4096), :564 (PREPARATION_MAX_BYTES); survey_origin_budget.py:93 _json (MAX_PAYLOAD_BYTES) and :629-632; graph_survey_budget.py:146-158 (segment=numeric.MAX_BYTES); current_survey_predictors.py:512-523 (segment=survey_population_preparation.MAX_PAYLOAD_BYTES); graph_puf_diagnostic_consumer.py:403 (CURRENT_SURVEY_MAX_BYTES); current_survey_amounts.py:362-369" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT", + "TRANSPORT_ENCODING" + ], + "encodes": "an encoder width: the largest single accumulation (or single segment) any canonical JSON document may occupy through the shared encoder", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "largest single accumulation on the 19-node path through _bounded_json proper: the allocation frame context (~5.4 KB); largest segment through _canonical_pieces: one 64 MiB segment of the allocation payload / predictor projection", + "bytes_at_full_source": "same by construction; every whole-roster document that exceeds it is segmented by its owner (allocation payload 8 segments, predictor projection 2, origin budget ~18)", + "measurement_basis": "code; prep-measurements.json and byte-census-measurements.json for the callers' totals", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "every _bounded_json/_segmented_json call in the runners' prefix: run_authenticated_survey_population:1028 -> _allocation_output:564/:565; current_survey_predictors:512", + "classification": "must-not-move", + "notes": "docs/us-native-row-ceilings.md section 5c: 'so the shared encoder refuses any cap above 64 MiB before it encodes a byte ... The 64 MiB is the encoder's, and moving it is the transport change.' Quoted: `_require(type(limit) is int and 0 < limit <= 64 * 1024**2, \"TRANSPORT_LIMIT\")` (:272)." + }, + { + "constant": "graph_survey_population.py:349 64 * 1024**2 (_json_sha256 token cap) and :369 (_json_matches token cap)", + "value": "64 * 1024**2 = 67,108,864 per JSON token; the total is the caller's `maximum`", + "enforced_at": [ + "graph_survey_population.py:349 -> _canonical_pieces(value, 64 * 1024**2) -> :303/:305 TRANSPORT_LIMIT SurveyPopulationGraphError (one token); :350 TRANSPORT_ROSTER_LIMIT (caller total) in _json_sha256", + "graph_survey_population.py:369 -> _canonical_pieces(value, 64 * 1024**2) -> :303/:305 TRANSPORT_LIMIT; :371 TRANSPORT_ROSTER_LIMIT; :367 TRANSPORT_ENCODING (payload not bytes) in _json_matches", + "callers: survey_origin_budget.py:605 selection_sha256 = graph._json_sha256(view.receipt['selection'], maximum=MAX_ROSTER_BYTES); survey_origin_budget.py:935 graph._json_matches(document, entry[1], maximum=MAX_ROSTER_BYTES) FINAL_BUDGET_VIEW_DOCUMENT" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT", + "TRANSPORT_ROSTER_LIMIT", + "TRANSPORT_ENCODING" + ], + "encodes": "a per-token width inside a streamed digest / streamed comparison: the selection block (stacked households, 243 B/row SCALED: 38,594,135 B at 1/10, 385,942,815 B full, byte-census-measurements.json selection_block) and the origin-budget document (742 B/group: 117,782,854 B at 1/10, 1,177,832,992 B full, byte-census-measurements.json origin_budget); largest token a 64-hex digest or 22-char PERIDNUM", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "per token <= 64 chars; the streamed documents 38.6 MB (selection) and 117.8 MB (budget), no accumulation", + "bytes_at_full_source": "per token unchanged; streamed documents 386 MB and 1.18 GB", + "measurement_basis": "byte-census-measurements.json selection_block and origin_budget; code :341-374", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration:529 budgets.freeze_survey_origin_budget -> survey_origin_budget:605 _json_sha256; :535 budget.checked_view -> survey_origin_budget:935 _json_matches. Not reached by the 19- or 45-node graphs (neither imports survey_origin_budget on its prefix).", + "classification": "must-not-move", + "notes": "_json_matches and the segment-less _json_sha256 signature are uncommitted working-tree changes (git diff HEAD); the cited lines are working-tree lines. The token cap is the encoder width; the totals are the origin-budget family's MAX_ROSTER_BYTES rows." + }, + { + "constant": "survey_catalogue_selection.py:19 _BATCH_HOUSEHOLDS", + "value": "10_000 = 10,000 households per classification batch", + "enforced_at": [ + "survey_catalogue_selection.py:127-132 BATCH_LIMITS CatalogueSelectionError(ValueError) (static self-consistency: 0 < _BATCH_HOUSEHOLDS <= min(10_000, domains.MAX_HOUSEHOLDS))", + "survey_catalogue_selection.py:196-201 (the flush `if batch and (len(batch) == _BATCH_HOUSEHOLDS or ...): consume()`; no data refusal)" + ], + "refusal_codes": [ + "BATCH_LIMITS" + ], + "encodes": "no document: the number of catalogue households held between two classify_households calls", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a (row count); <= 10,000 households per batch at every fraction; the whole catalogue (1,587,376 supplied households) is classified in ~159 batches regardless of fraction because the draw follows classification", + "bytes_at_full_source": "n/a; same", + "measurement_basis": "code :139-202", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1943 plan_catalogue_selection:112", + "classification": "not-a-transport", + "notes": "A memory ceiling with no document behind it; it is what keeps survey_population_domains.MAX_HOUSEHOLDS dead. Sealed into the producer identity at :697." + }, + { + "constant": "survey_catalogue_selection.py:20 _BATCH_PEOPLE", + "value": "100_000 = 100,000 persons per classification batch", + "enforced_at": [ + "survey_catalogue_selection.py:127-132 BATCH_LIMITS CatalogueSelectionError (static: <= min(100_000, domains.MAX_TOTAL_MEMBERS))", + "survey_catalogue_selection.py:186-189 HOUSEHOLD_MEMBER_BOUND CatalogueSelectionError (`len(row.persons) <= min(domains.MAX_MEMBERS, _BATCH_PEOPLE)` = 20)", + "survey_catalogue_selection.py:196-201 flush when batch_people + len(row.persons) > _BATCH_PEOPLE" + ], + "refusal_codes": [ + "BATCH_LIMITS", + "HOUSEHOLD_MEMBER_BOUND" + ], + "encodes": "no document: persons held between two classify_households calls (the whole catalogue's 3,565,013 persons stream through)", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a (row count); <= 100,000 per batch", + "bytes_at_full_source": "n/a; same", + "measurement_basis": "code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1943 -> plan_catalogue_selection", + "classification": "not-a-transport", + "notes": "Keeps survey_population_domains.MAX_TOTAL_MEMBERS dead. Sealed at :698." + }, + { + "constant": "survey_population_domains.py:17 MAX_TOKEN_CHARS", + "value": "128 characters", + "enforced_at": [ + "survey_population_domains.py:187 LITERAL_TYPE_SIZE DomainInputError(ValueError) via _token, from _key:204, _unsigned:192, _person:291/:296/:329, classify_households:569", + "survey_population_domains.py:219 (digit width for the ASEC H_SEQ parse) and :500 (digit width for HSUP_WGT)", + "fixed-width companions in the same module: :208 SERIALNO regex `2024(?:HU|GQ)[0-9]{7}` (13 chars), :381 and :571 PERIDNUM `[0-9]{22}`" + ], + "refusal_codes": [ + "LITERAL_TYPE_SIZE", + "HOUSEHOLD_KEY", + "MEMBER_PERSON_KEY" + ], + "encodes": "one raw source literal (SERIALNO, H_SEQ, PERIDNUM, AGEP, WGTP, HSUP_WGT, ...)", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "<= 22 characters in practice (PERIDNUM 22, SERIALNO 13, H_SEQ <= 5); identical at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "code regexes", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "plan_catalogue_selection:139 consume -> domains.classify_households:545 -> classify_household -> _key/_members/_person", + "classification": "must-not-move", + "notes": "The upstream bound behind _MAX_SCALAR_BYTES and SELECTED_KEY." + }, + { + "constant": "survey_population_domains.py:18 MAX_MEMBERS", + "value": "20 persons per household", + "enforced_at": [ + "survey_population_domains.py:244 QUALIFIER_ROSTER DomainInputError", + "survey_population_domains.py:358-360 MEMBER_TUPLE_BOUND DomainInputError (_members)", + "survey_population_domains.py:558-560 MEMBER_TUPLE_BOUND (classify_households)", + "survey_catalogue_selection.py:186-188 HOUSEHOLD_MEMBER_BOUND as min(MAX_MEMBERS, _BATCH_PEOPLE)" + ], + "refusal_codes": [ + "QUALIFIER_ROSTER", + "MEMBER_TUPLE_BOUND", + "HOUSEHOLD_MEMBER_BOUND" + ], + "encodes": "no document: persons in one household (the publisher's NP/H_NUMPER top code, parsed at :459-464 with maxima 20/16)", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a (row count), <= 20", + "bytes_at_full_source": "n/a", + "measurement_basis": "code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "as MAX_TOKEN_CHARS", + "classification": "not-a-transport", + "notes": "A structural invariant of the source, not a scale bound. Sealed at survey_population_preparation.py:694." + }, + { + "constant": "survey_population_domains.py:19 MAX_HOUSEHOLDS", + "value": "100_000 households per classify_households batch", + "enforced_at": [ + "survey_population_domains.py:551 HOUSEHOLD_BATCH DomainInputError", + "survey_catalogue_selection.py:129 BATCH_LIMITS (static upper bound on _BATCH_HOUSEHOLDS)" + ], + "refusal_codes": [ + "HOUSEHOLD_BATCH", + "BATCH_LIMITS" + ], + "encodes": "no document: one batch's household count", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a; batches are capped at 10,000 by _BATCH_HOUSEHOLDS, so this is dead at any fraction", + "bytes_at_full_source": "n/a", + "measurement_basis": "code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "as MAX_TOKEN_CHARS", + "classification": "not-a-transport", + "notes": "Dead behind survey_catalogue_selection._BATCH_HOUSEHOLDS (row-count rule: batch flush at 10,000). Sealed at :695." + }, + { + "constant": "survey_population_domains.py:20 MAX_TOTAL_MEMBERS", + "value": "1_000_000 persons per classify_households batch", + "enforced_at": [ + "survey_population_domains.py:563 BATCH_MEMBER_BOUND DomainInputError", + "survey_catalogue_selection.py:131 BATCH_LIMITS (static upper bound on _BATCH_PEOPLE)" + ], + "refusal_codes": [ + "BATCH_MEMBER_BOUND", + "BATCH_LIMITS" + ], + "encodes": "no document: one batch's person count", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a; dead behind _BATCH_PEOPLE (100,000)", + "bytes_at_full_source": "n/a", + "measurement_basis": "code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "as MAX_TOKEN_CHARS", + "classification": "not-a-transport", + "notes": "Sealed at :696." + }, + { + "constant": "survey_observed_age.py:19 MAX_ROWS", + "value": "14_000_000 (comment :14-18: four times the 3,422,888 ACS persons, rounded up)", + "enforced_at": [ + "survey_observed_age.py:52 ROW_BOUND, raised as ValueError('SURVEY_OBSERVED_AGE_ROW_BOUND') (:37-39)" + ], + "refusal_codes": [ + "SURVEY_OBSERVED_AGE_ROW_BOUND" + ], + "encodes": "no document: len(raw_age) for one channel's person rows at a time (_normalized_source_copy :1079-1083 normalizes ACS and ASEC separately)", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a (row count): ACS 342,288 / ASEC 14,212 persons", + "bytes_at_full_source": "n/a: ACS 3,422,888 (4.09x headroom) / ASEC 142,125", + "measurement_basis": "roster-census.json counts; code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1974 _normalized_source_copy:1079 -> normalize_observed_age:44; also _verify_normalized_copy:1140", + "classification": "not-a-transport", + "notes": "Base value = head value (14,000,000 already on origin/native-row-ceilings). Sealed at survey_population_preparation.py:701." + }, + { + "constant": "survey_observed_age.py:20 MAX_EXACT_FLOAT64_INTEGER", + "value": "2**53 = 9,007,199,254,740,992", + "enforced_at": [ + "survey_observed_age.py:60 RAW_REPRESENTATION ValueError('SURVEY_OBSERVED_AGE_RAW_REPRESENTATION')", + "survey_observed_age.py:81 INCUMBENT_REPRESENTATION ValueError" + ], + "refusal_codes": [ + "SURVEY_OBSERVED_AGE_RAW_REPRESENTATION", + "SURVEY_OBSERVED_AGE_INCUMBENT_REPRESENTATION" + ], + "encodes": "a fixed-width field: the float64 exact-integer range of the derived `age` column (docstring :49-51: 'The float bound is a representation limit')", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "8 bytes per cell; values <= 99; independent of the roster", + "bytes_at_full_source": "same", + "measurement_basis": "code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "as MAX_ROWS", + "classification": "must-not-move", + "notes": "docs/us-native-row-ceilings.md line 188 lists it under 'A fixed-width encoding'. Sealed at survey_population_preparation.py:702." + }, + { + "constant": "operator_column_contracts.py:64 PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID (defined outside this family; enforced in spine_assembly)", + "value": "10**15 - 1 = 999,999,999,999,999", + "enforced_at": [ + "spine_assembly.py:630-640 ValueError (every entity's source ids per channel in _validate_source_frame)", + "spine_assembly.py:763-767 ValueError (after collision remap in _id_offsets); companion int64 overflow check :756 (`int(ids.max()) + offset > int(info.max)`)", + "puf_support._remap_ids re-check (other family; comment puf_support.py:2324-2328)" + ], + "refusal_codes": [ + "ValueError (no static code)" + ], + "encodes": "a fixed-width field: the magnitude of an entity id so the clone stage's decimal remap `id + clone_index * 10**digits(max_id)` stays inside int64", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "stacked ids <= 356,501 persons (6 digits), far below 10**15; 8 bytes per id", + "bytes_at_full_source": "stacked ids <= 3,565,013 persons (7 digits); clone ids <= 7 + 10**7 digits", + "measurement_basis": "roster-census.json counts; code", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "prepare_authenticated_survey_population:1977 spine_assembly.stack_survey_spines:322 -> _stack_source_tables:213 -> _validate_source_frame:596 / _id_offsets:732", + "classification": "must-not-move", + "notes": "spine_assembly has no byte constant of its own; its digests (_numeric_digest :170, _channel_digest :175) stream contiguous copies (8 B x households) with no cap." + }, + { + "constant": "native_household_origin.py:48 MAX_SOURCE_BYTES", + "value": "512 * 1024**2 = 536,870,912", + "enforced_at": [ + "native_household_origin.py:125-130 ASEC_MEMBER_BOUNDS NativeOriginError(ValueError) (`0 < self.size_bytes <= MAX_SOURCE_BYTES` on each pinned hhpub member)", + "native_household_origin.py:196 SOURCE_SIZE NativeOriginError (`len(payload) <= MAX_SOURCE_BYTES` in _source_document, before :198 json.loads and :199 SOURCE_CANONICAL `canonical_json(value) == payload`, a consumer re-encoding whose encoder cap is this same 512 MiB by order of checks); reached from _issue:241-244, AuthenticatedNativeOriginSource.document:189, graph_native_household_origin.py:180 and :190" + ], + "refusal_codes": [ + "ASEC_MEMBER_BOUNDS", + "SOURCE_SIZE", + "SOURCE_CANONICAL" + ], + "encodes": "(a) three upstream files hhpub23/24/25.csv; (b) the origin source document: one `[period, native_id, member_index, origin_key]` record per ACS housing record (all 1,631,969, vacancies included: produce_acs_native_origins :279-283 iterates the whole housing source) or per ASEC hhpub row (267,383 across three cohorts), fraction-invariant", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "(a) 30,259,450 / 30,448,089 / 33,298,479 B (pins :150-176); (b) ACS 92 B x 1,631,969 = 150,141,148 B (0.280 x), ASEC 82 B x 267,383 = 21,925,406 B; identical at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "prep-measurements.json native_household_origin.source_document (canonical_json on invented records at 13-char SERIALNO / 5-digit H_SEQ / 64-hex key); counts from roster-census.json catalogues.acs.households and the pins' rows", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Only graph_native_household_origin.NativeACSOriginKernel.run:69-82 / NativeASECOriginKernel.run:89-104 / verify_materialized_population_origins:325; those nodes are declared by native_origin_source_nodes:239 and appear in neither the 19-node nor the 45-node roster (brief section 2) and no runner in this family imports graph_native_household_origin.", + "classification": "must-not-move", + "notes": "Base value = head value. The :199 re-encoding holds the source document twice (payload and canonical_json copy)." + }, + { + "constant": "native_household_origin.py:129 0 < self.rows <= 1_000_000", + "value": "1_000_000 rows per pinned hhpub member", + "enforced_at": [ + "native_household_origin.py:126-130 ASEC_MEMBER_BOUNDS NativeOriginError; the pin's rows is then enforced on the file at :350 and :352 ASEC_MEMBER_ROWS" + ], + "refusal_codes": [ + "ASEC_MEMBER_BOUNDS", + "ASEC_MEMBER_ROWS" + ], + "encodes": "no document: an upstream file's row count (88,978 / 89,473 / 88,932 pinned)", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a (row count); fraction-invariant", + "bytes_at_full_source": "n/a", + "measurement_basis": "pins :150-176", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "as MAX_SOURCE_BYTES", + "classification": "not-a-transport", + "notes": "A row count on an upstream file; must-not-move by nature though classified as the row-count rule the brief gives." + }, + { + "constant": "native_household_origin.py:81 [1-9][0-9]{0,17} and :335 [0-9]{1,18}", + "value": "<= 18 decimal digits for an ASEC H_SEQ token", + "enforced_at": [ + "native_household_origin.py:80-83 CARRIED_NATIVE_INTEGER NativeOriginError (_carried_native_integer on a carried asec_H_SEQ cell)", + "native_household_origin.py:334-338 ASEC_MEMBER_NATIVE_KEY NativeOriginError (produce_asec_native_origins per hhpub row)" + ], + "refusal_codes": [ + "CARRIED_NATIVE_INTEGER", + "ASEC_MEMBER_NATIVE_KEY" + ], + "encodes": "one native id token (real width <= 5 digits)", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "<= 5 characters; fraction-invariant", + "bytes_at_full_source": "same", + "measurement_basis": "code; survey_population_domains.py:219 (H_SEQ <= 99999)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "as MAX_SOURCE_BYTES; :81 also from _bind_population_origins:530", + "classification": "must-not-move", + "notes": "Token width only." + }, + { + "constant": "graph_native_household_origin.py:210 canonical_json(doc) == value.payload and :408 canonical_json(document) == artifact.payload (consumer re-encodings of the population-origin binding; encoder cap: none)", + "value": "no cap: microcosm.graph.canonical.canonical_json is json.dumps with no size bound (canonical.py:43-56); native_household_origin.py:636 produces the binding with the same encoder and :394 json.loads decodes it whole", + "enforced_at": [ + "graph_native_household_origin.py:208-212 ORIGIN_PARENT_PAYLOAD NativeOriginError (parent binding: json.loads then canonical_json == payload)", + "graph_native_household_origin.py:404-409 ORIGIN_MATERIALIZED_LINEAGE NativeOriginError (after a full bind_population_origins rebuild: `document == expected and canonical_json(document) == artifact.payload`)", + "native_household_origin.py:636-650 (producer canonical_json of the whole document, uncapped); :394 PopulationOriginBinding.document json.loads (uncapped)" + ], + "refusal_codes": [ + "ORIGIN_PARENT_PAYLOAD", + "ORIGIN_MATERIALIZED_LINEAGE" + ], + "encodes": "the population-origin binding document: one row per household of the bound population (202 B) plus one row per entity row of all six entities (229 B; 353 B per person with membership_source_ids) plus per-entity ordered-id digests; roster = the bound population's households/persons/groups (stacked, or clone when bound after the EXPAND)", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "379,709,051 B at the stacked 1/10 population (158,737 households, 356,501 persons, groups SCALED) through canonical_json at full-width invented rows; a second whole copy on every re-encode", + "bytes_at_full_source": "3,797,101,483 B at the stacked full population; 7,594,203,653 B at the clone population (3,174,752 households, 7,130,026 persons); no cap anywhere", + "measurement_basis": "prep-measurements.json native_household_origin.binding_document (canonical_json per row + comma, counts from roster-census.json, groups SCALED by the 1/1000 ratios)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "PopulationOriginBindingKernel.run:146-244 and verify_materialized_population_origins:325-413 only; not declared in either pilot graph (brief section 2)", + "classification": "consumer", + "notes": "Recorded because the brief counts a consumer-side re-encoding as a bound even without an explicit cap; the encoder's cap is 'none'. The binds flags are false only because there is no value to exceed; at full source the document is 3.8-7.6 GB held at least twice (:404-409 holds document, expected, canonical_json(document) and artifact.payload). graph_combined_clone's lineage digest (:216-217, :310-318) uses the same uncapped canonical_json over [[source_id, 1, target_id]] per base row and IS on the 19-node path (see open questions)." + }, + { + "constant": "graph_sources.py:441-442 canonical_json(preparation) != prepared.payload (consumer re-encoding of the stacked-spine preparation receipt; encoder cap: none)", + "value": "no cap: canonical_json; the producer USAssemblyCreateKernel.run:395 emits `canonical_json(_json_data(result.receipt))` and the consumer USSpineHarmonizeKernel.run:441 json.loads then re-encodes", + "enforced_at": [ + "graph_sources.py:441-443 ValueError('US spine preparation must be canonical JSON.')" + ], + "refusal_codes": [ + "ValueError (no static code)" + ], + "encodes": "stacked_spine.prepare_stacked_spine's receipt (:1005-1021): sampling receipts, assembly_context (spine_assembly:223-248: counts, digests, `household_order` = one int per stacked household when the concatenated household ids are not already sorted, else None; source_mass_logs), assembly_metadata, acs_native_group_quarters digests. Roster-shaped only through household_order.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "not measured (not on any path): fixed part ~ a few KB; household_order, when present, ~8 B per stacked household (1.3 MB at 1/10)", + "bytes_at_full_source": "not measured; household_order ~12.7 MB at 1,587,376 households when present", + "measurement_basis": "code reading; the legacy assembly graph is outside the pilot rosters so no measurement effort was spent (brief section 2)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "us_assembly_graph:483 / us_assembly_registry:562 only; graph_sources' star exports and _stacked_alias:171 are how stacked_spine is reached, and survey_population_preparation imports graph_sources only for _canonical_assembly:1979 and frame_column_declarations", + "classification": "consumer", + "notes": "graph_sources also emits the frame context at :470 via canonical_json (fixed). stacked_spine.py carries no byte constant at all (14,231 lines read): its json.dumps materialisations are _integer_rows_sha256 (:1987-1995, every native ACS household and person lineage as 4- and 5-int rows) and _canonical_sha256 (:2712-2719), both uncapped and off both paths." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prep/measure_prep.py", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prep/prep-measurements.json (receipt)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prep/run.log", + "cited: experiments/native-byte-transports/byte-census-measurements.json (predictor_projection, selection_block, origin_budget); experiments/native-scale-transport/ceiling-receipt.json (replicated preparation receipt 109,804,304 B at 1/10 and 1,099,892,722 B at full)" + ], + "open_questions": [ + "The recovered 1/1000 artifact (_recovered/pilot-runs/native19-required-20260912/... and native45-v5/) is not present in this worktree or at the path the brief gives, so the 45,922 B fixed preamble, the 168 B request size and the graph.json node rosters were taken from the row-ceilings census, roster-census.json and the brief rather than re-read; the per-row and roster bytes above are this lane's own measurements through the module encoders on invented full-width rows and sit 12-15% above the transport lane's replicated-artifact figures because invented ids use maximal 7-digit widths and unreduced 1/10 inclusion fractions.", + "graph_survey_population.py and survey_origin_budget.py differ in the working tree from HEAD 77bafbb3a (git diff HEAD): _json_matches is added, _json_sha256 loses its segment parameter, and survey_origin_budget:935 switches to _json_matches. All cited line numbers are working-tree lines; a verifier reading the committed HEAD will find the _json_sha256/_json_matches rows shifted or absent.", + "Uncapped whole-roster materialisations on the 19-node path that have no bound and therefore no row: (1) graph_combined_clone._digest:216-217 encodes `[[source_id, 1, target_id], ...]` for every base row of every entity via canonical_json (about 25 B per row: stacked persons 3,565,013 -> ~90 MB, all six entities ~250 MB per EXPAND), computed in USCombinedSurveyCloneExpandKernel.run:502 and again in run_authenticated_survey_population.observe:1150-1152 and _verify_cloned_frame; (2) survey_population_preparation._frame_identity column blobs (_float_cells 37+37 B x rows per float column, _string_cells/_integer_cells whole-column bytes) and graph_context._row_identity:113-115 (8 B x rows per entity); (3) graph_survey_population._final_artifact:899-900 and graph_atomic_survey_population:264 read a second whole copy of the preparation receipt and allocation payload from the store for an equality check, and _checked_preparation:397 json.loads the whole receipt into the view on every borrow. None refuses; each is a multiple of full source with no ceiling.", + "survey_population_preparation._file_limits enforces four constants defined in other families (acs_population_catalogue._ARCHIVE_BYTES, asec_2024_native_population._MAX_CHECKPOINT_BYTES, asec_person_coverage_source.MAX_MEMBER_BYTES, asec_original_household_weights._MAX_MEMBER_BYTES); they are recorded here as rows keyed by their definitions so the owning families' rows can absorb the enforcement sites (:426, :506-514, :551-552, :561). The real sizes of asec/parent.h5 and asec/household-attachment.h5 are not pinned anywhere in this tree and neither census established them.", + "The wage projection (_current_survey_wage_projection:2097, rows = ASEC persons, 47 B/row, 6.68 MB at full source) is a whole-roster receipt through _roster_payload but is reached only from graph_puf_diagnostic_consumer.qualify_current_survey_host:393, whose callers (graph_current_survey_puf_transfer, graph_puf_diagnostic_consumer) the brief lists as off-path; it is folded into the MAX_SEGMENT_BYTES/MAX_ROSTER_BYTES rows as reachable_from none.", + "The binds_at_* flags for PREPARATION_ROSTER_BYTES and the segment rows follow the brief's literal rule (measured bytes exceed the base value / the document exceeds one segment); a reader who wants 'refuses at head' should use the head_value column and the classification.", + "stacked_spine.py (14,231 lines) and graph_implementation.py, support_provenance.py, survey_population_replay.py, graph_atomic_survey_clone.py and graph_context.py contain no byte constant, encoder limit argument, len(payload) check, read cap or struct-width product; their canonical_json/json.dumps uses are either fixed-shape (replay metadata/mass_log, implementation manifests, frame context) or uncapped off-path materialisations (stacked_spine _integer_rows_sha256, _canonical_sha256)." + ] + }, + { + "family": "Property income, property tax and completion", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_property.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income_receipts.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_property_income_sources.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/property_income_constants.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_completion.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_completion_host.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_child_property_income.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_child_property_income_source.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_property_completion_routing.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_dividend_source.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py", + "packages/microcosm-fit/src/microcosm/fit/graph_joint_empirical.py", + "packages/microcosm-fit/src/microcosm/fit/joint_empirical.py", + "packages/microcosm-fit/src/microcosm/fit/_graph_legacy_qrf.py (encode_json/decode_json/artifact/encode_raw_target/read_raw_target, :18-200)", + "packages/microcosm-fit/src/microcosm/fit/model_input.py (:1-140)", + "packages/microcosm-fit/src/microcosm/fit/qrf.py (_IndexIdentity :577-638)", + "packages/microcosm-fit/src/microcosm/fit/graph_signed_reconciliation.py (:212-240)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py (:30-45, :74-76, :225-345)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py (:90-155)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py (:110-175)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py (:355-480, :820-900, :1275-1360, :1460-1520, :1575-1600, :1685-1900)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py (_states :79-135)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py (_check_node_states :903-925)", + "packages/microcosm-graph/src/microcosm/graph/canonical.py (:43-58)", + "packages/microcosm-graph/src/microcosm/graph/manifest.py (receipt freezing, to_json_bytes :733)", + "experiments/native-byte-transports/census-brief.md", + "experiments/native-byte-transports/byte-census-measurements.json", + "experiments/native-byte-transports/byte_census_measure.py", + "experiments/native-row-ceilings/census.json (family entries)", + "experiments/native-row-ceilings/roster-census.json", + "docs/us-native-row-ceilings.md (:220-240)", + "/Users/maxghenis/PolicyEngine/_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json (origins shape)", + "/Users/maxghenis/PolicyEngine/_recovered/pilot-runs/native45-v5/run/execution-config.proposed.json, PROGRESS.md, out.md (options only; no manifest was exported)", + "git diff origin/native-row-ceilings for graph_child_property_income.py, current_child_property_income_source.py, graph_joint_empirical.py" + ], + "rows": [ + { + "constant": "graph_child_property_income.py:79 MAX_ARTIFACT_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "graph_child_property_income.py:110 (_json, per-segment accumulation: `require(len(current) + len(encoded) <= MAX_ARTIFACT_BYTES, \"GRAPH_ARTIFACT_SIZE\")`; require is child.require, ValueError(\"CHILD_PROPERTY_GRAPH_ARTIFACT_SIZE\"))" + ], + "refusal_codes": [ + "GRAPH_ARTIFACT_SIZE (ValueError CHILD_PROPERTY_GRAPH_ARTIFACT_SIZE)" + ], + "encodes": "Every document this module's _json encodes. Roster-shaped: the child draw document (one row per eligible child: adapter._draw_document rows support_id/coordinate/values/pattern/donor_key, encoded at :691 in _reconstruct) and the completion document (:728, which re-embeds every draw row as original_draws). Fixed-shape: donor/recipient projections (:389/:404 digests+counts), scenario (:542/:760/:945), model metadata (:694), verification (:745), host_pins (:550/:947-948), receipt stamps (:1176/:1190).", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "draw 11.9-19.0 MB (65,952 eligible children ESTIMATE x 180.5-288.5 B + 770 fixed); completion the same + ~1 KB", + "bytes_at_full_source": "draw 119.0-190.3 MB (659,527 eligible children ESTIMATE x 180.5-288.5 B); completion the same + ~1 KB", + "measurement_basis": "Measured through graph_child_property_income._json(adapter._draw_document(...)) at full-source widths (7-digit support ids, 13-char ACS SERIALNO keys, 22-char ASEC PERIDNUM donor keys) as a difference of 2 and 2,002 rows: 288.5 B/row for all-positive (pattern 3) rows, 180.5 B/row for zero-pattern rows (donor_key null), fixed 770 B; asserted byte-identical to adapter._json. byte-census-measurements.json child_draw gives 273.0 B/row with a shorter invented donor key. Eligible-child count is an ESTIMATE (18.5% of stacked persons, the row-count lane's share; no recovered artifact records eligible_child_rows and the 45-node pilot exported no manifest). Scripts: scratchpad/census/prop/prop_measure.py, prop_measure_2.py; receipts prop-measurements*.json, prop-derived-totals.json.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "64 * 1024**2 = 67,108,864 as a whole-document cap: base _json was `payload = adapter._json(value); require(len(payload) <= MAX_ARTIFACT_BYTES, \"GRAPH_ARTIFACT_SIZE\")`", + "head_value": "64 * 1024**2 = 67,108,864 as a per-segment ceiling inside json.JSONEncoder.iterencode accumulation (:102-116); the whole total is bounded by MAX_ROSTER_BYTES", + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> graph_survey_completion_host.extend -> _CompletionHost.__init__ -> child.ChildPropertyBoundary.__init__ -> child_property_nodes -> _projections -> _json; _extend -> _child_expected -> _reconstruct -> _json (:691 draw, :728 completion); host.run_graph -> _ChildKernel.run -> _reconstruct; boundary.verify -> ChildPropertyBoundary.verify_materialized -> _reconstruct", + "classification": "segment", + "notes": "binds_at_full_source is true at the base value's whole-document semantics (119 MB lower bound > 64 MiB; the verdict holds for any pattern mix once eligible children exceed 371,800, i.e. 10.4% of stacked persons). At head the per-segment check cannot refuse a roster (a segment closes at :106 before it exceeds 64 MiB), so it does not bind at any fraction. Segments are joined once at :116 `payload = b\"\".join(segments)`, so the whole payload is still materialised after the check." + }, + { + "constant": "graph_child_property_income.py:86 MAX_ROSTER_BYTES", + "value": "64 * MAX_ARTIFACT_BYTES = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "graph_child_property_income.py:112 (_json running total: `require(total + len(encoded) <= MAX_ROSTER_BYTES, \"GRAPH_ARTIFACT_LIMIT\")`)", + "graph_child_property_income.py:117 (joined payload: `require(len(payload) == total <= MAX_ROSTER_BYTES, \"GRAPH_ARTIFACT_LIMIT\")`)" + ], + "refusal_codes": [ + "GRAPH_ARTIFACT_LIMIT (ValueError CHILD_PROPERTY_GRAPH_ARTIFACT_LIMIT)" + ], + "encodes": "The same documents as MAX_ARTIFACT_BYTES; the roster-shaped ones are the child draw document (eligible children) and the completion document (re-embedded draw rows).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "draw 11.9-19.0 MB; completion the same + ~1 KB (65,952 eligible children ESTIMATE)", + "bytes_at_full_source": "draw 119.0-190.3 MB; completion the same + ~1 KB (659,527 eligible children ESTIMATE)", + "measurement_basis": "Same measurement as the MAX_ARTIFACT_BYTES row (prop_measure.py child_draw, prop_measure_2.py draw_rows): 180.5-288.5 B per eligible child through the module's own _json. At 288.5 B the 4 GiB total admits 14,887,286 children, more than four times the stacked person roster.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none: the base branch had no whole-roster total; its only cap was the whole-document MAX_ARTIFACT_BYTES", + "head_value": "64 * MAX_ARTIFACT_BYTES = 4,294,967,296", + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> graph_survey_completion_host.extend -> _CompletionHost.__init__/_extend -> child._reconstruct -> _json (:691, :728); host.run_graph -> _ChildKernel.run; boundary.verify -> verify_materialized -> _reconstruct", + "classification": "segment", + "notes": ":112 streams (running total during iterencode); :117 is checked on the joined whole payload, so the bytes are materialised once at the end. Headroom at full source is about 22x on the all-positive upper bound. The comment at :85 cites docs/us-native-byte-transports.md, which does not exist in this tree." + }, + { + "constant": "graph_child_property_income.py:89 MAX_ORIGINALS", + "value": "15_000_000", + "enforced_at": [ + "graph_child_property_income.py:340 (_projections: `0 < len(recipients) <= MAX_ORIGINALS`, \"ORIGINAL_RECIPIENT_ROSTER\")" + ], + "refusal_codes": [ + "ORIGINAL_RECIPIENT_ROSTER (ValueError CHILD_PROPERTY_ORIGINAL_RECIPIENT_ROSTER)" + ], + "encodes": "A row count, not bytes: qualified.recipients, the stacked person roster (index-identical to the origins persons table, :337-339).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "not a byte count: 356,501 stacked persons", + "bytes_at_full_source": "not a byte count: 3,565,013 stacked persons", + "measurement_basis": "Brief \u00a73 stacked person counts; the site compares len() of a DataFrame.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "600_000", + "head_value": "15_000_000", + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> extend -> _CompletionHost.__init__ -> ChildPropertyBoundary.__init__ -> child_property_nodes (:957) -> _projections (:539) -> :340", + "classification": "not-a-transport", + "notes": "Row count wearing no byte name; listed because brief \u00a75 records it as moved. binds_at_full_source is true at the base value (3,565,013 > 600,000); false at head. Row-count rule (comment :87-88): four times the measured full-source stacked person roster, rounded up to the next whole million (docs/us-native-row-ceilings.md)." + }, + { + "constant": "graph_child_property_income.py:138 0 < len(self.scenario_id) <= 256", + "value": "256 characters", + "enforced_at": [ + "graph_child_property_income.py:136-144 (ChildPropertyOptions.document, \"SCENARIO_CONTRACT\")" + ], + "refusal_codes": [ + "SCENARIO_CONTRACT (ValueError CHILD_PROPERTY_SCENARIO_CONTRACT)" + ], + "encodes": "A single scalar token: the operator's scenario_id string inside the scenario document.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "38 characters in the 45-node pilot (\"us-native-completion45-v5-candidate\"); fraction-independent", + "bytes_at_full_source": "same 38 characters; fraction-independent", + "measurement_basis": "execution-config.proposed.json child_property_document.scenario_id; a scalar, not a roster.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> graph_survey_completion_host.validate_options -> options.document() (:45); also ChildPropertyBoundary.__init__ :945 and _pure :976", + "classification": "must-not-move", + "notes": "Token cap on a scalar; nothing grows with the fraction." + }, + { + "constant": "graph_child_property_income.py:812 value.payload == expected[edge.producer, edge.artifact]", + "value": "consumer-side byte equality against bytes re-encoded through the segmented _json (encoder cap MAX_ROSTER_BYTES = 4,294,967,296; per segment MAX_ARTIFACT_BYTES = 67,108,864)", + "enforced_at": [ + "graph_child_property_income.py:811-814 (_artifacts, \"RECONSTRUCTED_ARTIFACT_CHANGED\"), reached from :1040 and :1062 (verify_materialized), :1153 (ATTACH/VERIFY kernels); :1130 passes expected={} for DONOR/RECIPIENT so only host pins are checked there" + ], + "refusal_codes": [ + "RECONSTRUCTED_ARTIFACT_CHANGED (ValueError CHILD_PROPERTY_RECONSTRUCTED_ARTIFACT_CHANGED)" + ], + "encodes": "Roster-shaped: the draw document produced in the graph run by microcosm.fit.graph_joint_empirical.JointEmpiricalDrawKernel.run (:567-568, adapter._json, one json.dumps) and the completion document produced by the ATTACH kernel; both are re-produced by _reconstruct (:691, :728) through the segmented _json and compared whole. Fixed-shape: projections, scenario, model, metadata.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "draw 11.9-19.0 MB and completion the same + ~1 KB (65,952 eligible children ESTIMATE)", + "bytes_at_full_source": "draw 119.0-190.3 MB and completion the same + ~1 KB (659,527 eligible children ESTIMATE)", + "measurement_basis": "Same encoder measurement as the MAX_ARTIFACT_BYTES row (180.5-288.5 B per eligible child). The check has no cap of its own; the re-encoder's total is MAX_ROSTER_BYTES.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> extend -> host.run_graph -> _ChildKernel.run (ATTACH_REF/VERIFY_REF) -> _reconstruct -> _artifacts (:1153) -> :812; _extend -> boundary.verify -> ChildPropertyBoundary.verify_materialized -> _artifacts (:1040, :1062)", + "classification": "consumer", + "notes": "The stored draw bytes come from the fit adapter's kernel, an unsegmented json.dumps under MAX_DRAW_BYTES (graph_joint_empirical.py:568-569); the comparison here re-materialises the same bytes through the segmented host encoder. Two whole copies of the draw document exist at the comparison. _reconstruct also does `json.loads(draw_payload)[\"rows\"]` (:737) to re-embed the rows into the completion document." + }, + { + "constant": "graph_joint_empirical.py:48 MAX_DRAW_BYTES", + "value": "64 * 64 * 1024**2 = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "graph_joint_empirical.py:569 (JointEmpiricalDrawKernel.run: `_require(len(payload) <= MAX_DRAW_BYTES, \"DRAW_BYTES\")` on `payload = _json(document)` at :568, a single json.dumps)", + "graph_joint_empirical.py:84 (_document default limit, \"PAYLOAD_BYTES\"): reached for support_json (:108), transport_json (:143) and model_metadata (:542), all fixed-shape; for the draw document only via read_joint_empirical_draw :629, which no runner calls (only packages/microcosm-fit/tests/test_graph_joint_empirical.py:193)" + ], + "refusal_codes": [ + "DRAW_BYTES (ValueError GRAPH_JOINT_EMPIRICAL_DRAW_BYTES)", + "PAYLOAD_BYTES (ValueError GRAPH_JOINT_EMPIRICAL_PAYLOAD_BYTES)" + ], + "encodes": "The child draw document (one row per eligible child) as produced by the draw kernel in the graph run; adapter._json is json.dumps with no segmentation.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "11.9-19.0 MB (65,952 eligible children ESTIMATE x 180.5-288.5 B + 770)", + "bytes_at_full_source": "119.0-190.3 MB (659,527 eligible children ESTIMATE x 180.5-288.5 B)", + "measurement_basis": "prop_measure.py child_draw asserts adapter._json(document) == graph_child_property_income._json(document) byte for byte, so the per-row figures (180.5 zero-pattern, 288.5 all-positive) apply. byte-census-measurements.json child_draw: 273.0 B/row, children_the_64MiB_cap_admits 245,820.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "64 * 1024**2 = 67,108,864", + "head_value": "64 * 64 * 1024**2 = 4,294,967,296", + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> extend -> host.run_graph -> adapter.JointEmpiricalDrawKernel.run -> :568 _json(document) -> :569", + "classification": "consumer", + "notes": "binds_at_full_source true at the base value (119 MB lower bound > 64 MiB). Brief \u00a75 and the comment at :44-47 describe this as the decoder's ceiling on the producer's segmented total, but the reachable enforcement (:569) is the kernel's own producer-side check on an unsegmented json.dumps; the decoder path (`_document(payload)` on a draw) is reachable only through read_joint_empirical_draw, which has no runner caller. The receipt at :572-583 carries only counts and digests." + }, + { + "constant": "graph_joint_empirical.py:87 _json(value) == payload", + "value": "consumer-side canonical re-encoding inside _document (encoder _json has no cap; the preceding length check at :84 uses the limit argument)", + "enforced_at": [ + "graph_joint_empirical.py:83-90 (_document, \"CANONICAL_JSON\"); callers on the runner path: :108 _support (support_json), :143 _transport (transport_json), :371 _metadata (model bytes, limit=empirical.MAX_MODEL_BYTES), :542 draw kernel (model_metadata payload); :629 read_joint_empirical_draw (draw document) has no runner caller" + ], + "refusal_codes": [ + "CANONICAL_JSON (ValueError GRAPH_JOINT_EMPIRICAL_CANONICAL_JSON)", + "PAYLOAD_BYTES (ValueError GRAPH_JOINT_EMPIRICAL_PAYLOAD_BYTES) at :84", + "GRAPH_JOINT_EMPIRICAL_JSON at :89 on decode errors" + ], + "encodes": "On the reachable path the largest document is the joint-empirical model (one donor record per ASEC 15-17 person with a known O/D pair; upstream, fraction-independent); support, transport and metadata are fixed-shape. The roster-shaped draw document only reaches this check through read_joint_empirical_draw (unreachable).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "model ~1.25 MB at an ESTIMATED ~5,400 teen donors (231 B/donor + 437); hard ceiling 32.8 MB if every one of the 142,125 ASEC persons were a donor; identical at every fraction", + "bytes_at_full_source": "same as at 1/10 (the donor roster is the whole 2025 ASEC member cohort, current_child_property_income_source.py:639-640 donor_scope complete_original_current_year_source, donor_sampling_fraction_applied False)", + "measurement_basis": "prop_measure.py model_document: joint_empirical.fit_joint_empirical(...).to_bytes() on invented donors at full widths, 4 vs 1,004 rows: 231.0 B/donor, fixed 437 B. Donor count not established from any artifact (ESTIMATE of the ASEC 15-17 share).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> host.run_graph -> adapter.JointEmpiricalFitKernel.run -> _metadata (:465) -> _document(model.to_bytes(), MAX_MODEL_BYTES) (:371); JointEmpiricalDrawKernel.run -> _metadata (:537), _document(metadata_value.payload) (:542), _support/_transport via _draw_expected (:404-407)", + "classification": "stays", + "notes": "Headroom against MAX_MODEL_BYTES is at least 2.0x even at the impossible maximum donor count and about 54x at the estimate. read_joint_empirical_draw (:587) and its _document(payload) at :629 are `reachable_from: none`; the function that would reach it is only the fit package's test." + }, + { + "constant": "graph_joint_empirical.py:49 MAX_RECIPIENTS", + "value": "1_048_576", + "enforced_at": [ + "graph_joint_empirical.py:530 (JointEmpiricalDrawKernel.run: `_require(len(table) <= MAX_RECIPIENTS, \"RECIPIENT_COUNT\")` on the whole RECIPIENT support table, before the eligibility filter at :552)", + "graph_joint_empirical.py:614 (read_joint_empirical_draw, \"RECIPIENT_COUNT\"; no runner caller)" + ], + "refusal_codes": [ + "RECIPIENT_COUNT (ValueError GRAPH_JOINT_EMPIRICAL_RECIPIENT_COUNT)" + ], + "encodes": "A row count, not bytes: the RECIPIENT support population's person table, which graph_child_property_income._projections builds from every stacked person (:378 `recipient_table = _key_table(tuple(recipients.recipient_key))`, comment :376-377 'A nonempty original-recipient Frame remains valid when no child is eligible. The generic draw declares this bool and selects internally.'), so len(table) is the stacked person roster.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "not a byte count: 356,501 stacked persons", + "bytes_at_full_source": "not a byte count: 3,565,013 stacked persons", + "measurement_basis": "Brief \u00a73 stacked person counts; _table(context) at :339 returns the full projected table and :530 checks it before `table = table.loc[table[eligibility]]` at :552.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "1_048_576 (unchanged on this branch)", + "head_value": "1_048_576", + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> extend -> host.run_graph -> adapter.JointEmpiricalDrawKernel.run -> _table (:528) -> :530", + "classification": "not-a-transport", + "notes": "Binds at full source (3,565,013 > 1,048,576; first binds near fraction 0.294) and was not lifted on this branch, unlike MAX_ORIGINALS and MAX_RECIPIENT_ROWS which guard the same roster upstream and were raised to 15,000,000. It is the first refusal the child lane would meet at full source now that the two upstream row ceilings moved. No row-count rule from docs/us-native-row-ceilings.md has been applied to it." + }, + { + "constant": "graph_joint_empirical.py:67 _hash len(value) == 64", + "value": "64 lowercase hex characters (sha256)", + "enforced_at": [ + "graph_joint_empirical.py:67-72 (_hash), used at :198 (\"SOURCE_HASH\"), :267 (\"DRAW_HASHES\"), :623 (\"READER_HASHES\", unreachable), and graph_child_property_income.py:490 (\"HOST_PIN\", every host pin value)" + ], + "refusal_codes": [ + "SOURCE_HASH", + "DRAW_HASHES", + "HOST_PIN (ValueError CHILD_PROPERTY_HOST_PIN)" + ], + "encodes": "A fixed-width scalar field: sha256 hex digests in node params and host pins.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 bytes per digest; fraction-independent", + "bytes_at_full_source": "64 bytes per digest; fraction-independent", + "measurement_basis": "Fixed-width field by definition.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> ChildPropertyBoundary.__init__ -> child_property_nodes -> _bindings (:520) -> :490; joint_empirical_fit_node/draw_node (:582/:598) -> :198/:267", + "classification": "must-not-move", + "notes": "An equality on a digest width, not a capacity bound." + }, + { + "constant": "joint_empirical.py:24 MAX_MODEL_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "joint_empirical.py:376-379 (JointEmpiricalModel._checked: `0 < len(self.payload) <= MAX_MODEL_BYTES`, \"MODEL_BYTES\"), reached by from_bytes :391, to_bytes :395, sha256 :400, diagnostics :404, draw_joint_empirical :498", + "graph_joint_empirical.py:371 (_metadata: `_document(model.to_bytes(), empirical.MAX_MODEL_BYTES)`, \"PAYLOAD_BYTES\")" + ], + "refusal_codes": [ + "MODEL_BYTES (ValueError JOINT_EMPIRICAL_MODEL_BYTES)", + "PAYLOAD_BYTES (ValueError GRAPH_JOINT_EMPIRICAL_PAYLOAD_BYTES)" + ], + "encodes": "The joint-empirical model payload: protocol, targets, support, weight_kind and one donor record per ASEC 15-17 person with a known O/D pair (key, household, values, weight) \u2014 an upstream roster read whole at every fraction.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "~1.25 MB at an ESTIMATED ~5,400 teen donors (231 B/donor + 437); at most 32.8 MB if all 142,125 ASEC persons were donors; identical at every fraction", + "bytes_at_full_source": "same as at 1/10", + "measurement_basis": "prop_measure.py model_document through joint_empirical.fit_joint_empirical(...).to_bytes(): 231.0 B/donor, 437 B fixed; 64 MiB admits 290,514 donors, more than double the whole ASEC member.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> _child_expected -> child._reconstruct -> empirical.fit_joint_empirical (:667) -> JointEmpiricalModel.from_bytes (:483) -> _checked; host.run_graph -> JointEmpiricalFitKernel.run (:457) and JointEmpiricalDrawKernel.run -> from_bytes (:536)", + "classification": "stays", + "notes": "Headroom at least 2.0x at the impossible maximum, ~54x at the estimate. Every _checked() call re-parses and re-encodes the whole model (see the :385 row)." + }, + { + "constant": "joint_empirical.py:25 MAX_DONORS", + "value": "1_048_576", + "enforced_at": [ + "joint_empirical.py:304 (_prepare: `1 <= len(rows) <= MAX_DONORS`, \"DONOR_COUNT\")", + "joint_empirical.py:457-461 (fit_joint_empirical: `1 <= len(frame) <= MAX_DONORS`, \"DONOR_AXIS\")" + ], + "refusal_codes": [ + "DONOR_COUNT (ValueError JOINT_EMPIRICAL_DONOR_COUNT)", + "DONOR_AXIS (ValueError JOINT_EMPIRICAL_DONOR_AXIS)" + ], + "encodes": "A row count, not bytes: donor records in the model (ASEC 15-17 known-pair cohort, at most the 142,125-row member).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "not a byte count: ~5,400 donors ESTIMATE, at most 142,125; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Upstream member row count from the brief \u00a73 (142,125 ASEC persons).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> child._reconstruct -> fit_joint_empirical (:667) -> :458; JointEmpiricalFitKernel.run (:457); _prepare via _checked", + "classification": "not-a-transport", + "notes": "Row count; cannot bind because the donor roster is bounded by the ASEC member (142,125 < 1,048,576)." + }, + { + "constant": "joint_empirical.py:385 _json(document) == self.payload", + "value": "consumer-side canonical re-encoding of the model payload in _checked (encoder _json has no cap; :377 MAX_MODEL_BYTES precedes it)", + "enforced_at": [ + "joint_empirical.py:385 (JointEmpiricalModel._checked, \"MODEL_CANONICAL_BYTES\"), run on every to_bytes (:395), sha256 (:400), diagnostics (:404), from_bytes (:391) and draw_joint_empirical (:498) call" + ], + "refusal_codes": [ + "MODEL_CANONICAL_BYTES (ValueError JOINT_EMPIRICAL_MODEL_CANONICAL_BYTES)" + ], + "encodes": "The model document (upstream donor roster).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "~1.25 MB ESTIMATE, at most 32.8 MB; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Same as the MAX_MODEL_BYTES row (231 B/donor).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> child._reconstruct -> model.sha256 (:731, :749, :773) and draw_joint_empirical (:684) -> _checked -> :385; JointEmpiricalDrawKernel.run -> from_bytes (:536)", + "classification": "stays", + "notes": "Each call re-parses (json.loads) and re-encodes the whole model, plus _prepare's per-donor _json of keys and households (:318, :321); a fixed cost at every fraction." + }, + { + "constant": "current_child_property_income_source.py:34 MAX_ROWS", + "value": "600_000", + "enforced_at": [ + "current_child_property_income_source.py:192 (_raw_axis: `0 < len(frame) <= MAX_ROWS`, \"FULL_SOURCE_AXIS\"; the whole 2025 ASEC literal frame, 142,125 rows)", + "current_child_property_income_source.py:209 (_catalogue_members: `len(households) <= MAX_ROWS`, \"CATALOGUE_TYPE\"; 55,762 ASEC households)", + "current_child_property_income_source.py:223 (H_NUMPER numeric range via _integer_literal, \"H_NUMPER_RANGE\")" + ], + "refusal_codes": [ + "FULL_SOURCE_AXIS", + "CATALOGUE_TYPE", + "H_NUMPER_RANGE (all ValueError CHILD_PROPERTY_ + code)" + ], + "encodes": "Row counts of the upstream ASEC member and catalogue, not bytes; at head no longer applied to the stacked roster (the :399 site moved to MAX_RECIPIENT_ROWS).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "not a byte count: 142,125 ASEC persons / 55,762 ASEC households; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Brief \u00a73 ASEC counts.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "600_000, also enforced at base :393 on the stacked person roster (\"RECIPIENT_AXIS\")", + "head_value": "600_000, ASEC-source sites only", + "reachable_from": "45", + "reach_chain": "extend -> ChildPropertyBoundary.__init__ -> child.qualify_child_property_sources (:955) -> project_child_property_donors (:611) -> _raw_axis (:274-275), _catalogue_members (:286)", + "classification": "not-a-transport", + "notes": "Row count on an upstream file; headroom 4.2x/10.8x; must not be lowered below the real member sizes. The row-count lane's binding verdict for this constant referred to the base :393 site, which this branch re-keyed to MAX_RECIPIENT_ROWS." + }, + { + "constant": "current_child_property_income_source.py:39 MAX_RECIPIENT_ROWS", + "value": "15_000_000", + "enforced_at": [ + "current_child_property_income_source.py:399 (project_child_property_recipients: `len(table) <= MAX_RECIPIENT_ROWS`, \"RECIPIENT_AXIS\")" + ], + "refusal_codes": [ + "RECIPIENT_AXIS (ValueError CHILD_PROPERTY_RECIPIENT_AXIS)" + ], + "encodes": "A row count, not bytes: origins[\"persons\"][\"rows\"] rebuilt as a DataFrame (:384), the stacked person roster.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "not a byte count: 356,501 stacked persons", + "bytes_at_full_source": "not a byte count: 3,565,013 stacked persons", + "measurement_basis": "Brief \u00a73 stacked person counts.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "none: the site used MAX_ROWS = 600_000", + "head_value": "15_000_000", + "reachable_from": "45", + "reach_chain": "extend -> ChildPropertyBoundary.__init__ -> qualify_child_property_sources (:955) -> project_child_property_recipients (:619) -> :395-401", + "classification": "not-a-transport", + "notes": "binds_at_full_source true at the base value (3,565,013 > 600,000), false at head. Rule (comment :35-38): four times the measured full-source stacked person roster rounded up to the next whole million. Recorded in the _live contract (:114) so its value is part of the implementation seal." + }, + { + "constant": "current_child_property_income_source.py:40 MAX_PROJECTION_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "current_child_property_income_source.py:55 (_json: `require(len(result) <= MAX_PROJECTION_BYTES, \"PROJECTION_SIZE\")`), called per coordinate tuple in _table_stamp (:556) and on the evidence dict (:579 seal, :656 qualify)" + ], + "refusal_codes": [ + "PROJECTION_SIZE (ValueError CHILD_PROPERTY_PROJECTION_SIZE)" + ], + "encodes": "Fixed-shape documents only: one encoded key coordinate at a time (tens of bytes) and the aggregate evidence dict (protocol strings, digests, three nested source-evidence dicts, counts).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "a few KB (evidence dict); fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Read from :631-654: the evidence carries counts and digests, no rows. Not measured.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> ChildPropertyBoundary.__init__ -> qualify_child_property_sources -> _json (:656) and child_property_sources_seal -> _table_stamp -> _json (:556)", + "classification": "not-a-transport", + "notes": "A memory ceiling with no roster document behind it; cannot bind on the fraction. Note _table_stamp encodes every recipient_key coordinate one at a time (3,565,013 small _json calls at full source) before streaming into physical._table_stamp." + }, + { + "constant": "current_child_property_income_source.py:178 len(value) <= 64", + "value": "64 characters per integer literal", + "enforced_at": [ + "current_child_property_income_source.py:173-179 (_integer_literal, name + \"_LITERAL\"), entered from :202 PH_SEQ, :203 A_LINENO, :223 H_NUMPER, :228 H_SEQ, :232 HSUP_WGT, :239 A_LINENO" + ], + "refusal_codes": [ + "PH_SEQ_LITERAL", + "A_LINENO_LITERAL", + "H_NUMPER_LITERAL", + "H_SEQ_LITERAL", + "HSUP_WGT_LITERAL (ValueError CHILD_PROPERTY_ + name + _LITERAL)" + ], + "encodes": "One ASEC source literal token at a time (printed widths 1-9).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "1-9 characters per token; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Printed ASEC field widths (COORDINATE_WIDTHS PH_SEQ 5, A_LINENO 2; HSUP_WGT up to 9 digits).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> qualify_child_property_sources -> project_child_property_donors -> _raw_axis / _catalogue_members -> _integer_literal", + "classification": "must-not-move", + "notes": "Token width guard; the numeric range caps at the same sites (99999, 16, MAX_ROWS, 999999999) are domain checks, not byte bounds." + }, + { + "constant": "current_child_property_income_source.py:197 0 < len(v) <= 64 (PERIDNUM)", + "value": "64 characters", + "enforced_at": [ + "current_child_property_income_source.py:196-200 (_raw_axis, \"PERIDNUM\")" + ], + "refusal_codes": [ + "PERIDNUM (ValueError CHILD_PROPERTY_PERIDNUM)" + ], + "encodes": "One ASEC PERIDNUM literal per member row (22 digits in the source; current_asec_income_routing_source.py:1084 enforces exactly 22 on the read).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "22 characters; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "preparation.json ASEC raw_native_person_id sample '2156659610400131701102' (22 chars); routing :1084 regex [0-9]{22}.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "extend -> qualify_child_property_sources -> project_child_property_donors -> _raw_axis (:274-275)", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "current_asec_income_routing_source.py:50 TOKEN_MAX_CHARS", + "value": "64", + "enforced_at": [ + "current_asec_income_routing_source.py:476 (literal_code, \"TOKEN_BOUND\"; every _codes_frame call in this module, interest and dividend)", + "current_asec_income_routing_source.py:1106 (_read_capture, \"TOKEN_BOUND:\" + name; every non-coordinate, non-pattern field of every member record; interest :331 and dividend :355 call it with patterns={} so all their amount fields hit this site too)", + "current_asec_income_routing_source.py:1228 (project_income_routing, \"ROUTING_TOKEN_CONTRACT:\" + name)", + "current_asec_retirement_detail_source.py:226 (\"TOKEN_BOUND\"; not on the runners' path)" + ], + "refusal_codes": [ + "TOKEN_BOUND", + "TOKEN_BOUND:", + "ROUTING_TOKEN_CONTRACT: (all ValueError CURRENT_ASEC_INCOME_ROUTING_ + reason)" + ], + "encodes": "One ASEC CSV field literal at a time.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "1-8 characters per token; fraction-independent (the 2025 member is read whole: 142,125 rows x 46 READ_COLUMNS at every fraction)", + "bytes_at_full_source": "same", + "measurement_basis": "Printed widths in RECEIPT_ENTRIES/ACCOUNT_ENTRIES/ALLOCATION_ENTRIES and AMOUNT_FIELDS (max 8).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial -> register_property_kernels -> _Kernel._qualified -> sources.qualify_current_property_income_sources -> routing.qualify_current_asec_income_routing (:339) -> _read_capture (:1321); interest/dividend _capture_member -> routing._read_capture (:331/:355); child.qualify_child_property_sources -> interest/dividend qualifiers (:600, :602)", + "classification": "must-not-move", + "notes": "Also recorded in current_child_property_income_source._live (:138) as part of the implementation seal. The member row cap at :1081/:1110 (`len(records) < rows`, `== rows`) is pinned to coverage._MEMBER_PINS rows (142,125), an upstream row count, not a byte bound." + }, + { + "constant": "current_asec_income_routing_source.py:49 COORDINATE_WIDTHS", + "value": "{PH_SEQ: 5, A_LINENO: 2, A_AGE: 2} characters", + "enforced_at": [ + "current_asec_income_routing_source.py:1085-1092 (_read_capture regex `[0-9]{1,width}`, \"COORDINATE:\" + name) on every member record, entered from :1321 and from interest :331 / dividend :355" + ], + "refusal_codes": [ + "COORDINATE:PH_SEQ", + "COORDINATE:A_LINENO", + "COORDINATE:A_AGE (ValueError CURRENT_ASEC_INCOME_ROUTING_COORDINATE:)" + ], + "encodes": "Fixed-width printed coordinate fields of one ASEC record.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "at most 5/2/2 characters; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Printed dictionary widths.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "same as TOKEN_MAX_CHARS (routing/interest/dividend qualifiers -> _read_capture)", + "classification": "must-not-move", + "notes": "Recorded in child._live (:137)." + }, + { + "constant": "current_asec_income_routing_source.py:1084 [0-9]{22} (PERIDNUM)", + "value": "exactly 22 digits", + "enforced_at": [ + "current_asec_income_routing_source.py:1084 (_read_capture, \"PERSON_KEY\") on every member record" + ], + "refusal_codes": [ + "PERSON_KEY (ValueError CURRENT_ASEC_INCOME_ROUTING_PERSON_KEY)" + ], + "encodes": "The ASEC person key literal.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "22 characters; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Regex width.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "same as TOKEN_MAX_CHARS", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "current_asec_income_routing_source.py:1488-1491 _sha(out.to_json(orient=\"table\").encode()) == evidence[\"projection_sha256\"] and _sha(literals.to_json(orient=\"table\").encode()) == evidence[\"literals_sha256\"]", + "value": "consumer-side re-encoding compared by digest; encoder cap none (pandas to_json, produced at :1472-1473 in the same function)", + "enforced_at": [ + "current_asec_income_routing_source.py:1487-1492 (qualify_current_asec_income_routing, \"FINAL_PROJECTION_CHANGED\")" + ], + "refusal_codes": [ + "FINAL_PROJECTION_CHANGED (ValueError CURRENT_ASEC_INCOME_ROUTING_FINAL_PROJECTION_CHANGED)" + ], + "encodes": "Two roster-shaped tables: `out` = the selected ASEC persons x 158 columns (five family projections + 27 amount status/validity/zero_origin columns + native_person_id), and `literals` = the whole 2025 member x 46 READ_COLUMNS (upstream, 142,125 rows at every fraction).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "out 104.2 MB (14,212 selected ASEC persons x 7,329 B + 12,039); literals 96.4 MB (142,125 x 678 B, fraction-independent)", + "bytes_at_full_source": "out 1.04 GB (142,125 x 7,329 B); literals 96.4 MB", + "measurement_basis": "prop_measure_2.py routing_projection: routing.project_income_routing(raw, ages) on invented arrays plus the 27 status columns and native_person_id added at :1354-1369, then .to_json(orient='table').encode(), 3 vs 503 rows: 7,329 B/row, 158 columns. routing_literals: 46-column string frame, 3 vs 1,003 rows: 678 B/row.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "register_property_kernels -> _Kernel._qualified -> sources.qualify_current_property_income_sources -> routing.qualify_current_asec_income_routing (:339, and again per requalification iteration :370) -> :1472-1473, :1488-1491", + "classification": "stays", + "notes": "No cap exists, so it never binds; each qualify call materialises both strings twice (once for the evidence digest, once for the final check), ~2.3 GB of transient string at full source. The dividend and interest seals (dividend_values_seal :307-341, interest_values_seal :283-317) stream through hashlib via physical._series_digest and carry no bound; their `person`/`asec_literals` DataFrames (142,125 rows each) are retained whole in memory." + }, + { + "constant": "asec_coverage_authentication.py:39 _BODY_MAX (entered from this family as `budget=[coverage._BODY_MAX]`)", + "value": "268_435_456 (256 MiB)", + "enforced_at": [ + "asec_coverage_authentication.py:283-286 (_CsvBounds.feed: `self.budget[0] -= 4 + _NUMBERS.size + 12 + 22 + 32` per record, `-= 1` per PRPERTYP token byte, `_require(self.budget[0] >= 0, \"COVERAGE_BODY_BYTES\")`), inside _capture (:309-345)", + "entry sites in this family: current_asec_income_routing_source.py:1314-1320, current_asec_interest_source.py:324-330, current_asec_dividend_source.py:348-354" + ], + "refusal_codes": [ + "COVERAGE_BODY_BYTES (AsecCoverageAuthenticationError, a ValueError subclass)" + ], + "encodes": "The pre-charge for an encoded coverage body of the pinned 2025 ASEC person member (pppub25.csv, 142,125 records, 277,882,549 bytes): 119 bytes per record (4 + 48 for struct '<6q' + 12 + 22 + 32 + 1 PRPERTYP byte).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "16,912,875 charged (142,125 x 119); fraction-independent", + "bytes_at_full_source": "same 16,912,875", + "measurement_basis": "prop_measure.py asec_body_charge from coverage._NUMBERS.size (48) and the :283/:285 arithmetic; the budget admits 2,255,760 records, headroom 15.87x.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "register_property_kernels -> _Kernel._qualified -> sources.qualify_current_property_income_sources -> interest (:338) / routing (:339) / dividend (:340) qualifiers -> coverage._capture(budget=[coverage._BODY_MAX]); child.qualify_child_property_sources -> interest/dividend (:600/:602)", + "classification": "must-not-move", + "notes": "Defined and owned by the ASEC coverage family; this family only passes it as the encoder limit argument. The same _capture also enforces literal.MAX_MEMBER_BYTES = 1,000,000,000 (asec_person_coverage_source.py:33, at :314-317), _ROW_MAX = 1,048,576, _TOKEN_MAX = 65,536 and _CSV_HEADER_MAX = 65,536 (:272-276), all fraction-independent guards on the same upstream member; they are keyed by that family. Each of the three qualifiers copies the 278 MB member into a private temp dir once per call, and qualify_current_property_income_sources calls each qualifier twice (:351, :370)." + }, + { + "constant": "current_acs_income_anchor_source.py:44 len(value) <= 64 (_literal)", + "value": "64 characters", + "enforced_at": [ + "current_acs_income_anchor_source.py:43-45 (_literal, \"LITERAL_TYPE_OR_BOUND\"), applied to token/age/adjustment/allocation in parse_anchor (:57-59) for every selected ACS person and both anchors (_parsed_table :298-307)" + ], + "refusal_codes": [ + "LITERAL_TYPE_OR_BOUND (ValueError ACS_INCOME_ANCHOR_LITERAL_TYPE_OR_BOUND)" + ], + "encodes": "One ACS PUMS field literal at a time (INTP/RETP up to 7 chars, AGEP 2, ADJINC 7, allocation flag 1).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "1-7 characters per token; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "Printed PUMS widths and the parse_anchor regexes (:60-63, :71-76).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "sources.qualify_current_property_income_sources -> acs.qualify_current_acs_income_anchors (:337) -> _parsed_table (:413) -> parse_anchor -> _literal; child.qualify_child_property_sources -> acs (:604)", + "classification": "must-not-move", + "notes": "Token width guard." + }, + { + "constant": "current_acs_income_anchor_source.py:134 all(len(v) <= 64 for v in row.values())", + "value": "64 characters per CSV cell", + "enforced_at": [ + "current_acs_income_anchor_source.py:134 (_scan, \"SOURCE_TOKEN_BOUND\") on every record of every person member of the captured ACS archive" + ], + "refusal_codes": [ + "SOURCE_TOKEN_BOUND (ValueError ACS_INCOME_ANCHOR_SOURCE_TOKEN_BOUND)" + ], + "encodes": "The eight COLUMNS cells of one ACS person record; the whole archive (3,422,888 records) is scanned at every fraction to pick the wanted keys.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "at most 13 characters per cell (SERIALNO); fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "byte-census-measurements.json acs_household_archive.serialno_widths {\"13\": 1,631,969}; other columns are 1-7 digits.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "acs.qualify_current_acs_income_anchors -> _capture_person (:405) -> _scan (:372) -> :134", + "classification": "must-not-move", + "notes": "The row cap at :132 (`count <= maximum`, \"SOURCE_ROW_SHAPE\") is pinned to the catalogue's people count (3,422,888) via :376 `maximum=total_rows - count`; an upstream row count, not a byte bound. _capture_person copies the 603 MB person archive to a temp dir on every qualify call (twice per qualify_current_property_income_sources, once per qualify_child_property_sources)." + }, + { + "constant": "current_acs_income_anchor_source.py:104/:108 _key regexes 2024(?:HU|GQ)[0-9]{7} and [0-9]{1,2} with 1 <= SPORDER <= 20", + "value": "SERIALNO exactly 13 characters; SPORDER 1-2 digits, at most 20", + "enforced_at": [ + "current_acs_income_anchor_source.py:102-112 (_key, \"SOURCE_HOUSEHOLD_KEY\" / \"SOURCE_PERSON_KEY\"), called at :135 (every scanned archive record), :189 (every ACS origin), :212 (_raw_table re-check)" + ], + "refusal_codes": [ + "SOURCE_HOUSEHOLD_KEY", + "SOURCE_PERSON_KEY (ValueError ACS_INCOME_ANCHOR_ + reason)" + ], + "encodes": "The fixed-width ACS household/person key.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "13 + 1-2 characters; fraction-independent", + "bytes_at_full_source": "same", + "measurement_basis": "byte-census-measurements.json: all 1,631,969 SERIALNOs are 13 wide; 4,139 person rows have a two-digit SPORDER.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "acs.qualify_current_acs_income_anchors -> _origins (:397) -> _key (:189); _capture_person -> _scan -> _key (:135)", + "classification": "must-not-move", + "notes": "The SPORDER <= 20 domain is the same household-size domain current_survey_household_roles.MAX_HOUSEHOLD_MEMBERS = 20 asserts." + }, + { + "constant": "current_acs_income_anchor_source.py:336 qualified.projection == qualified.anchors.reset_index().to_json(orient=\"table\", index=False).encode()", + "value": "consumer-side re-encoding in income_anchor_seal; encoder cap none (pandas to_json; produced at :414)", + "enforced_at": [ + "current_acs_income_anchor_source.py:334-340 (income_anchor_seal, \"PROJECTION_BINDING\"), called at :432 and :440 (qualify), current_property_income_sources.py:70 (_source_seals, inside property_income_sources_seal :91, itself called at :367, :372 x5 loop iterations, :393, graph_current_survey_property.py:185, :928, :931, current_property_completion_routing.py:342, :593), current_property_income_sources.py:344/:353/:370 (qualifier loop), current_child_property_income_source.py:609, :684" + ], + "refusal_codes": [ + "PROJECTION_BINDING (ValueError ACS_INCOME_ANCHOR_PROJECTION_BINDING)" + ], + "encodes": "The ACS anchors projection: one pandas table-orient row per selected ACS person (8 raw COLUMNS + native_person_id, source_year, dollar_year + two anchors x six parsed fields).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "233.8 MB (342,288 selected ACS persons x 683 B + 1,291)", + "bytes_at_full_source": "2.34 GB (3,422,888 x 683 B)", + "measurement_basis": "prop_measure.py acs_projection: acs._parsed_table(raw, origins).reset_index().to_json(orient='table', index=False).encode() on invented rows at full widths, 3 vs 1,003 rows: 683.0 B/row, fixed 1,291 B.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "register_property_kernels -> _Kernel._qualified (every property kernel run) -> sources.qualify_current_property_income_sources -> acs.qualify_current_acs_income_anchors (:337) -> :414 produce, :432/:440 seal; property_income_sources_seal -> _source_seals -> income_anchor_seal; child.qualify_child_property_sources -> :604, :609, :684", + "classification": "stays", + "notes": "No cap, so it never binds, but every seal call re-materialises the whole 2.34 GB string at full source and compares it to the retained 2.34 GB `projection` bytes held on QualifiedAcsIncomeAnchors; qualify_current_property_income_sources alone triggers it at least 2 (qualify) + 1 (:353) + 5x2 (:370-372 loop) + 2 (:367, :393) times, and each property kernel run (six kernel classes across 16 nodes) requalifies. The routing module's evidence digest of the same shape is separate (see the :1488 row)." + }, + { + "constant": "model_input.py:18 MATRIX_HEADER_MAX_BYTES (microcosm.fit; this family produces two more matrices through it)", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "model_input.py:86-87 (encode_recipient_matrix: `len(header) > MATRIX_HEADER_MAX_BYTES` -> ValueError \"Recipient matrix header exceeds its size bound.\"), entered from current_property_income_sources.py:239 (recipient_matrix) and graph_current_survey_property.py:173 (donor_matrix)", + "model_input.py:105-106 (decode_recipient_matrix: `length > MATRIX_HEADER_MAX_BYTES` -> ValueError \"Recipient matrix header is truncated or exceeds its bound.\"), entered from current_property_income_sources.py:213 (the 19-node family's predictor matrix)" + ], + "refusal_codes": [ + "ValueError(\"Recipient matrix header exceeds its size bound.\")", + "ValueError(\"Recipient matrix header is truncated or exceeds its bound.\")" + ], + "encodes": "The JSON header of a recipient matrix (schema_version, entity, rows, columns, index_name, index_identity = qrf._IndexIdentity digest, fixed size per qrf.py:577-600), length-prefixed by a 4-byte big-endian field (:91). The float64 body (8 x (1 + columns) bytes per row) is uncapped.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "header 488 B incl. magic and length at every fraction; body (not bounded): recipient_matrix 15.6 MB (278,964 ACS adults ESTIMATE x 56 B), donor_matrix at most 0.65 MB", + "bytes_at_full_source": "header 488 B; body: recipient_matrix 156.2 MB (2,789,653 ACS adults ESTIMATE x 56 B), donor_matrix at most 6.5 MB", + "measurement_basis": "prop_measure.py matrices: model_input.encode_recipient_matrix with the 45-node pilot's demographic_conditioning=True roster (5 predictors, byte-census-measurements.json predictor_projection.matrix) + property_reported_total = 6 float64 columns, 1 vs 1,001 rows: 56.01 B/row, 488 B fixed. Recipient rows = ACS selected persons aged 15+ with a known INTP (:230-234), ESTIMATED as 81.5% of ACS persons; donor rows = ASEC joint-eligible persons (<= the ASEC selected roster).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "register_property_kernels -> _Kernel._qualified -> sources.qualify_current_property_income_sources -> _compose -> encode_recipient_matrix (:239) and decode_recipient_matrix (:213); graph_current_survey_property._projection_values (:173) from _params (:338), _qualified (:853), _projection_result (:225)", + "classification": "must-not-move", + "notes": "The header is fixed-shape (the index identity is a sha256 of pandas hashes, not the index values), so the 64 KiB cap cannot bind; it protects the length-prefixed header parse. Definition and the predictor matrix belong to the 19-node family's model_input row; recorded here for the two property matrices." + }, + { + "constant": "graph_current_survey_property.py:245 artifacts[PROJECTION_NODE, name] == payload (donor_matrix, recipient_matrix, projection)", + "value": "consumer-side byte equality of re-encoded projection-node artifacts; encoder caps: none on the matrix bodies (MATRIX_HEADER_MAX_BYTES on the header), none on the evidence projection (codec.encode_json)", + "enforced_at": [ + "graph_current_survey_property.py:241-249 (_checked_projection_result, \"PROJECTION_ARTIFACTS\")", + "graph_current_survey_property.py:854-858 (_Kernel._qualified for every non-projection property node: `context.artifacts[edge.name].payload == projected[edge.artifact]`, \"SOURCE_ARTIFACT_BYTES\")" + ], + "refusal_codes": [ + "PROJECTION_ARTIFACTS (ValueError CURRENT_SURVEY_PROPERTY_PROJECTION_ARTIFACTS)", + "SOURCE_ARTIFACT_BYTES (ValueError CURRENT_SURVEY_PROPERTY_SOURCE_ARTIFACT_BYTES)" + ], + "encodes": "donor_matrix (encode_recipient_matrix over ASEC joint-eligible persons, re-encoded at :173 on every _projection_values call), recipient_matrix (over ACS eligible adults, encoded once in sources._compose :239 and retained), projection (fixed-shape evidence dict :180-215).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "recipient_matrix 15.6 MB (ESTIMATE 278,964 ACS adults x 56 B); donor_matrix at most 0.65 MB; projection ~2 KB", + "bytes_at_full_source": "recipient_matrix 156.2 MB (ESTIMATE 2,789,653 x 56 B); donor_matrix at most 6.5 MB", + "measurement_basis": "prop_measure.py matrices (56.01 B/row through model_input.encode_recipient_matrix); adult share ESTIMATED at 81.5% of ACS persons.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial (base call of the child path) -> reconstruct_property_results (:1695) -> _checked_projection_result (:654); verify_materialized_property_income (:1857; _pure_run :881) -> reconstruct_property_results (:1112); run_graph -> CurrentSurveyPropertyAttachKernel.run -> _reconstruct_attachment_results (:675); run_graph -> Filter/Columns/Attach kernels -> _qualified (:852-859)", + "classification": "stays", + "notes": "No cap on the bodies, so it never binds. The donor matrix is re-encoded on every _projection_values call (:173): in _params (once per current_survey_property_nodes call), in _qualified for each of the five non-projection property kernel runs, and in each reconstruct. The retained recipient_matrix bytes (156 MB at full source) live on QualifiedPropertyIncomeSources for the run's lifetime." + }, + { + "constant": "graph_current_survey_property.py:245 artifacts[PROJECTION_NODE, \"completion_routing\"] == routing.payload", + "value": "consumer-side byte equality of the completion routing payload rebuilt by current_property_completion_routing.build_property_completion_routing (:228); encoder cap none (json.dumps at current_property_completion_routing.py:577-591)", + "enforced_at": [ + "graph_current_survey_property.py:241-249 (_checked_projection_result, \"PROJECTION_ARTIFACTS\") when options.completion_routing is true (the 45-node pilot's execution-config.proposed.json sets property_income.completion_routing: true); _reconstruct_attachment_results (:676) passes completion_routing=False so the attach kernel does not rebuild it" + ], + "refusal_codes": [ + "PROJECTION_ARTIFACTS (ValueError CURRENT_SURVEY_PROPERTY_PROJECTION_ARTIFACTS)" + ], + "encodes": "The private completion routing payload: five _table_document tables \u2014 person (one row per stacked person, 12 columns), components (two rows per stacked person, 8 columns), reasons (one row per stacked person, 22 booleans), clones (one row per clone person, 4 columns + index), summary (fixed).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "212.9 MB (356,501 stacked persons x 597.33 B + 2,852)", + "bytes_at_full_source": "2.13 GB (3,565,013 x 597.33 B)", + "measurement_basis": "prop_measure.py completion_routing: tables shaped from build_property_completion_routing :442-576 (ACS-arm rows) encoded with the module's own _table_document and the identical json.dumps call, 2 vs 1,002 stacked persons: 597.33 B per stacked person (person + 2 component rows + reasons row + 2 clone rows), fixed 2,852 B. The 'stage' and flag keys match :578-585.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial (base call with property_income.completion_routing=True) -> run_graph -> CurrentSurveyPropertyProjectionKernel.run -> _projection_result (:914) -> build_property_completion_routing (:228) [producer]; reconstruct_property_results (:1695) and verify_materialized_property_income (:1857, :1112; _pure_run :881) -> _checked_projection_result -> _projection_result -> rebuild -> :245", + "classification": "stays", + "notes": "No cap anywhere: the payload is materialised whole by json.dumps (:577-591) at least three times per cold run (projection kernel, reconstruct at :1695, verify at :1857) and once more on every _pure_run/checked_view, each 2.13 GB at full source, and stored whole as the projection node's completion_routing artifact. The public receipt carries only the allowlisted summary (:230-234, property_completion_public_summary). The row-count lane did not census this document." + }, + { + "constant": "graph_property_tax_leaves.py:542 context.artifacts[\"rebase\"].payload == codec.encode_json(document)", + "value": "consumer-side re-encoding of the leaves kernel's diagnostics by the gate kernel; encoder cap none (codec.encode_json is a bare json.dumps, _graph_legacy_qrf.py:26-29)", + "enforced_at": [ + "graph_property_tax_leaves.py:541-544 (PropertyTaxLeafGateKernel.run, \"DIAGNOSTIC_PAYLOAD\")", + "graph_atomic_survey_financial.py:445-451 (_reconstruct_tax: `loaded[node.id, name] == payload` for the diagnostics and verification artifacts, \"PROPERTY_TAX_ARTIFACT\")" + ], + "refusal_codes": [ + "DIAGNOSTIC_PAYLOAD (ValueError PROPERTY_TAX_LEAVES_DIAGNOSTIC_PAYLOAD)", + "PROPERTY_TAX_ARTIFACT (ValueError from graph_atomic_survey_financial.require)" + ], + "encodes": "The tax-leaf diagnostics document (_diagnostics :338-371): fixed keys plus three roster lists over every row of the receiving population \u2014 person_ids (str per clone person), interest_known and dividend_known (bool per clone person). The population is the property receiving frame (FILTER-all over the completed clone population in the 45-node union, host._tax_nodes at graph_survey_completion_host.py:754), i.e. clone persons.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "14.3 MB (713,002 clone persons x 20 B + 2,724)", + "bytes_at_full_source": "142.6 MB (7,130,026 x 20 B + 2,724)", + "measurement_basis": "prop_measure.py tax_diagnostics: codec.encode_json(tax._diagnostics(person, split, params, bindings)) on an invented person frame at 7-digit clone ids with known interest and dividends, 2 vs 2,002 rows: 20.0 B/clone person, 2,724 B fixed (three binding entries).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial(child_property=...) -> extend -> host._tax_nodes (:754) -> property_tax_leaf_nodes; host.run_graph -> PropertyTaxLeavesKernel.run (:486-489 produce) and PropertyTaxLeafGateKernel.run (:540-547) -> :542; host._reconstruct_tax (:805 in extend, :834 in _pure_run) -> both kernels again -> :447", + "classification": "stays", + "notes": "No cap in the runtime, so it never binds, but the document is materialised whole at least four times per tax pass (:489 artifact, :542 re-encoding, :547 verification artifact, and again in _reconstruct_tax) and \u2014 because :491 `receipt=document` and :551 `\"evidence\": document` place the roster in both node receipts \u2014 it is frozen into the manifest (graph_atomic_survey_population._states :107 `_freeze_json(receipt)`), compared by _check_node_states (graph_survey_population.py:921 `node.receipt == state[\"receipt\"]`) and canonical_json'd whole by manifest.to_json_bytes (graph_survey_completion_host.py:851, :867): about 285 MB of manifest JSON at full source for the two copies. The 45-node pilot harness's own export limit for manifest.json is 16 MiB (execution-config.proposed.json external_export_limits_bytes), which two copies at 20 B per clone person exceed above about 420,000 clone persons, roughly 1/17 of source \u2014 a harness limit, not a runtime bound. The signed-reconciliation summary compared at :741-745 and the draw summary at :506 are fixed-shape aggregates (graph_signed_reconciliation.py:220-228, graph_property_income.py:327-336)." + }, + { + "constant": "graph_property_income_receipts.py:224 codec.encode_raw_target(result.raw_draw, target=target, index=recipient.index) == raw_payload", + "value": "consumer-side re-encoding of each of the four raw property draws (LegacyQRFApplyKernel outputs); encoder cap none (_graph_legacy_qrf.encode_raw_target :170-175: magic + fixed metadata + float64 body)", + "enforced_at": [ + "graph_property_income_receipts.py:222-229 (verify_property_model_receipts, \"APPLICATION_VALUES\"), once per PROPERTY_COMPONENTS target" + ], + "refusal_codes": [ + "APPLICATION_VALUES (ValueError PROPERTY_MODEL_RECEIPTS_APPLICATION_VALUES)" + ], + "encodes": "A raw draw: 8 bytes per recipient row (ACS eligible adults) plus a fixed metadata line whose index identity is a digest (qrf._IndexIdentity), for each of the four components.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "2.23 MB per target (ESTIMATE 278,964 ACS adults x 8.003 B + 242), four targets", + "bytes_at_full_source": "22.3 MB per target (ESTIMATE 2,789,653 x 8 B), four targets", + "measurement_basis": "prop_measure.py raw_target: codec.encode_raw_target on invented rows at full id widths, 1 vs 1,001 rows: 8.003 B/row, 242 B fixed. Recipient count ESTIMATED as 81.5% of ACS persons.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "45", + "reach_chain": "run_atomic_survey_financial (base call of the child path, property_graph is not None) -> graph_atomic_survey_financial.py:1798-1808 verify_property_model_receipts(property_nodes, expected[DONOR_COLUMNS_NODE], expected[RECIPIENT_COLUMNS_NODE], loaded) -> :219-229", + "classification": "stays", + "notes": "A numpy body compared whole against the stored artifact; no bound to meet. The same raw draws are decoded by codec.read_raw_target in graph_current_survey_property._read_draws (:493) and graph_property_income.PropertyIncomeDrawColumnsKernel.run (:320), whose `len(body) != len(index) * 8` check (_graph_legacy_qrf.py:190-193) is an equality with the actual row count, not a ceiling. The declaration check at :90-98 (canonical_json of normative nodes) is fixed-shape." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prop/prop_measure.py", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prop/prop-measurements.json", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prop/prop_measure_2.py", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prop/prop-measurements-2.json", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/prop/prop-derived-totals.json" + ], + "open_questions": [ + "Eligible-child count is not established by any recovered artifact: the 45-node pilot (native45-v5) exported no manifest (cold.log is empty, no manifest.json/graph.json outside the graph store) and the 1/1000 preparation carries no age column. Every draw-document byte figure uses the row-count lane's 18.5% under-15 share (ESTIMATE); the base-64 MiB binding verdict at full source is robust for any share above 10.4% of stacked persons, and the pattern mix (180.5 B zero rows vs 288.5 B positive rows) only widens the range. The recipient-matrix and raw-draw figures likewise use an ESTIMATED 81.5% ACS adult share.", + "graph_joint_empirical.MAX_RECIPIENTS = 1,048,576 (:49, enforced :530 before the eligibility filter on the whole RECIPIENT support table, which graph_child_property_income._projections :378 builds from every stacked person) binds at full source (3,565,013 rows; first at about fraction 0.294) and was not lifted on this branch, while the two upstream ceilings on the same roster (MAX_ORIGINALS, MAX_RECIPIENT_ROWS) were raised to 15,000,000. It is now the first refusal on the child lane above roughly 1/3 of source. Its sibling MAX_DONORS cannot bind.", + "The completion routing payload (current_property_completion_routing.py:577-591, produced on the 45 path because the pilot config sets completion_routing=true) has no cap at all and measures 597 B per stacked person: 2.13 GB at full source, materialised whole by json.dumps at least three times per cold run and once per _pure_run, and compared byte-for-byte at graph_current_survey_property.py:245. It is not in the row-count census. Whether it should take the segmented shape, a cap, or be dropped from the stored artifact roster is a design question for the lane.", + "The tax-leaf diagnostics roster (20 B per clone person, 142.6 MB at full source) is carried inside two node receipts (graph_property_tax_leaves.py:491 and :551) and therefore inside the manifest JSON (about 285 MB at full source) with no runtime cap; the pilot harness's 16 MiB manifest export limit would refuse it above about 1/17 of source. Whether receipts should carry per-row lists at all is outside this census's remit but affects the transport lane.", + "current_acs_income_anchor_source retains a 683 B-per-ACS-person to_json projection (2.34 GB at full source) as bytes on QualifiedAcsIncomeAnchors and re-encodes it on every income_anchor_seal call (dozens of times per property kernel run through property_income_sources_seal); current_asec_income_routing_source materialises a 7,329 B-per-selected-ASEC-person to_json string (1.04 GB at full source) twice per qualify. Neither has a cap, so neither binds, but both are whole-copy materialisations the transport lane may want to convert to streamed digests like the interest/dividend seals already are.", + "current_property_income_sources.py:320 re-encodes the whole origins block of the preparation (codec.encode_json(json.loads(entry[1])[\"origins\"])) with no cap: 718,800 bytes at 1/1000, SCALED to about 720 MB at full source and 72 MB at 1/10 (prop-measurements-2.json origins_document); child (:259, :618), acs (:396) and completion routing (:130) each json.loads the same origins/preparation payload again. These are consumer-side parses of the 19-node family's segmented preparation document, not bounds, and are recorded here only as sizes.", + "graph_child_property_income.py:85 cites docs/us-native-byte-transports.md, which does not exist in this tree; the neighbouring docs are us-native-scale-transport.md and us-native-row-ceilings.md.", + "No byte bounds of their own were found in graph_survey_completion.py, graph_survey_completion_host.py, current_asec_property_basis.py, property_income_constants.py, graph_property_income.py, current_property_income_sources.py, current_property_completion_routing.py, current_asec_dividend_source.py or current_asec_interest_source.py; the last two enter coverage._BODY_MAX and routing's _read_capture widths (recorded), copy the 278 MB member per call, and seal through streaming digests. asec_coverage_authentication's sibling limits (_ROW_MAX, _TOKEN_MAX, _CSV_HEADER_MAX, literal.MAX_MEMBER_BYTES) are enforced in the same _capture this family enters but are keyed by the ASEC family.", + "The 45-node graph's household-roles and person-status modules were not in this family's module list and were not read; per the brief they are reachable only from the 49/51-node variants." + ] + }, + { + "family": "Roles, person status and the PUF modules the runners import", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_household_roles.py (1-1506, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_household_roles.py (1-566, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_person_status.py (1-822, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_person_status.py (1-365, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_person_status_source.py (1-541, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py (1-1714, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py (1-2037, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_raw_source.py (1-1727, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py (1-1215, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_qrf_chain.py (1-1112, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py (1-4127, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source.py (1-520, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_aggregate_records.py (1-1541, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py (1-690, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py (1-801, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py (1-868, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_puf_transfer.py (1-408, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py (1-460, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py (1-4684, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/post_transfer_calibration.py (1-1856, full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_raw_source_definition.json (single line; csv_profile, fields, sources)", + "packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py:64 (PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID definition)", + "packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py:22,630-638,763-766 (grep only: enforcement of the clone-safe id bound)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py:45-60,76-82,116-130 (_encode, MAX_PAYLOAD_BYTES, error class)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py:265-300 (_bounded_json), :790-812 and :1035-1050 (lazy puf_support imports)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py:120-160 (_person_status_module lazy import)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_completion_host.py:25-70 (_roles lazy import, household_roles flag)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py:88-92,149-150,308-322,520-545 (puf_support call sites, COMBINED_CLONE_ATTACHMENT_FRACTION = 1.0)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py:1055-1090 (acs_household_reference_states diagnostics shape)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py:22-27,41-60,108-152 (_origins columns, _key widths)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py:36-50,315,378,389 (constants and helper names used by the roles/status scans)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py:42-43,131-140,160 (_copy cap/exact_size)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py:39,206,244-286,309-324,414,424,519,632-644 (grep: _BODY_MAX budget)", + "packages/microcosm-fit/src/microcosm/fit/_graph_legacy_qrf.py:20-40 (encode_json: no cap)", + "packages/microcosm-graph/src/microcosm/graph/canonical.py:43-58 (canonical_json: no cap)", + "experiments/native-byte-transports/census-brief.md (full)", + "experiments/native-byte-transports/byte_census_measure.py (full)", + "experiments/native-byte-transports/byte-census-measurements.json (full)", + "experiments/native-row-ceilings/census.json (the 21 bounds and 6 verdicts in this family)", + "_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json (origins.persons.columns and one row per arm, for row shapes)" + ], + "rows": [ + { + "constant": "current_survey_household_roles.py:86 MAX_PERSONS", + "value": "64 * 1024**2 // 32 = 2,097,152", + "base_value": "2,097,152", + "head_value": "2,097,152", + "enforced_at": [ + "current_survey_household_roles.py:663-669 (_origins: `and 0 < len(original) <= MAX_PERSONS,` -> \"ORIGIN_ROSTER\"); ValueError(\"CURRENT_SURVEY_HOUSEHOLD_ROLES_ORIGIN_ROSTER\") via require at :239-241" + ], + "refusal_codes": [ + "ORIGIN_ROSTER" + ], + "encodes": "Row count of the origins roster rebuilt from document[\"origins\"][\"persons\"][\"rows\"] (:651-652): one row per stacked original person (ACS + ASEC channels, pre-clone), compared row-for-row to preparation_frame.person (:672-683).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "356,501 rows (a row count, not bytes)", + "bytes_at_full_source": "3,565,013 rows (a row count, not bytes)", + "measurement_basis": "Counts from census-brief \u00a73 (stacked persons); no encoding is involved at this site.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "reachable_from": "none", + "reach_chain": "graph_survey_completion_host(household_roles=True) -> _roles() (graph_survey_completion_host.py:37-40) -> graph_current_survey_household_roles.register_household_role_kernels / qualify_current_survey_household_roles (:1104) -> _origins (:1127). The 45-node pilot passed household_roles=False; only the 49/51-node variants reach it.", + "classification": "not-a-transport", + "notes": "A row cap wearing a byte-derived expression (64 MiB / 32 B per row); nothing serialises it. It is the first refusal in qualify: `origins = _origins(state.frame, document)` at :1127 runs before `_acs_roster` (:1136) and before `projection = _projection_bytes(table)` (:1151), so at full source it pre-empts every byte bound in this family. First binds at fraction 2,097,152 / 3,565,013 = 0.588. Unchanged on this branch (git diff origin/native-row-ceilings is empty for the module)." + }, + { + "constant": "current_survey_household_roles.py:88 MAX_HOUSEHOLD_MEMBERS", + "value": "20 (literal)", + "base_value": "20", + "head_value": "20", + "enforced_at": [ + "current_survey_household_roles.py:715-720 (`and 1 <= int(line) <= MAX_HOUSEHOLD_MEMBERS` -> \"ACS_LINE_KEY\")", + "current_survey_household_roles.py:749-753 (\"ASEC_LINE_KEY\")", + "current_survey_household_roles.py:792-796 (`and 1 <= int(row[\"SPORDER\"]) <= MAX_HOUSEHOLD_MEMBERS` -> \"ACS_SOURCE_LINE\")" + ], + "refusal_codes": [ + "ACS_LINE_KEY", + "ASEC_LINE_KEY", + "ACS_SOURCE_LINE" + ], + "encodes": "A single printed within-household line number (ACS SPORDER, ASEC A_LINENO), one scalar per person; not a byte count.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "20 (scalar domain; independent of the fraction)", + "bytes_at_full_source": "20 (same)", + "measurement_basis": "Read from the code; each site pairs it with a `[0-9]{1,2}` regex on one token.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Same chain as MAX_PERSONS (49/51-node variants only): qualify -> _acs_keys (:698) / _asec_keys (:728) / _acs_roster -> _scan_acs_relationships (:760).", + "classification": "must-not-move", + "notes": "Protects a token/scalar domain (the published line-number range); current_survey_health_source._key hardcodes the same 20 at :53. Not a transport of any kind." + }, + { + "constant": "current_survey_household_roles.py:89 MAX_ROSTER_ROWS", + "value": "MAX_ROSTER_ROWS = MAX_PERSONS = 2,097,152", + "base_value": "2,097,152", + "head_value": "2,097,152", + "enforced_at": [ + "current_survey_household_roles.py:799 (_scan_acs_relationships: `require(len(selected) < MAX_ROSTER_ROWS, \"ACS_ROSTER_BOUND\")`, checked before each insertion into the `roster` dict)" + ], + "refusal_codes": [ + "ACS_ROSTER_BOUND" + ], + "encodes": "Row count of the retained ACS person rows: every member of every selected ACS household read from the pinned person archive (selection is by household, :760-766). At full source every ACS household is selected, so this is the ACS person roster.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "about 342,288 rows (ACS persons at 1/10; a row count)", + "bytes_at_full_source": "3,422,888 rows (all ACS persons; a row count)", + "measurement_basis": "Counts from census-brief \u00a73 (ACS persons); the dict holds one (SERIALNO, SPORDER) -> RELSHIPP entry per retained row.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: qualify (:1104) -> _acs_roster (:1136, :1054) -> _scan_acs_relationships (:1084).", + "classification": "not-a-transport", + "notes": "A row cap aliasing MAX_PERSONS. Exceeded at full source (3,422,888 > 2,097,152) but never reached there, because ORIGIN_ROSTER (:667) refuses first at :1127. The scan also carries a per-file row cap from the catalogue, `maximum=document[\"counts\"][\"people\"] - count` (:1088) -> \"ACS_SOURCE_ROW_SHAPE\" at :783, which is an assertion about the real PUMS file's row count." + }, + { + "constant": "current_survey_household_roles.py:90 MAX_RECEIPT_BYTES", + "value": "64 * 1024 = 65,536", + "base_value": "65,536", + "head_value": "65,536", + "enforced_at": [ + "current_survey_household_roles.py:1152-1179 (`receipt = source._encode({...}, maximum=MAX_RECEIPT_BYTES)`), which raises inside survey_population_preparation._encode at survey_population_preparation.py:116-121 (`_require(len(result) + len(encoded) <= maximum, \"PAYLOAD_LIMIT\")`, SurveyPopulationPreparationError, a ValueError subclass at :78)" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT" + ], + "encodes": "The household-roles receipt: a fixed-shape JSON object of aggregates, digests, the canonical-requirement block and `acs_relationship_diagnostics` (14 scalar counters plus a 4-entry state_counts map from asec_demographic_source.py:1064-1084). No per-row data.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "3,809 (measured; the shape does not move with the fraction, only integer widths)", + "bytes_at_full_source": "3,809 (measured with every counter set to full-source magnitudes)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> roles_receipt: the document built with the code's keys at :1153-1177 and encoded through survey_population_preparation._encode(maximum=MAX_RECEIPT_BYTES); headroom 17.2x.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: qualify_current_survey_household_roles (:1104) -> :1152.", + "classification": "not-a-transport", + "notes": "_encode appends chunk by chunk with the bound checked before each append (streams), but the document is fixed-shape by contract (PUBLIC_RECEIPT_KEYS at :205-236 admits only counts, digests and flags), so no fraction can move it." + }, + { + "constant": "current_survey_household_roles.py:609 ", + "value": "len(token) <= 64 -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "current_survey_household_roles.py:609 (acs_relationship_token: `require(type(token) is str and len(token) <= 64, \"ACS_TOKEN_BOUND\")`)" + ], + "refusal_codes": [ + "ACS_TOKEN_BOUND" + ], + "encodes": "One RELSHIPP token from the roster dict, checked per person before the `[0-9]{1,2}` parse.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "2 characters per real token (RELSHIPP is a two-digit code); cap 64", + "bytes_at_full_source": "2 characters per token; cap 64", + "measurement_basis": "Read from the code (:610 `re.fullmatch(r\"[0-9]{1,2}\", token, re.ASCII)`).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: qualify -> _acs_states (:1140, :805) -> acs_relationship_token (:826).", + "classification": "must-not-move", + "notes": "A token width; the same 64 is the coverage family's MAX_TOKEN_BYTES scale for ACS cells." + }, + { + "constant": "current_survey_household_roles.py:786 ", + "value": "len(row[column]) <= 64 for column in ACS_ROSTER_COLUMNS -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "current_survey_household_roles.py:785-788 (_scan_acs_relationships: `require(all(len(row[column]) <= 64 for column in ACS_ROSTER_COLUMNS), \"ACS_SOURCE_TOKEN_BOUND\")`)" + ], + "refusal_codes": [ + "ACS_SOURCE_TOKEN_BOUND" + ], + "encodes": "The three cells (SERIALNO, SPORDER, RELSHIPP) of one ACS person record as the pinned member is streamed.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13 characters (SERIALNO `2024HU0001234`) is the widest real cell; cap 64", + "bytes_at_full_source": "13 characters; cap 64", + "measurement_basis": "Read from the code (:712 `2024(?:HU|GQ)[0-9]{7}`).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: qualify -> _acs_roster (:1136) -> _scan_acs_relationships (:1084).", + "classification": "must-not-move", + "notes": "The record itself is produced by acs_person_coverage_authentication._records / _decode_record (:769-770), whose own line/token caps (that module's MAX_RECORD_BYTES 400,000, MAX_TOKEN_BYTES 65,536, MAX_CSV_HEADER_BYTES 65,536 at :48-50) apply first; those are the coverage family's rows." + }, + { + "constant": "current_survey_household_roles.py:937 ", + "value": "none (consumer-side re-encoding: `value.projection == _projection_bytes(value.rows)`; the encoder `_projection_bytes` at :634-635 has no cap)", + "base_value": "none", + "head_value": "none", + "enforced_at": [ + "current_survey_household_roles.py:933-940 (household_role_projection_seal: `value.projection == _projection_bytes(value.rows) and json.loads(value.receipt)[\"projection_sha256\"] == _sha(value.projection)` -> \"PROJECTION_BINDING\"); called at :1183, :1220 (FINAL_PROJECTION) and from graph_current_survey_household_roles.py:202, :418, :478, :497", + "graph_current_survey_household_roles.py:266-281 (_check_artifacts: `value.payload == expected[edge.producer, edge.artifact]` -> \"ARTIFACT_PAYLOAD\", against the retained qualified.projection bytes from _expected_artifacts at :261-262; not a re-encoding but a whole-bytes equality)" + ], + "refusal_codes": [ + "PROJECTION_BINDING", + "ARTIFACT_PAYLOAD" + ], + "encodes": "The roles projection: `table.reset_index().to_json(orient=\"table\", index=False).encode()` (:635) over the 13 declared COLUMN_TOKENS plus person_id, one row per stacked original person.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "231,725,650 (650.0 B/person x 356,501)", + "bytes_at_full_source": "2,317,258,450 (650.0 B/person x 3,565,013)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> roles_projection: rows built through `_observation_table` with the real ORIGIN/UNIVERSE literals and person_id/native ids at full-source widths, encoded by `_projection_bytes`, 3 vs 1,203 rows (2,969 vs 782,969 bytes). Not the lane JSON's 324 B/person: that table carried only 7 of the 13 columns and placeholder strings (see open_questions).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: qualify (:1151, :1183, :1220) and the roles graph kernel (_HouseholdRoleKernel.__init__ :418, run :478/:497).", + "classification": "segment", + "notes": "The producer has no cap, so binds is false by definition; the whole 2.3 GB string is re-encoded on every seal call (at least five times per run at full source). The bytes meet the consumer cap MAX_ARTIFACT_BYTES (next row) from 103,244 persons upward. `_table_stamp` at :638-642 also materialises `table[name].to_json()` per column for a digest with no cap." + }, + { + "constant": "current_survey_household_roles.py:1384 ", + "value": "0 < len(receiving_version) <= 256 -> 256", + "base_value": "256", + "head_value": "256", + "enforced_at": [ + "current_survey_household_roles.py:1382-1387 (household_role_binding_receipt: `and 0 < len(receiving_version) <= 256` -> \"RECEIPT_INPUT\")" + ], + "refusal_codes": [ + "RECEIPT_INPUT" + ], + "encodes": "The receiving population version string (a node id), one scalar.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "a node id of a few dozen characters; cap 256", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: graph _result (graph_current_survey_household_roles.py:343-348) -> household_role_binding_receipt.", + "classification": "must-not-move", + "notes": "A string-length width on an identifier." + }, + { + "constant": "graph_current_survey_household_roles.py:70 MAX_ARTIFACT_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "base_value": "67,108,864", + "head_value": "67,108,864", + "enforced_at": [ + "graph_current_survey_household_roles.py:266-276 (_check_artifacts: `and len(value.payload) <= MAX_ARTIFACT_BYTES,` -> \"ARTIFACT_TYPE\", over every artifact input of the node: the ordering edge and host edges of the source node (:228) and the ordering edge plus the projection edge of the bind node (:250)); require is roles.require (:57), ValueError(\"CURRENT_SURVEY_HOUSEHOLD_ROLES_ARTIFACT_TYPE\")" + ], + "refusal_codes": [ + "ARTIFACT_TYPE" + ], + "encodes": "Whole artifact payloads as bytes objects; on the bind node this is the roles projection (previous row: roster of stacked persons, 13 columns + person_id, pandas orient=\"table\" JSON), plus the host-edge payloads.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "231,725,650 (3.45x the cap)", + "bytes_at_full_source": "2,317,258,450 (34.5x the cap)", + "measurement_basis": "Same measurement as the projection row (650.0 B/person through roles._projection_bytes); the cap admits 103,244 persons (67,108,864 // 650).", + "binds_at_tenth": true, + "binds_at_full_source": true, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: graph_survey_completion_host(household_roles=True) -> _roles() -> register_household_role_kernels (:538) -> _HouseholdRoleKernel.run (:471) -> _check_artifacts (:485); also expected_survey_household_roles_population (:361).", + "classification": "consumer", + "notes": "A consumer-side cap on bytes the producer issues whole; the producer `_projection_bytes` carries no cap, so this row takes the producer's total. At full source the row cap MAX_PERSONS (:667) refuses before any projection exists, so this is the binding refusal only between 103,244 persons and 2,097,152 persons (fractions 0.029 to 0.588). The bind node's own output `canonical_json(receipt)` at :351 is fixed-shape. `_seal_result` at :452-469 materialises `value.to_json()` of the bound is_household_head column (clone persons, 7,130,026 at full source) for a digest with no cap." + }, + { + "constant": "graph_current_survey_household_roles.py:131 ", + "value": "len(value) == 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "graph_current_survey_household_roles.py:130-131 (_digest) used at :146-152 (_check_host_binding: `all(_digest(value) for value in pin.values())` -> \"HOST_PIN_DIGESTS\")" + ], + "refusal_codes": [ + "HOST_PIN_DIGESTS" + ], + "encodes": "Three sha256 hex strings per host pin (producer_key, artifact_key, payload_sha256).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: current_survey_household_roles_nodes (:192) -> _check_host_binding (:210).", + "classification": "must-not-move", + "notes": "A fixed-width field (sha256 hex)." + }, + { + "constant": "graph_current_survey_household_roles.py:205 ", + "value": "0 < len(receiving_version) <= 256 -> 256", + "base_value": "256", + "head_value": "256", + "enforced_at": [ + "graph_current_survey_household_roles.py:203-209 (current_survey_household_roles_nodes: `and 0 < len(receiving_version) <= 256` -> \"FRAGMENT_INPUT\")" + ], + "refusal_codes": [ + "FRAGMENT_INPUT" + ], + "encodes": "The receiving population version string, one scalar.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "a node id; cap 256", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "49/51-node variants only: register_household_role_kernels -> current_survey_household_roles_kernels -> current_survey_household_roles_nodes.", + "classification": "must-not-move", + "notes": "Identifier width; independent literal from current_survey_household_roles.py:1384." + }, + { + "constant": "graph_current_survey_person_status.py:59 MAX_ARTIFACT_BYTES", + "value": "2**31 = 2,147,483,648", + "base_value": "2,147,483,648", + "head_value": "2,147,483,648", + "enforced_at": [ + "graph_current_survey_person_status.py:387 (projection_payload: `require(len(payload) <= MAX_ARTIFACT_BYTES, \"PROJECTION_BYTES\")`, producer side)", + "graph_current_survey_person_status.py:562-568 (_check_artifacts: `and len(value.payload) <= MAX_ARTIFACT_BYTES,` -> \"ARTIFACT_TYPE\", consumer side, every artifact input of all four nodes)", + "graph_current_survey_person_status.py:569-573 (consumer-side re-encoding: `value.payload == expected[edge.producer, edge.artifact]` -> \"ARTIFACT_PAYLOAD\", where expected = projection_payload(qualified) at :548, re-run per kernel run and verify)" + ], + "refusal_codes": [ + "PROJECTION_BYTES", + "ARTIFACT_TYPE", + "ARTIFACT_PAYLOAD" + ], + "encodes": "The person-status source projection: canonical_json of {protocol, source_receipt_sha256, origins: _table_document(origins), raw: _table_document(raw)} (:377-388), where origins carries the 10 preparation columns (person_id, source, source_year, survey_year, raw_native_household_id, raw_native_person_id, native_line_numeric_original, selected_receiving_person_id, selected_receiving_household_id, household_id) and raw the 45 RAW_COLUMNS (24 ASEC + 21 ACS literals, the opposite arm null), integers as {\"integer_literal\": \"...\"} (:355-363); one row per stacked original person.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "155,036,654 (433.65 B per ACS person x 342,288 + 464.64 B per ASEC person x 14,212)", + "bytes_at_full_source": "1,550,372,341 (433.65 x 3,422,888 + 464.64 x 142,125); 0.722 of the cap, headroom 1.39x; the cap admits 4,938,044 persons at the weighted 434.89 B/person", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> person_status_projection: origins rows copied from the recovered preparation.json's real ACS and ASEC rows with ids reset to full-source widths (7-digit person ids, 13-char SERIALNO, 22-digit PERIDNUM), raw rows at real literal widths, typed through current_survey_person_status_source._table and encoded through status_graph._table_document + canonical_json, 3 vs 1,203 rows per arm (ACS 2,750 vs 523,130 bytes; ASEC 2,849 vs 560,423).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "graph_atomic_survey_financial._person_status_module (:132-136) is imported only inside _PersonStatusBoundary (:139-160), constructed only for the explicitly enabled person-status graph (person_status=True; the 19-node run and the 45-node pilot pass person_status=False). Would reach: _PersonStatusBoundary -> graph._result (:613) -> status_tables/projection_payload (:387) and _StatusKernel.run (:769) -> _check_artifacts (:782).", + "classification": "stays", + "notes": "Headroom 1.39x at full source, so it does not meet its bound; but the 1.55 GB payload is materialised whole and re-encoded by `projection_payload(qualified)` on every _check_artifacts (:556-558) and every _result (:614). Two larger unbounded materialisations sit beside it: `_columns_payload` (:391-401) digests `canonical_json(_table_document(table))` for the literal table (816,423,843 B at full source) and for the 121-column recoded table (4,853,789,741 B at full source; 485,377,090 at 1/10) with no cap at all (see open_questions). The row-ceilings lane's 447 B/person estimate is replaced by the measured 434.89 weighted." + }, + { + "constant": "graph_current_survey_person_status.py:432 ", + "value": "len(v) == 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "graph_current_survey_person_status.py:427-436 (_check_pins: `type(v) is str and len(v) == 64 and not set(v) - set(\"0123456789abcdef\")` -> \"HOST_PIN_DIGESTS\")" + ], + "refusal_codes": [ + "HOST_PIN_DIGESTS" + ], + "encodes": "Three sha256 hex strings per host pin.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "person_status=True graphs only: current_survey_person_status_nodes (:451) -> _check_pins (:462).", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "current_survey_person_status.py:137 ", + "value": "len(token) <= 64 -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "current_survey_person_status.py:137 (literal_code: `require(type(token) is str and len(token) <= 64, \"LITERAL_TYPE_OR_BOUND\")`)" + ], + "refusal_codes": [ + "LITERAL_TYPE_OR_BOUND" + ], + "encodes": "One published status/flag/age literal, checked before the width-1/2 numeric regex (:141).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "1-2 characters per real token; cap 64", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code; `width=` defaults 1 or 2 at :131, :152, :175-177, :206-207, :240, :243, :290, :295.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "person_status=True graphs only: current_survey_person_status_source._combine (:167, :177) and status_graph.status_tables (:327) -> recode_person_status -> literal_code.", + "classification": "must-not-move", + "notes": "Token width on a code literal; the 1/2-digit `width` arguments are the code grammar, not byte bounds." + }, + { + "constant": "current_survey_person_status.py:236 ", + "value": "len(row[c]) <= 64 for c in columns -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "current_survey_person_status.py:235-238 (recode_person_status: `require(all(type(row[c]) is str and len(row[c]) <= 64 for c in columns), \"LITERAL_TYPE_OR_BOUND\")`)" + ], + "refusal_codes": [ + "LITERAL_TYPE_OR_BOUND" + ], + "encodes": "Every cell of one person's arm columns (24 ASEC or 21 ACS literals) before recoding.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "22 characters (PERIDNUM) is the widest real cell; cap 64", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code and current_survey_health_source._key (:44 `[0-9]{22}`).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "person_status=True graphs only: _combine (:177) / status_tables (:327) -> recode_person_status.", + "classification": "must-not-move", + "notes": "Token width." + }, + { + "constant": "current_survey_person_status_source.py:96 ", + "value": "len(row[c]) <= 64 for c in columns -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "current_survey_person_status_source.py:96 (_scan: `require(all(len(row[c]) <= 64 for c in columns), \"SOURCE_TOKEN_BOUND\")`)" + ], + "refusal_codes": [ + "SOURCE_TOKEN_BOUND" + ], + "encodes": "The arm's cells of one CSV record while the pinned ASEC member (:389-396) or the ACS person archive members (:413-427) are streamed.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "22 characters (PERIDNUM) is the widest real cell; cap 64", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "person_status=True graphs only: qualify_current_survey_person_status (:302) -> _scan (:390, :418).", + "classification": "must-not-move", + "notes": "The same helper carries the per-file row cap `count <= maximum` -> \"SOURCE_ROW_SHAPE\" at :94, where maximum is the pinned member's row count (:395) or the ACS catalogue's people count (:423): assertions about the real files. Records come from acs_person_coverage_authentication._records/_decode_record (:80-81), whose caps are the coverage family's rows." + }, + { + "constant": "survey_population_preparation.py:50 MAX_PAYLOAD_BYTES (definition owned by the survey-population family; this row records the enforcement sites inside this family)", + "value": "64 * 1024**2 = 67,108,864", + "base_value": "67,108,864", + "head_value": "67,108,864", + "enforced_at": [ + "current_survey_person_status_source.py:436-482 (`receipt = source._encode({...})` with the default maximum=MAX_PAYLOAD_BYTES -> \"PAYLOAD_LIMIT\", SurveyPopulationPreparationError at survey_population_preparation.py:116-121)", + "current_survey_amounts.py:362-370 (`projection = predictors.host.survey_graph._bounded_json({...}, predictors.source.MAX_PAYLOAD_BYTES)` -> \"TRANSPORT_LIMIT\", SurveyPopulationGraphError, graph_survey_population.py:270-290)" + ], + "refusal_codes": [ + "PAYLOAD_LIMIT", + "TRANSPORT_LIMIT" + ], + "encodes": "Two fixed-shape documents: the person-status receipt (digests, periods, column rosters, 187 observation column names; :437-481) and the amounts projection (evidence dict plus three digests; :326-370). Neither carries per-row data.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "person-status receipt 12,379 B (measured); amounts projection not measured (fixed shape: evidence keys plus sha256 strings)", + "bytes_at_full_source": "12,379 B (same document; counters at full-source magnitudes)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> person_status_receipt (document built with the keys at :437-481 and encoded through survey_population_preparation._encode).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Neither site is on a runner path: qualify_current_survey_person_status runs only for person_status=True graphs; current_survey_amounts is listed off every path in census-brief \u00a72.", + "classification": "not-a-transport", + "notes": "Both documents are fixed-shape at these sites (5,400x headroom for the receipt). The definition row belongs to the survey-population family; merge these two sites into it. The amounts projection's inputs (origins, native, reports, donor_columns via to_json(orient=\"table\") at :139, :345, :366-367) are roster-sized digest inputs with no cap (open_questions)." + }, + { + "constant": "puf_growth.py:193 MAX_EXACT_AMOUNT", + "value": "2**53 = 9,007,199,254,740,992", + "base_value": "2**53", + "head_value": "2**53", + "enforced_at": [ + "puf_growth.py:1243-1250 (_money_values: `bool((np.abs(series.to_numpy(dtype=object)) <= MAX_EXACT_AMOUNT).all())` -> \"MONEY_MAGNITUDE\"); PufGrowthRefusalError (ValueError subclass, :207-213)" + ], + "refusal_codes": [ + "MONEY_MAGNITUDE" + ], + "encodes": "The magnitude of one integer money value before the float64 working conversion (float64's exact-integer width).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "PUF amounts are at most 12 digits (< 10**12); width 2**53", + "bytes_at_full_source": "same; invariant to the survey fraction", + "measurement_basis": "Read from the code; puf_monetary_source._MAX_AMOUNT_DIGITS = 12 bounds the delivered tokens.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports puf_growth (only puf_price_baseline, puf_monetary_agi_projection, graph_puf_detail_transfer, puf_detail_transfer, puf_growth_graph, graph_full_puf_enrichment do). Would reach: apply_puf_growth (:1305) -> _money_values (:1348, :1373).", + "classification": "must-not-move", + "notes": "An encoder width (float64 mantissa), not a resource ceiling." + }, + { + "constant": "puf_growth.py:197 MAX_ROWS", + "value": "2_000_000 (literal)", + "base_value": "2,000,000", + "head_value": "2,000,000", + "enforced_at": [ + "puf_growth.py:1207 (_checked_source_table: `_require(0 < rows <= MAX_ROWS, \"SOURCE_TABLE_ROWS\")`)", + "puf_growth.py:1441 (apply_puf_design_weight_growth: `_require(values.ndim == 1 and 0 < values.size <= MAX_ROWS, \"WEIGHT_SHAPE\")`)" + ], + "refusal_codes": [ + "SOURCE_TABLE_ROWS", + "WEIGHT_SHAPE" + ], + "encodes": "Row count of the PUF money view / design-weight vector at IRS return grain (207,696 delivered returns; 207,692 after the four disclosure aggregates are removed by puf_price_baseline.adapt_projection_to_money_view).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "207,696 rows (a row count; the PUF file does not scale with the survey fraction)", + "bytes_at_full_source": "207,696 rows (same)", + "measurement_basis": "puf_2015_raw_source_definition.json sources.main.data_records = 207696; comment at :195-196 calls it an allocation guard.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path (see MAX_EXACT_AMOUNT); would reach via apply_puf_growth / apply_puf_design_weight_growth.", + "classification": "not-a-transport", + "notes": "A row count with 9.6x headroom over the fixed PUF delivery; the row-count lane's PUF-grain rule applies." + }, + { + "constant": "puf_growth.py:200 MAX_DOCUMENT_BYTES", + "value": "512 * 1024 = 524,288", + "base_value": "524,288", + "head_value": "524,288", + "enforced_at": [ + "puf_growth.py:233-236 (_parse: `type(document) is bytes and 0 < len(document) <= MAX_DOCUMENT_BYTES` -> \"DOCUMENT_SIZE_OR_TYPE\"); reached from parse_growth_recipe (:555), GrowthFactorTable.from_bytes (:738) and GrownPufTable.receipt (:1195)", + "puf_growth.py:708-712 (GrowthFactorTable.__post_init__: `and 0 < len(self.document) <= MAX_DOCUMENT_BYTES` -> \"DOCUMENT_SIZE_OR_TYPE\")" + ], + "refusal_codes": [ + "DOCUMENT_SIZE_OR_TYPE" + ], + "encodes": "Fixed-shape JSON documents: a growth factor table, a recipe identity, and the apply_puf_growth receipt. No per-return data.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "factor document 1,008 B; recipe identity 3,204 B; growth receipt 898 B (measured with the CPI-U price baseline over the 13 monetary fields)", + "bytes_at_full_source": "same (row-independent; the receipt carries `rows` as one integer)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> growth_documents: puf_price_baseline.price_baseline_factor_document / compile_price_baseline on invented resource bytes carrying the two published CPI-U records, then apply_puf_growth on an 8-row table.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path; would reach via GrowthFactorTable.from_bytes / parse_growth_recipe / GrownPufTable.receipt.", + "classification": "not-a-transport", + "notes": "520x headroom; a parser allocation guard on fixed documents." + }, + { + "constant": "puf_growth.py:261 ", + "value": "len(value) == 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_growth.py:258-263 (_digest) used at :847-849 (`_require(_digest(factors.provenance[\"resource_sha256\"]), \"DEVELOPMENTAL_RESOURCE_DIGEST\")`)" + ], + "refusal_codes": [ + "DEVELOPMENTAL_RESOURCE_DIGEST" + ], + "encodes": "One sha256 hex string in a factor table's provenance.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path; CompiledPufGrowth.__post_init__ (:941-942) -> _check_developmental_provenance.", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "puf_growth.py:268 ", + "value": "0 < len(value) <= 128 -> 128", + "base_value": "128", + "head_value": "128", + "enforced_at": [ + "puf_growth.py:266-272 (_identifier: `type(value) is str and 0 < len(value) <= 128 and value.strip() == value`), raising the caller's reason: RULE_FIELD_NAME (:376), RULE_FACTOR_SERIES (:401), RECIPE_ID (:437), SERIES_NAME (:623), TABLE_ID (:713), RESOLVED_FIELD (:882), RESOLVED_SERIES (:889), ROSTER_FIELD (:1500)" + ], + "refusal_codes": [ + "RULE_FIELD_NAME", + "RULE_FACTOR_SERIES", + "RECIPE_ID", + "SERIES_NAME", + "TABLE_ID", + "RESOLVED_FIELD", + "RESOLVED_SERIES", + "ROSTER_FIELD" + ], + "encodes": "Identifier strings (field names, series names, recipe/table ids), one scalar each.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "the longest packaged identifier is PRICE_BASELINE_SERIES_NAME (52 chars); cap 128", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code and puf_price_baseline.py:140.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Identifier width." + }, + { + "constant": "puf_growth.py:666 ", + "value": "depth <= 64 (JSON nesting depth) -> 64", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_growth.py:664-666 (_freeze_provenance: `_require(depth <= 64, \"TABLE_PROVENANCE_DEPTH\")`)" + ], + "refusal_codes": [ + "TABLE_PROVENANCE_DEPTH" + ], + "encodes": "Nesting depth of a factor table's provenance tree (a JSON token cap, not bytes).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "depth 1 in the packaged price-baseline provenance; cap 64", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code and puf_price_baseline.py:327-344.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path; GrowthFactorTable.__post_init__ (:730).", + "classification": "must-not-move", + "notes": "A token/structure cap on a fixed document." + }, + { + "constant": "puf_monetary_source.py:180 BODY_MAX_BYTES", + "value": "64 * 1024 * 1024 = 67,108,864", + "base_value": "67,108,864", + "head_value": "67,108,864", + "enforced_at": [ + "puf_monetary_source.py:1386-1389 (encode_monetary_projection: `total, sizes = payload_bound(rows, projection)` / `if total > BODY_MAX_BYTES: raise _refuse(\"BODY_OVER_BOUND\", total)`)", + "puf_monetary_source.py:1566-1568 (_envelope_columns, decode: `total += size` / `if total > BODY_MAX_BYTES: raise _refuse(\"ENVELOPE_BODY_OVER_BOUND\", total)`)", + "puf_monetary_agi_projection.py:292-294 (encode_agi_projection: same payload_bound check -> \"BODY_OVER_BOUND\"; decode goes through the imported _envelope_columns)" + ], + "refusal_codes": [ + "BODY_OVER_BOUND", + "ENVELOPE_BODY_OVER_BOUND" + ], + "encodes": "The column bodies of the monetary projection envelope over every delivered PUF return: 3 structural typed columns (RECID i8, disclosure_aggregate u1, demographic_status i1) plus, per projected column, an i8 typed body and a 13-byte fixed-width lexical body: 262 B/row for 12 columns, 283 B/row for the 13-column AGI edition.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "54,416,352 (12 columns x 207,696 rows) and 58,777,968 (13 columns); invariant to the survey fraction", + "bytes_at_full_source": "54,416,352 / 58,777,968 (same)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers: monetary.payload_bound(207696, packaged_projection()) and payload_bound(207696, packaged_agi_projection()); headroom 1.233x (cap admits 256,140 rows) and 1.142x (237,133 rows).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports puf_monetary_source; would reach via USPufMonetarySourceKernel.run (:1811) -> encode_monetary_projection, or decode_monetary_projection (:1616) -> _envelope_columns.", + "classification": "must-not-move", + "notes": "Sized by the upstream PUF delivery's row count (207,696) and the packaged column roster, never by the survey selection; the bodies exist whole (`bodies` list joined at :1457-1459). Only another projected column or a larger PUF vintage can move it." + }, + { + "constant": "puf_monetary_source.py:184 HEADER_MAX_BYTES", + "value": "64 * 1024 = 65,536", + "base_value": "65,536", + "head_value": "65,536", + "enforced_at": [ + "puf_monetary_source.py:1454-1456 (`encoded = canonical_json(header)` / `if len(encoded) > HEADER_MAX_BYTES: raise _refuse(\"HEADER_OVER_BOUND\", len(encoded))`)", + "puf_monetary_source.py:1642-1644 (decode: `length = int.from_bytes(payload[start : start + 4], \"big\")` / `if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: raise _refuse(\"ENVELOPE_HEADER_LENGTH\")`)", + "puf_monetary_agi_projection.py:356-358 (\"HEADER_OVER_BOUND\") and :413-415 (\"ENVELOPE_HEADER_LENGTH\")" + ], + "refusal_codes": [ + "HEADER_OVER_BOUND", + "ENVELOPE_HEADER_LENGTH" + ], + "encodes": "The canonical JSON envelope header: 27 column entries (12 columns) or 29 (13), source pins, and the facts block with column_facts for three row classes and column_metadata per column. No per-row data.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "25,916 B (12 columns) / 27,919 B (13 columns), measured with 207,696-row facts; headroom 2.53x / 2.35x", + "bytes_at_full_source": "same (row-independent)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers: header dict built exactly as :1418-1453 with facts from monetary._facts_from_arrays over invented 207,696-row arrays (sums of 10-digit amounts) and canonical_json.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path (see BODY_MAX_BYTES).", + "classification": "not-a-transport", + "notes": "Fixed-shape; grows only with more projected columns (about 2 KB per column from the 12->13 measurement). The 4-byte big-endian length prefix at :1458 and puf_monetary_agi_projection.py:360 is a wire width (2**32-1) that this cap keeps far from binding. The decoder's `canonical_json(recomputed) != canonical_json(dict(facts))` at :1722-1724 (and agi :484-486) is a consumer-side re-encoding of the fixed facts block." + }, + { + "constant": "puf_monetary_source.py:187 LEXICAL_WIDTH_MAX", + "value": "64 (literal)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_monetary_source.py:510-512 (_projected_column: `if width > LEXICAL_WIDTH_MAX: raise _refuse(\"PROJECTION_LEXICAL_WIDTH\", field)`)", + "puf_monetary_source.py:1549-1551 (_envelope_columns: `if width > LEXICAL_WIDTH_MAX or width != widths[field]: raise _refuse(\"ENVELOPE_COLUMN_WIDTH\", name)`; also used by puf_monetary_agi_projection.py:448)", + "per-token enforcement of the declared width (13) that this cap bounds: :872-873 \"AMOUNT_TOKEN_OVER_WIDTH\", :913-914 \"AGGREGATE_TOKEN_OVER_WIDTH\", :1329-1330 \"LEXICAL_OVER_WIDTH\", :840-841 \"KEY_TOKEN_OVER_WIDTH\" (RECID width 64 from the definition)" + ], + "refusal_codes": [ + "PROJECTION_LEXICAL_WIDTH", + "ENVELOPE_COLUMN_WIDTH", + "AMOUNT_TOKEN_OVER_WIDTH", + "AGGREGATE_TOKEN_OVER_WIDTH", + "LEXICAL_OVER_WIDTH", + "KEY_TOKEN_OVER_WIDTH" + ], + "encodes": "The per-cell allocation of a fixed-width NUL-padded ASCII lexical column (_LEXICAL_KIND :204), read back by `body[position * width : (position + 1) * width]` (:1343) and sized `rows * width` (:1365, :1552).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "13 declared for all 12 (and 13) packaged columns; cap 64", + "bytes_at_full_source": "13; invariant", + "measurement_basis": "puf_2015_monetary_source_projection.json and puf_2015_monetary_agi_source_projection.json (lexical_width 13 on every column, per the row-ceilings verdict read of both files).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "A wire-format width: changing it changes the stored artifact layout. The lower bound `width < digits + 1` at :515 makes 13 the exact declared width (12 digits + sign)." + }, + { + "constant": "puf_monetary_source.py:338 _MAX_AMOUNT_DIGITS", + "value": "12 (literal)", + "base_value": "12", + "head_value": "12", + "enforced_at": [ + "puf_monetary_source.py:514-516 (`if digits > _MAX_AMOUNT_DIGITS or width < digits + 1: raise _refuse(\"PROJECTION_WIDTH_BELOW_GRAMMAR\", field)`)", + "puf_monetary_source.py:879-880 (_amount: `if len(digits) > column.max_digits: raise _refuse(\"AMOUNT_TOKEN_OVER_DIGITS\", label)`)", + "puf_monetary_source.py:927-928 (_aggregate_token: \"AGGREGATE_TOKEN_OVER_DIGITS\")" + ], + "refusal_codes": [ + "PROJECTION_WIDTH_BELOW_GRAMMAR", + "AMOUNT_TOKEN_OVER_DIGITS", + "AGGREGATE_TOKEN_OVER_DIGITS" + ], + "encodes": "Decimal digits per delivered amount token (the publisher's 12-digit amount fields).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "12 declared; invariant", + "bytes_at_full_source": "12; invariant", + "measurement_basis": "Read from the code; the packaged projections declare field_width_digits 12.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "The AGGREGATE_SENTINEL = -(2**63) argument at :239-244 depends on tokens being at most twelve digits; not a resource ceiling." + }, + { + "constant": "puf_monetary_source.py:334 _INT64_MAX / :335 _INT64_MIN", + "value": "2**63 - 1 = 9,223,372,036,854,775,807 / -(2**63)", + "base_value": "2**63-1 / -(2**63)", + "head_value": "2**63-1 / -(2**63)", + "enforced_at": [ + "puf_monetary_source.py:845-846 (_key_token: `if number > _INT64_MAX: raise _refuse(\"KEY_TOKEN_INT64_OVERFLOW\")`)", + "puf_monetary_source.py:884-885 (_amount: `if not _INT64_MIN <= number <= _INT64_MAX: raise _refuse(\"AMOUNT_TOKEN_INT64_OVERFLOW\", label)`)", + "puf_monetary_source.py:1178-1180 (_column_facts: `total = int(sum(...))` / `if not _INT64_MIN <= total <= _INT64_MAX: raise _refuse(\"COLUMN_SUM_OVERFLOW\")`)" + ], + "refusal_codes": [ + "KEY_TOKEN_INT64_OVERFLOW", + "AMOUNT_TOKEN_INT64_OVERFLOW", + "COLUMN_SUM_OVERFLOW" + ], + "encodes": "The int64 storage width of typed bodies (`", + "value": "len(value) != 64 -> 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_monetary_source.py:429-436 (_hex64 -> \"PROJECTION_DIGEST\"), called at :585, :1428-1430, :1485, :1508, :1562, :1669-1672 and from puf_monetary_agi_projection.py:330, :372, :437-440" + ], + "refusal_codes": [ + "PROJECTION_DIGEST" + ], + "encodes": "sha256 hex strings in projection documents and envelope headers.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "puf_2015_raw_source_definition.json:1 csv_profile.record_cap_per_file", + "value": "1000000 (JSON literal) = 1,000,000 records per delivered file", + "base_value": "1,000,000", + "head_value": "1,000,000", + "enforced_at": [ + "puf_raw_source.py:724-738 (_records: `limit = profile[\"record_cap_per_file\"]` / `if count > limit: raise _refuse(\"RECORD_CAP\", pin.name)`)", + "puf_monetary_source.py:989-1004 (_records: same profile value -> \"RECORD_CAP\")" + ], + "refusal_codes": [ + "RECORD_CAP" + ], + "encodes": "Record count of one delivered PUF CSV as it is streamed (main 207,696 data records; demographic 119,675).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "207,696 records (main), 119,675 (demographic); invariant", + "bytes_at_full_source": "same", + "measurement_basis": "puf_2015_raw_source_definition.json sources.*.data_records.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path (USPufRawSourceKernel.run :1544 / decode_puf_monetary_source :1012).", + "classification": "must-not-move", + "notes": "A record cap on the real upstream files, defined in the packaged data document and pinned into the kernel identity through csv_acceptance_profile; the exact counts are separately asserted by _check_declared_expectations (:846-886, \"DECLARED_EXPECTATION\")." + }, + { + "constant": "puf_2015_raw_source_definition.json:1 csv_profile.logical_record_character_cap", + "value": "1048576 (JSON literal) = 1,048,576 characters per record", + "base_value": "1,048,576", + "head_value": "1,048,576", + "enforced_at": [ + "puf_raw_source.py:726-742 (_records: `cap = profile[\"logical_record_character_cap\"]` / `if sum(len(cell) for cell in record) > cap: raise _refuse(\"RECORD_CHARACTER_CAP\", pin.name)`)", + "puf_monetary_source.py:992-1008 (_records -> \"RECORD_CHARACTER_CAP\")" + ], + "refusal_codes": [ + "RECORD_CHARACTER_CAP" + ], + "encodes": "Character count of one delivered CSV record (222 fields for main, 7 for demographic).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "a main record averages about 607 characters (126,034,649 bytes / 207,697 lines); cap 1,048,576", + "bytes_at_full_source": "same; invariant", + "measurement_basis": "Derived from the pinned main bytes and data_records in puf_2015_raw_source_definition.json.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "A per-record width cap on the real upstream file; the paired `len(record) != width` (\"RECORD_WIDTH\", :739-740 / :1005-1006) is the exact 222/7 field count." + }, + { + "constant": "puf_2015_raw_source_definition.json:1 csv_profile.header_field_cap", + "value": "1024 (JSON literal) = 1,024 header fields", + "base_value": "1,024", + "head_value": "1,024", + "enforced_at": [ + "puf_raw_source.py:715-716 (_check_header: `if len(header) > profile[\"header_field_cap\"]: raise _refuse(\"HEADER_WIDTH_LIMIT\", pin.name)`)", + "puf_monetary_source.py:975-976 (_check_header -> \"HEADER_WIDTH_LIMIT\")" + ], + "refusal_codes": [ + "HEADER_WIDTH_LIMIT" + ], + "encodes": "The field count of the delivered CSV header row (222 main, 7 demographic).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "222 fields; cap 1,024", + "bytes_at_full_source": "same", + "measurement_basis": "puf_2015_raw_source_definition.json sources.main.delivered_header_width = 222.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Followed by the exact-header equality (\"HEADER_MISMATCH\", \"HEADER_DIGEST\")." + }, + { + "constant": "puf_2015_raw_source_definition.json:1 csv_profile.field_size_limit", + "value": "131072 (JSON literal) = 131,072 characters per CSV field (the stdlib csv module's process-global limit)", + "base_value": "131,072", + "head_value": "131,072", + "enforced_at": [ + "puf_raw_source.py:625-629 (_check_csv_profile: `if csv.field_size_limit() != declared: raise _refuse(\"CSV_FIELD_SIZE_LIMIT\")`, preceded by \"DEFINITION_CSV_FIELD_SIZE_LIMIT\")", + "puf_monetary_source.py:791-795 (_check_csv_profile: same two refusals)" + ], + "refusal_codes": [ + "DEFINITION_CSV_FIELD_SIZE_LIMIT", + "CSV_FIELD_SIZE_LIMIT" + ], + "encodes": "The width of one CSV cell the stdlib reader will accept; the code pins the process value to the declared one rather than setting it.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "the widest delivered cell is the 64-character RECID allocation; limit 131,072", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code and the definition document.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Part of the producer identity (csv_acceptance_profile, :589-612)." + }, + { + "constant": "puf_raw_source.py:117 PUF_RAW_SOURCE_MAX_BYTES", + "value": "128 * 1024 * 1024 = 134,217,728", + "base_value": "134,217,728", + "head_value": "134,217,728", + "enforced_at": [ + "puf_raw_source.py:318-319 (_pin: `if size > PUF_RAW_SOURCE_MAX_BYTES: raise _refuse(\"DEFINITION_SOURCE_OVER_CAP\", name)`)", + "puf_raw_source.py:502-503 (load_pinned_puf_bytes: `if pin.bytes > PUF_RAW_SOURCE_MAX_BYTES: raise _refuse(\"CODEC_PIN_OVER_CAP\", pin.name)`)", + "puf_raw_source.py:1218-1220 (_check_envelope_sources: `if size > PUF_RAW_SOURCE_MAX_BYTES: raise _refuse(\"ENVELOPE_SOURCE_OVER_CAP\", name)`)" + ], + "refusal_codes": [ + "DEFINITION_SOURCE_OVER_CAP", + "CODEC_PIN_OVER_CAP", + "ENVELOPE_SOURCE_OVER_CAP" + ], + "encodes": "The declared byte size of one delivered PUF file (main 126,034,649; demographic 2,225,050), read whole into memory by the codec (:519-542).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "126,034,649 (main); 2,225,050 (demographic); invariant", + "bytes_at_full_source": "same", + "measurement_basis": "puf_2015_raw_source_definition.json sources.main.bytes / sources.demographic.bytes; headroom 1.065x over the real main delivery.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path; would reach via puf_raw_source_codecs -> load_pinned_puf_bytes.", + "classification": "must-not-move", + "notes": "Chosen from the upstream file's real size (:29-31); exact identity is enforced separately by `before.st_size != pin.bytes` (\"CODEC_SIZE\", :513-514), `total != pin.bytes` (\"CODEC_LENGTH\", :532-533), sha256 and git-blob sha1." + }, + { + "constant": "puf_raw_source.py:521 ", + "value": "limit = pin.bytes + 1 -> 126,034,650 (main) / 2,225,051 (demographic): a read(N+1)-style cap keyed to the pin", + "base_value": "pin.bytes + 1", + "head_value": "pin.bytes + 1", + "enforced_at": [ + "puf_raw_source.py:521-528 (load_pinned_puf_bytes: `limit = pin.bytes + 1` / `chunk = os.read(descriptor, 1024 * 1024)` / `total += len(chunk)` / `if total > limit: raise _refuse(\"CODEC_GREW\", pin.name)`)" + ], + "refusal_codes": [ + "CODEC_GREW" + ], + "encodes": "Bytes read from the delivered PUF file, accumulated in 1 MiB chunks that are retained (`chunks.append(chunk)`, :531) and joined (:542).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "126,034,649 (main); invariant", + "bytes_at_full_source": "same", + "measurement_basis": "Definition document pins.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Protects the upstream file's real size: one byte over the pin refuses. The 1 MiB read size at :523 is a chunk, not a cap." + }, + { + "constant": "puf_raw_source.py:121 BODY_MAX_BYTES", + "value": "64 * 1024 * 1024 = 67,108,864", + "base_value": "67,108,864", + "head_value": "67,108,864", + "enforced_at": [ + "puf_raw_source.py:1086-1089 (encode_return_status: `total, sizes = _payload_bound(rows, definition)` / `if total > BODY_MAX_BYTES: raise _refuse(\"BODY_OVER_BOUND\", total)`)", + "puf_raw_source.py:1282-1284 (_envelope_columns: `total += size` / `if total > BODY_MAX_BYTES: raise _refuse(\"ENVELOPE_BODY_OVER_BOUND\", total)`)" + ], + "refusal_codes": [ + "BODY_OVER_BOUND", + "ENVELOPE_BODY_OVER_BOUND" + ], + "encodes": "The return-status artifact's column bodies over 207,696 returns: 14 typed columns (29 B/row) plus 12 fixed-width lexical columns (RECID 64, S006 32, FLPDYR 8, nine x 4 = 140 B/row) = 169 B/row.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "35,100,624 (207,696 x 169); invariant", + "bytes_at_full_source": "same; headroom 1.912x; the cap admits 397,093 rows", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers.puf_raw_source: raw._payload_bound(207696, packaged_definition()).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Sized by the upstream row count and the definition's widths; the survey fraction never enters." + }, + { + "constant": "puf_raw_source.py:124 HEADER_MAX_BYTES", + "value": "64 * 1024 = 65,536", + "base_value": "65,536", + "head_value": "65,536", + "enforced_at": [ + "puf_raw_source.py:1153-1155 (`encoded = canonical_json(header)` / `if len(encoded) > HEADER_MAX_BYTES: raise _refuse(\"HEADER_OVER_BOUND\", len(encoded))`)", + "puf_raw_source.py:1394-1396 (decode_return_status: `if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: raise _refuse(\"ENVELOPE_HEADER_LENGTH\")`)" + ], + "refusal_codes": [ + "HEADER_OVER_BOUND", + "ENVELOPE_HEADER_LENGTH" + ], + "encodes": "The return-status envelope header: 26 column entries, two source pins, and the fixed facts block (code_frequencies over a handful of code values).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "6,027 B measured with 207,696-row facts; headroom 10.9x", + "bytes_at_full_source": "same (row-independent)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers.puf_raw_source: header built exactly as :1124-1152 with raw._facts_from_arrays over invented 207,696-row typed arrays.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "not-a-transport", + "notes": "The 4-byte big-endian length prefix at :1157 is the wire width; the decoder's `canonical_json(recomputed) != canonical_json(dict(facts))` at :1464-1465 re-encodes the fixed facts block." + }, + { + "constant": "puf_raw_source.py:185 _LEXICAL_WIDTH_MAX", + "value": "64 (literal)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_raw_source.py:1271-1273 (_envelope_columns: `width = _envelope_int(entry[\"width\"], f\"{name}.width\", minimum=1)` / `if width > _LEXICAL_WIDTH_MAX: raise _refuse(\"ENVELOPE_COLUMN_WIDTH\", name)`), followed by the exact-width match at :1274-1275 whenever a definition resolves", + "the independent hardcoded 64 at puf_raw_source.py:381-383 (_definition_from_document: `not 0 < width <= 64` -> \"DEFINITION_LEXICAL_WIDTH\")", + "per-token enforcement of the declared widths: :664-665 (_token \"TOKEN_OVER_WIDTH\"), :1037-1038 (_fixed_ascii \"LEXICAL_OVER_WIDTH\")" + ], + "refusal_codes": [ + "ENVELOPE_COLUMN_WIDTH", + "DEFINITION_LEXICAL_WIDTH", + "TOKEN_OVER_WIDTH", + "LEXICAL_OVER_WIDTH" + ], + "encodes": "Per-cell allocation of the fixed-width NUL-padded lexical bodies (RECID declares exactly 64, S006 32, FLPDYR 8, nine fields 4).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 (RECID, exactly at the cap); invariant", + "bytes_at_full_source": "same", + "measurement_basis": "puf_2015_raw_source_definition.json fields[*].lexical_width.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "A wire-format width mirrored by three separate 64 literals (:185, :382, and puf_monetary_source.py:187)." + }, + { + "constant": "puf_raw_source.py:230 _INT64_MAX", + "value": "2**63 - 1 = 9,223,372,036,854,775,807", + "base_value": "2**63-1", + "head_value": "2**63-1", + "enforced_at": [ + "puf_raw_source.py:671-675 (_int64: `if value > _INT64_MAX: raise _refuse(\"TOKEN_INT64_OVERFLOW\", label)`)" + ], + "refusal_codes": [ + "TOKEN_INT64_OVERFLOW" + ], + "encodes": "The int64 storage width of the typed RECID/S006 columns (`", + "value": "len(value) != 64 / len(value) != 40 (sha256 / git-blob sha1 hex widths)", + "base_value": "64 / 40", + "head_value": "64 / 40", + "enforced_at": [ + "puf_raw_source.py:284-291 and :294-301 (-> \"DEFINITION_DIGEST\"), called at :325-329, :1192, :1215-1217, :1278" + ], + "refusal_codes": [ + "DEFINITION_DIGEST" + ], + "encodes": "Digest strings in the definition document and envelope header.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "64 / 40 characters exactly", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Fixed-width fields." + }, + { + "constant": "puf_full_source.py:27 FULL_SOURCE_MAX_BYTES", + "value": "128 * 1024 * 1024 = 134,217,728", + "base_value": "134,217,728", + "head_value": "134,217,728", + "enforced_at": [ + "puf_full_source.py:194-197 (decode_full_puf_source: `rows * len(PROJECTED_COLUMNS) * 8 <= FULL_SOURCE_MAX_BYTES` -> \"FULL_PROJECTION_BODY_LIMIT\")", + "puf_full_source.py:290 (encode_full_puf_source: `_require(sum(len(b) for b in blocks) <= FULL_SOURCE_MAX_BYTES, \"FULL_BODY_LIMIT\")`, blocks = status payload + 59 int64 columns)", + "puf_full_source.py:297-300 (decode_full_puf_artifact: `len(payload) <= len(_MAGIC) + 8 + _HEADER_LIMIT + FULL_SOURCE_MAX_BYTES` -> \"FULL_ARTIFACT_LIMIT\")", + "puf_full_source.py:346-350 (`0 < h[\"status_bytes\"] <= FULL_SOURCE_MAX_BYTES` -> \"FULL_STATUS_LENGTH\")" + ], + "refusal_codes": [ + "FULL_PROJECTION_BODY_LIMIT", + "FULL_BODY_LIMIT", + "FULL_ARTIFACT_LIMIT", + "FULL_STATUS_LENGTH" + ], + "encodes": "Two sums at PUF grain: the 59 x int64 projection body (472 B/row) at :195, and at :290 that body plus the embedded return-status payload (169 B/row of bodies plus its 6 KB header).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": ":195 arm 98,032,512; :290 arm about 133,139,209 (estimate: 98,032,512 + 35,100,624 body + 6,027 header + magic/prefix); invariant", + "bytes_at_full_source": "same; headroom at :290 about 1,078,519 B (0.8%); the encode cap admits about 209,378 rows against 207,696 delivered", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers.puf_full_source: exact column arithmetic plus the measured raw header; an actual status payload was not built (open_questions).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports puf_full_source (importers: graph_full_puf_enrichment, puf55_survey_recipients, graph_puf55_canonical_donor, puf59_canonical, puf_full_source_graph).", + "classification": "must-not-move", + "notes": "The tightest fixed ceiling in the family, but it cannot move with the survey fraction; the exact upstream sizes are asserted at :190 (`len(data) == pin.bytes` -> \"FULL_SOURCE_SIZE\") and :343-345 (\"FULL_ARTIFACT_ROWS\")." + }, + { + "constant": "puf_full_source.py:94 AGGREGATE_LEXEME_MAX_CHARACTERS", + "value": "64 (literal; the name is never compared - the value is duplicated into `_AGGREGATE = re.compile(r\"[\\x20-\\x7e]{0,64}\\Z\", re.ASCII)` at :95)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_full_source.py:219-222 (decode_full_puf_source: `all(_AGGREGATE.fullmatch(token) for token in tokens.values())` -> \"FULL_AGGREGATE_LEXICAL_GRAMMAR\")", + "puf_full_source.py:391-398 (decode_full_puf_artifact: `type(v) is str and _AGGREGATE.fullmatch(v)` -> \"FULL_AGGREGATE_LEXICAL_GRAMMAR\")" + ], + "refusal_codes": [ + "FULL_AGGREGATE_LEXICAL_GRAMMAR" + ], + "encodes": "Characters of one retained disclosure-aggregate token: 4 aggregate rows x 59 projected columns = 236 tokens, kept verbatim in the JSON header.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "at most 64 characters per token; the delivered widths live in the gated PUF file and were not read", + "bytes_at_full_source": "same; invariant", + "measurement_basis": "Read from the code; the header measurement below bounds the worst case (64-char tokens) at 24,845 B.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "A lexical grammar over upstream-delivered literals; editing the named constant alone changes nothing (maintenance trap noted by the row-ceilings lane)." + }, + { + "constant": "puf_full_source.py:254 _HEADER_LIMIT", + "value": "128 * 1024 = 131,072", + "base_value": "131,072", + "head_value": "131,072", + "enforced_at": [ + "puf_full_source.py:289 (encode: `_require(len(header) <= _HEADER_LIMIT, \"FULL_HEADER_LIMIT\")`)", + "puf_full_source.py:297-300 (a term of \"FULL_ARTIFACT_LIMIT\")", + "puf_full_source.py:303-304 (decode: `size = struct.unpack(\" 0, \"FULL_HEADER_LIMIT\")`)" + ], + "refusal_codes": [ + "FULL_HEADER_LIMIT", + "FULL_ARTIFACT_LIMIT" + ], + "encodes": "The full-source artifact's JSON header: 59 column entries plus the 236 retained aggregate tokens and source pins.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "12,573 B with 12-character aggregate tokens; 24,845 B worst case at 64-character tokens; headroom 10.4x / 5.3x", + "bytes_at_full_source": "same (row-independent)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> puf_headers.puf_full_source: header built as :272-288 with json.dumps(sort_keys, ascii).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "not-a-transport", + "notes": "The 8-byte `struct.pack(\" _MONEY / :91 _COUNT", + "value": "`-?[0-9]{1,12}` (12 digits) and `[0-9]{1,2}` (2 digits); mirrored by `cap = 99 if name in COUNT_COLUMNS else 999_999_999_999` at :404", + "base_value": "12 / 2 digits", + "head_value": "12 / 2 digits", + "enforced_at": [ + "puf_full_source.py:225-231 (decode_full_puf_source: `grammar.fullmatch(token) is not None` -> \"FULL_ORDINARY_TOKEN_GRAMMAR:\" + name)", + "puf_full_source.py:404-409 (decode_full_puf_artifact: `((value[~aggregate] >= -cap) & (value[~aggregate] <= cap)).all()` -> \"FULL_ORDINARY_TYPED_DOMAIN\")" + ], + "refusal_codes": [ + "FULL_ORDINARY_TOKEN_GRAMMAR", + "FULL_ORDINARY_TYPED_DOMAIN" + ], + "encodes": "Digit width of one ordinary-return amount or count token (the publisher's 12-digit amount fields; 2-digit exemption counts).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "12 / 2 digits; invariant", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "Not on any runner path.", + "classification": "must-not-move", + "notes": "Token grammar widths on upstream literals; the same 12-digit rule as puf_monetary_source._MAX_AMOUNT_DIGITS." + }, + { + "constant": "puf_price_baseline.py:153 _CPI_RESOURCE_MAX_BYTES", + "value": "16 * 1024 * 1024 = 16,777,216", + "base_value": "16,777,216", + "head_value": "16,777,216", + "enforced_at": [ + "puf_price_baseline.py:267-270 (verify_cpi_resource_bytes: `_require(len(resource_bytes) <= _CPI_RESOURCE_MAX_BYTES, \"PRICE_BASELINE_RESOURCE_OVERSIZE\")`); PufGrowthRefusalError via _require at :179-181" + ], + "refusal_codes": [ + "PRICE_BASELINE_RESOURCE_OVERSIZE" + ], + "encodes": "The retained BLS flat file cu.data.1.AllItems (an upstream resource), split into lines after the check (:216).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "about 2.7 MB per the comment at :150-152; no in-tree copy of the file exists, so not measured", + "bytes_at_full_source": "same; invariant", + "measurement_basis": "Comment at :150-152 only; tests/test_us_puf_price_baseline.py:421-426 exercises cap + 1.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports puf_price_baseline (importers: graph_full_puf_enrichment, graph_puf_detail_transfer).", + "classification": "must-not-move", + "notes": "Protects an upstream file's real size (6x headroom by the comment's figure); exact identity is by the whole-file sha256 at :272-273." + }, + { + "constant": "full_puf_enrichment.py:156 ", + "value": "abs(int(v)) <= 2**53 -> 9,007,199,254,740,992", + "base_value": "2**53", + "head_value": "2**53", + "enforced_at": [ + "full_puf_enrichment.py:151-160 (_numeric: `all(not isinstance(v, (int, np.integer)) or abs(int(v)) <= 2**53 for v in observed)` -> f\"PUF_FLOAT64_INTEGER_RANGE:{label}\"); ValueError(code) via _require at :130-132" + ], + "refusal_codes": [ + "PUF_FLOAT64_INTEGER_RANGE:{label}" + ], + "encodes": "The magnitude of one integer donor/recipient cell before the float64 model boundary.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "PUF-derived integers are at most 12 digits; width 2**53", + "bytes_at_full_source": "same", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports full_puf_enrichment (importers: graph_puf55_canonical_donor, graph_full_puf_enrichment, puf55_*).", + "classification": "must-not-move", + "notes": "An encoder width (float64), duplicated from puf_growth.MAX_EXACT_AMOUNT as a literal." + }, + { + "constant": "graph_full_puf_enrichment.py:580 ", + "value": "none (consumer-side re-encoding `values[\"placement\"].payload == _placement(binding)`; the encoder codec.encode_json has no cap and no len() check exists at the consumer)", + "base_value": "none", + "head_value": "none", + "enforced_at": [ + "graph_full_puf_enrichment.py:611-615 (_checked_artifacts: `require(values[\"placement\"].payload == _placement(binding), \"FULL_PUF_PLACEMENT_BINDING\")`), reached from FullPufAttachKernel.run (:725), verify_materialized_full_puf_attachment (:742-744) and load_full_puf_attachment_artifacts (:866)" + ], + "refusal_codes": [ + "FULL_PUF_PLACEMENT_BINDING" + ], + "encodes": "The placement artifact `codec.encode_json({**params, \"target_order\", \"selected_ids\": {entity: frame.table(entity).loc[mask, id_column].tolist()}, ...})` (:580-594): the id lists of every person and tax unit under the PUF clone masks, i.e. the clone-1 half of the clone roster (stacked persons plus stacked tax units).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "5,127,790 SCALED (9.0 B per 8-digit clone id x 356,501 persons + 9.0 x 213,253 tax units)", + "bytes_at_full_source": "51,277,932 SCALED (9.0 x 3,565,013 persons + 9.0 x 2,132,535 tax units; tax units are the 1/1000 roster ratio 2128/1584)", + "measurement_basis": "scratchpad receipt roles-status-puf-measurements.json -> id_lists: codec.encode_json({\"selected_ids\": {\"person\": ids}}) at 1 vs 100,001 ids with 8-digit clone-1 ids (10**7 + stacked ordinal).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports graph_full_puf_enrichment (its importers are the property/housing/health graphs and the puf55 route, none on the 19/45/age paths); would reach via full_puf_attachment_nodes -> FullPufMaskKernel.run (:699) producing it and FullPufAttachKernel.run (:718) re-encoding it.", + "classification": "segment", + "notes": "A whole-roster JSON produced whole and re-encoded whole by three consumers; no explicit cap anywhere, so binds is false by definition and the segmented shape is the lane's choice. The `to_bytes(8, \"little\")` at :113 is a digest length prefix, not a bound." + }, + { + "constant": "graph_current_survey_puf_transfer.py:131 ", + "value": "none (consumer-side re-encoding `placement.payload == _binding(...)`; codec.encode_json has no cap)", + "base_value": "none", + "head_value": "none", + "enforced_at": [ + "graph_current_survey_puf_transfer.py:288-298 (CurrentSurveyAttachKernel.run: `placement.payload == _binding(projection.payload, matrix.payload, matrix.producer_key, self.clone_population.frame, mask)` -> \"SURVEY_TRANSFER_PLACEMENT_BINDING\")", + "graph_current_survey_puf_transfer.py:388-394 (verify_materialized_current_survey_transfer: same equality)" + ], + "refusal_codes": [ + "SURVEY_TRANSFER_PLACEMENT_BINDING" + ], + "encodes": "The placement artifact `codec.encode_json({..., \"selected_ids\": frame.table(\"tax_unit\").tax_unit_id.to_numpy()[mask].tolist(), ...})` (:131-147): the tax-unit ids under detail.MASK (clone-1 tax units).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "1,919,281 SCALED (9.0 B x 213,253 tax units)", + "bytes_at_full_source": "19,192,815 SCALED (9.0 B x 2,132,535 tax units)", + "measurement_basis": "Same encoder measurement as the graph_full_puf_enrichment row (9.0 B per 8-digit clone id through codec.encode_json); census-brief \u00a72 says not to spend measurement effort here, so this is the one-line per-id figure only.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "census-brief \u00a72 lists graph_current_survey_puf_transfer as off every path; its only importer is graph_full_puf_enrichment.", + "classification": "segment", + "notes": "No cap at producer or consumer; recorded for completeness." + }, + { + "constant": "current_survey_amounts.py:219 ", + "value": "none (consumer-side re-encoding of another module's roster document: `codec.sha(uc.person.to_json(orient=\"table\").encode()) == uc.evidence[\"projection_sha256\"]`; pandas to_json has no cap)", + "base_value": "none", + "head_value": "none", + "enforced_at": [ + "current_survey_amounts.py:217-222 (qualify_current_survey_amounts: `require(codec.sha(uc.person.to_json(orient=\"table\").encode()) == uc.evidence[\"projection_sha256\"], \"UC_PROJECTION_CHANGED\")`)" + ], + "refusal_codes": [ + "UC_PROJECTION_CHANGED" + ], + "encodes": "The ASEC-arm unemployment person table produced by current_asec_unemployment_source.qualify_current_asec_unemployment (one row per ASEC original person), re-encoded whole as pandas orient=\"table\" JSON for a digest comparison.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "not measured (14,212 ASEC persons at 1/10; the producer's column roster lives in current_asec_unemployment_source, another family's module)", + "bytes_at_full_source": "not measured (142,125 ASEC persons)", + "measurement_basis": "Not measured: the document's schema is defined outside this family; see open_questions.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "census-brief \u00a72 lists current_survey_amounts as off every path (importers: graph_current_survey_housing, graph_full_puf_enrichment, graph_us_survey_enrichment).", + "classification": "segment", + "notes": "The same function also materialises four more roster-sized to_json(orient=\"table\") strings for digests with no cap: origins (:139, stacked persons), native and reports (:366-367, stacked persons x amount columns), and each route's donor_columns (:345, ASEC donors). Only the final evidence document is bounded (see the MAX_PAYLOAD_BYTES row)." + }, + { + "constant": "puf_support.py:2331 _INT64_MAX", + "value": "2**63 - 1 = 9,223,372,036,854,775,807", + "base_value": "2**63-1", + "head_value": "2**63-1", + "enforced_at": [ + "puf_support.py:2360-2368 (_remap_ids: `shift = int(clone_index) * int(id_multiplier)` / `if len(values) and shift + int(values.max()) > _INT64_MAX: raise ValueError(\"PUF support ID remap would overflow int64: ... Assembled source IDs must not exceed {PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID}.\")`, a prose ValueError with no short code)" + ], + "refusal_codes": [ + "(none: prose ValueError \"PUF support ID remap would overflow int64: ...\")" + ], + "encodes": "The int64 storage width of the structural id columns after the decimal clone remap (id + clone_index * 10**digits(max_id)).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "max remapped id 10**6 + 356,501 = 1,356,501 (stacked persons set the pre-clone maximum)", + "bytes_at_full_source": "max remapped id 10**7 + 3,565,013 = 13,565,013; width 9.2 x 10**18 (11.8 orders of magnitude of headroom)", + "measurement_basis": "Derived from _id_multiplier_for_values (:2334-2345, `10 ** max(1, len(str(max_id)))`) computed once on the pre-clone frame (:578) and the census-brief stacked-person counts.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "graph_combined_clone (nodes combined_survey_puf_support_clone / .owned) -> CombinedCloneExpandKernel.run (graph_combined_clone.py:527-531) -> puf_support.clone_us_frame_for_puf_support (:491) -> _clone_preassembled_entity_table (:583, :2082-2103) and _clone_link_table (:598) -> _remap_ids (:2348-2369). Also graph_survey_population._verify_cloned_frame (graph_survey_population.py:797-808) -> puf_support._id_multiplier_for_frame (no bound).", + "classification": "must-not-move", + "notes": "An encoder width. The same 19-node chain continues into validate_puf_clone_attachment (fraction 1.0, graph_combined_clone.py:149 and :533) -> _validate_full_clone_identity (:1128-1179) -> _source_ids_sha256 (:1182-1187), which materialises a JSON list of every detail household source id (8.0 B/id: 1,269,896 B at 1/10, 12,699,008 B at full source) for a digest with no cap, twice per clone (also at :636-641 inside clone_us_frame_for_puf_support); not a bound, so no row (open_questions)." + }, + { + "constant": "operator_column_contracts.py:64 PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID (re-exported at puf_support.py:53-55)", + "value": "10**15 - 1 = 999,999,999,999,999", + "base_value": "10**15 - 1", + "head_value": "10**15 - 1", + "enforced_at": [ + "spine_assembly.py:630-638 (`oversized_ids = np.unique(ids[ids > PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID])`, prose ValueError naming \"the clone-safe bound\"; module outside this family, grep-verified at this head)", + "spine_assembly.py:763-766 (post-remap re-check, prose ValueError)", + "puf_support.py:2362-2368 names it in the _remap_ids overflow message only (not a comparison)" + ], + "refusal_codes": [ + "(none: prose ValueErrors in spine_assembly.py:632-641 and :764-767)" + ], + "encodes": "The largest assembled structural id value the clone remap may shift (an id-space width so that id + clone_index * 10**16 stays inside int64).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "largest assembled id about 356,501 (stacked persons)", + "bytes_at_full_source": "about 3,565,013; the bound is 2.8 x 10**8 times larger", + "measurement_basis": "Census-brief counts; the comment at puf_support.py:2324-2329 gives the derivation.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "The enforcing module spine_assembly is not imported by any of the three runners or by graph_combined_clone (which imports only spine_assembly_manifest from support_provenance); the 19-node chain reaches only the message at puf_support.py:2367.", + "classification": "must-not-move", + "notes": "An int64 arithmetic-width argument; the definition is in operator_column_contracts (another family's module); recorded here because the row-ceilings census keyed it to puf_support.py:54." + }, + { + "constant": "puf_support.py:1079 ", + "value": "len(tail_sha) != 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "puf_support.py:1077-1082 (validate_puf_clone_attachment, version-2 manifests: `not isinstance(tail_sha, str) or len(tail_sha) != 64 or any(character not in \"0123456789abcdef\" ...)` -> ValueError(f\"{boundary}: tail-descendant manifest SHA-256 is malformed.\"))" + ], + "refusal_codes": [ + "(prose ValueError \"...tail-descendant manifest SHA-256 is malformed.\")" + ], + "encodes": "One sha256 hex string in the tail-descendant attachment manifest.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "The 19-node combined clone uses fraction 1.0 with no manifest, so validate_puf_clone_attachment returns from _validate_full_clone_identity (:913-917) before this branch; only a version-2 (tail descendant, bind_puf_clone_attachment_tail_descendant) manifest reaches :1053-1082.", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "post_transfer_calibration.py:1189 ", + "value": "len(value) != 64 (hex digest width)", + "base_value": "64", + "head_value": "64", + "enforced_at": [ + "post_transfer_calibration.py:1187-1195 (validate_post_transfer_calibration_receipt: nine scope/weights digests `not isinstance(value, str) or len(value) != 64 or any(character not in \"0123456789abcdef\" ...)` -> ValueError(f\"{boundary}: post-transfer calibration context digest is invalid.\"))" + ], + "refusal_codes": [ + "(prose ValueError \"...post-transfer calibration context digest is invalid.\")" + ], + "encodes": "sha256 hex strings in a calibration receipt.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 characters exactly", + "bytes_at_full_source": "64 characters exactly", + "measurement_basis": "Read from the code.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "No runner imports post_transfer_calibration (importers: graph_full_puf_enrichment, stacked_spine).", + "classification": "must-not-move", + "notes": "The only byte-shaped check in the module; its value arrays are digested whole via `np.ascontiguousarray(values).tobytes()` (:565-574) with no cap, which is a digest, not a bound." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/other/measure_roles_status_puf.py (run with /Users/maxghenis/PolicyEngine/_worktrees/microcosm-native-byte-transports/.venv/bin/python, sys.path set to packages/*/src, every imported module checked to resolve under the worktree)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/other/roles-status-puf-measurements.json (receipt: roles_projection, roles_receipt, person_status_projection, person_status_receipt, id_lists, puf_headers, growth_documents)" + ], + "open_questions": [ + "byte-census-measurements.json roles_projection (324.0 B/person, 1,155,064,212 at full source) is not the document at this head: byte_census_measure.roles_projection builds a 7-column DataFrame directly (omitting UNBOUND_REASON, HOUSEHOLD_VERDICT, VALUE_KNOWN, ALLOCATION_KNOWN, ORIGIN, OBSERVATION_YEAR and VALUE) with placeholder strings (\"observed_reference_person\" in the int64 ROLE_STATE column, \"housing_unit\" for UNIVERSE). Through roles._observation_table with all 13 COLUMN_TOKENS and the real ORIGIN/UNIVERSE literals (55 and 27 characters) the projection is 650.0 B/person: 231,725,650 at 1/10 and 2,317,258,450 at full source; the 64 MiB cap admits 103,244 persons, not 207,126. The row-ceilings census's computed 648 B/row agrees. The lane JSON should be re-measured.", + "Unbounded whole-roster materialisation on the 19-node path: puf_support._source_ids_sha256 (:1182-1187) json.dumps's the list of every detail household source id (8.0 B/id: 1,269,896 B at 1/10, 12,699,008 B at full source) for a sha256, twice per clone (clone_us_frame_for_puf_support :636-641 and graph_combined_clone.py:533, both via _validate_full_clone_identity :1178). No cap exists so it is not a row; a streaming digest would remove the copy.", + "Unbounded materialisations beside the person-status projection (reachable only from person_status=True graphs): graph_current_survey_person_status._columns_payload (:391-401) computes `_sha(canonical_json(_table_document(table)))` for the 45-column literal table (816,423,843 B at full source) and the 121-column recoded table (1,351 B per ACS person, 1,606 per ASEC person: 485,377,090 B at 1/10 and 4,853,789,741 B at full source), each on every _expected_artifacts/_result call; and projection_payload re-encodes the 1.55 GB projection on every _check_artifacts. Likewise current_survey_household_roles._table_stamp (:638-642) and the roles graph _seal_result (:452-469) materialise `to_json()` per column (the bound is_household_head column spans 7,130,026 clone persons at full source) for digests with no cap.", + "current_survey_amounts materialises five roster-sized to_json(orient=\"table\") strings for digests with no cap (:139 origins, :219 uc.person, :345 donor_columns, :366 native, :367 reports); the UC_PROJECTION_CHANGED row's bytes were not measured because uc.person's schema is defined in current_asec_unemployment_source (another family). Off every runner path.", + "Cross-family bounds this family calls into without owning them: current_survey_person_status_source.py:387 passes `budget=[original.asec._BODY_MAX]` (asec_coverage_authentication.py:39, 268,435,456) to that module's _capture (:309-330, \"COVERAGE_BODY_BYTES\" at :286) on the ASEC pppub25.csv member; roles :1075 and status_source :406-410 call acs_housing_universe_source._copy(..., exact_size=size) (\"FILE_TOO_LARGE\"/\"SOURCE_SIZE\", :131-140) on the ACS person archive; roles :769-770 and status_source :80-81 stream records through acs_person_coverage_authentication._records/_decode_record, whose MAX_RECORD_BYTES 400,000 / MAX_TOKEN_BYTES 65,536 / MAX_CSV_HEADER_BYTES 65,536 (:48-50) apply first. Those rows belong to the coverage families; the MAX_PAYLOAD_BYTES row here lists only this family's two enforcement sites and must be merged into the survey-population family's definition row.", + "Read chunk sizes that are not caps and were not given rows: current_survey_household_roles.py:1091 and current_survey_person_status_source.py:426 `while stream.read(65536)` (exhausting auxiliary zip members for CRC), puf_qrf_chain.py:1063 and puf_raw_source.py:523 1 MiB chunks. Digest length prefixes `to_bytes(8, ...)` at graph_full_puf_enrichment.py:113 and multispine_pool.py:3484, and `struct.pack(\" MAX_PAYLOAD_BYTES: segments.append(bytes(payload)); payload = bytearray()` then `_require(len(piece) <= MAX_PAYLOAD_BYTES - len(payload), \"TRANSPORT_LIMIT\")` \u2014 SurveyOriginBudgetError(ValueError) (:79-85)", + "survey_origin_budget.py:92-93 `_json = graph._bounded_json(value, MAX_PAYLOAD_BYTES)` \u2192 graph_survey_population.py:272 (limit itself \u2264 64 MiB), :281, :283 \"TRANSPORT_LIMIT\" \u2014 SurveyPopulationGraphError(ValueError) (graph_survey_population.py:116-122); pieces built with it at :645 (one id/group entry), :653 (key), :659 (one origin record), :664 (one fixed header value)", + "survey_origin_budget.py:277 `_json([asdict(record) for record in population.mass_ledger])` in _population_identity (fixed-shape ledger, one record per node transition)", + "survey_financial_successor.py:116-131 `budget._json({...})` for the fixed-shape financial successor document (reachable_from none; see that row)" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT" + ], + "encodes": "The sampling-origin budget document: one origin record per selected household (`origins`, 709-742 B each through _json at :659) plus two header lists with one entry per clone household row (`household_ids`, `group_indices`, streamed element by element at :637-646), plus a fixed header (:594-616; 26,258 B with the 1/1000 receipt's producer/native/catalogues). At this head the constant bounds one piece and one accumulation (segment) of that stream; the whole document is bounded by MAX_ROSTER_BYTES (:633, :668).", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "117,809,350 = 742.0 B/group \u00d7 158,737 + 26,258 B preamble (measured, variant 'measurements_json_params'); byte-census-measurements.json origin_budget.tenth_bytes = 117,782,854 = 742 \u00d7 158,737 without the preamble \u2014 same encoder and same record fields at this head (the diff moved only the stream, not the fields). Tenth-like fraction widths (p=[4221,42211] / [5570,55697]) measure 764-775 B/group \u2192 121,301,564-123,047,671.", + "bytes_at_full_source": "1,177,861,631 = 742.0 B/group \u00d7 1,587,376 + preamble (byte-census-measurements.json origin_budget.full_source_bytes = 1,177,832,992). ASEC-like widths (d=37601/100) give 747 B/group \u2192 1,185,798,511. Group indices measured at 7-digit widths; the true average digit count over 0..1,587,375 is 6.30 (ids 1..3,174,752: 6.65), so the true per-group cost is ~2 B lower.", + "measurement_basis": "measure_budget_document.py: an exact replica of _document's append/append_list stream (survey_origin_budget.py:625-668) over invented records built with the module's own _reference (:105-140) and _json at full-source id widths, asserted byte-identical to graph._segmented_json({**header, 'origins': records}, segment=MAX_PAYLOAD_BYTES, maximum=MAX_ROSTER_BYTES); per-group cost = (bytes at 2002 groups \u2212 bytes at 2 groups)/2000 so the fixed header cancels; header fields taken from the 1/1000 preparation.json. Record bytes via _json: 709 (JSON params), 714 (ASEC-like), 731/742 (tenth-like).", + "binds_at_tenth": true, + "binds_at_full_source": true, + "base_value": "64 MiB on the WHOLE document: the base append() was `_require(len(piece) <= MAX_PAYLOAD_BYTES - len(payload), \"TRANSPORT_LIMIT\")` into a single bytearray with no segment close, and household_ids/group_indices were single _json pieces (25.4 MB each at full source) \u2014 git diff origin/native-row-ceilings, \u2212 side of :619-635 and :661-662. At that scope: 1.76\u00d7 at 1/10, 17.55\u00d7 at full source (90,443 groups admitted, 5.7% of source).", + "head_value": "64 MiB on one piece and one accumulation (:629-632); the largest piece measured is one origin record, 742 B; each accumulation closes at 64 MiB (~18 segments at full source); the whole document is under MAX_ROSTER_BYTES.", + "reachable_from": "age", + "reach_chain": "survey_age_calibration.run_survey_age_calibration | run_survey_age_development \u2192 _run_survey_age_calibration \u2192 budgets.freeze_survey_origin_budget (:529; again with candidate at :660-666) \u2192 _document (:1005) \u2014 and budget.checked_view (:535, :855 via verify_survey_origin_budget) \u2192 _validate_budget \u2192 _document (:777). The 19/45-node runners touch survey_origin_budget only at graph_atomic_survey_financial.py:721/:1034/:1353/:1890 (_config_payload, _live()), which never reach _initial or _document.", + "classification": "segment", + "notes": "The shared encoder's own ceiling is also 64 MiB \u2014 graph_survey_population.py:272 `_require(type(limit) is int and 0 < limit <= 64 * 1024**2, \"TRANSPORT_LIMIT\")` \u2014 so MAX_PAYLOAD_BYTES cannot be raised above 64 MiB without changing _bounded_json/_canonical_pieces. It is part of the _live() producer-seal contract tuple (:198), so moving it changes the seal. Test pins: test_us_native_row_ceilings.py:153 asserts == 64 MiB and :154 divides by 764 B/group (the row-ceilings lane's tenth-like widths; this head measures 742 at full-source widths, 764-775 at tenth-like widths). The comment at :55-62 quotes 742 B/group, 1,177,832,992 B and 90,443 households \u2014 all reproduced here." + }, + { + "constant": "survey_origin_budget.py:63 MAX_ROSTER_BYTES [budget document total]", + "value": "64 * MAX_PAYLOAD_BYTES = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "survey_origin_budget.py:633 append(): `_require(total + len(piece) <= MAX_ROSTER_BYTES, \"TRANSPORT_ROSTER_LIMIT\")` \u2014 SurveyOriginBudgetError", + "survey_origin_budget.py:666-668 after `document = b\"\".join(segments)`: `_require(len(document) == total <= MAX_ROSTER_BYTES, \"TRANSPORT_ROSTER_LIMIT\")`", + "survey_origin_budget.py:975 freeze_survey_origin_budget: `_require(candidate is None or len(candidate) <= MAX_ROSTER_BYTES, \"CANDIDATE_LIMIT\")` (candidate = the same budget bytes, passed at survey_age_calibration.py:664)", + "graph_survey_budget.py:49-53 _document: `_require(type(payload) is bytes and 0 < len(payload) <= budgets.MAX_ROSTER_BYTES, \"PAYLOAD\")` \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_PAYLOAD\") (:40-42), then json.loads(payload) at :55; reached from numeric_survey_budget_payload :71, SurveySamplingBudgetKernel.__init__ :201 and .run :217" + ], + "refusal_codes": [ + "TRANSPORT_ROSTER_LIMIT", + "CANDIDATE_LIMIT", + "SURVEY_BUDGET_TRANSPORT_PAYLOAD" + ], + "encodes": "The whole sampling-origin budget document (the same bytes as the accumulation row above): one origin record per selected household plus two per-clone-row header lists, joined once from the segments.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "117,809,350 (see the MAX_PAYLOAD_BYTES accumulation row; byte-census-measurements.json 117,782,854 without preamble)", + "bytes_at_full_source": "1,177,861,631 (byte-census-measurements.json 1,177,832,992 without preamble) = 0.274 of 4 GiB", + "measurement_basis": "measure_budget_document.py (same replica stream, same difference-of-two-counts); the joined length equals the streamed total by :668.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none \u2014 the definition was added on this branch (git diff origin/native-row-ceilings: `+MAX_ROSTER_BYTES = 64 * MAX_PAYLOAD_BYTES` at :63); the base's ceiling on this document was MAX_PAYLOAD_BYTES (recorded in that row). CANDIDATE_LIMIT at :975 was `len(candidate) <= MAX_PAYLOAD_BYTES` in the base; graph_survey_budget.py:51 was `<= budgets.MAX_PAYLOAD_BYTES` in the base.", + "head_value": "4,294,967,296; full-source document is 0.274 of it (headroom 3.65\u00d7); also in the _live() contract tuple (:199)", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration \u2192 budgets.freeze_survey_origin_budget (:529, :660 with candidate=budget_payload) \u2192 _document :633/:668 and :975; \u2192 transport.numeric_survey_budget_payload (:537) and SurveySamplingBudgetKernel (:557, run inside run_graph :697) \u2192 graph_survey_budget._document :51/:55.", + "classification": "stays", + "notes": "This is the segmented transport's explicit total (comment :59-62: 'the total a process will materialise for the document is 64 accumulations, the same multiple survey_population_preparation uses'). The document is nonetheless held whole several times per run: the join at :667, the store artifact (graph_survey_budget.py:262 artifacts={'budget': payload}), json.loads at :917 (checked_view) and :1126, graph_survey_budget.py:55, and two more whole encodings compared byte-for-byte at :786 (`expected == payload`, BUDGET_RECONSTRUCTION) and :1125 \u2014 at full source each is ~1.18 GB. Test pin: test_us_native_row_ceilings.py:158-161 (== 64 \u00d7 MAX_PAYLOAD_BYTES; 764 \u00d7 STACKED_HOUSEHOLDS \u2264 it)." + }, + { + "constant": "survey_origin_budget.py:63 MAX_ROSTER_BYTES [consumer-side re-encoding of the issued budget bytes at :934-937]", + "value": "4,294,967,296 (head); the base used MAX_PAYLOAD_BYTES 67,108,864 here", + "enforced_at": [ + "survey_origin_budget.py:934-937 SamplingOriginBudget.checked_view: `_require(graph._json_matches(document, entry[1], maximum=MAX_ROSTER_BYTES), \"FINAL_BUDGET_VIEW_DOCUMENT\")` \u2192 graph_survey_population.py:356-375: each canonical token (\u2264 64 MiB via _canonical_pieces :303/:305 \"TRANSPORT_LIMIT\") is compared against a memoryview slice of the payload; :371 `_require(end <= maximum, \"TRANSPORT_ROSTER_LIMIT\")` \u2014 SurveyPopulationGraphError", + "survey_origin_budget.py:786 `_require(expected == payload, \"BUDGET_RECONSTRUCTION\")` and :1125 `_require(payload == entry[1], \"FINANCIAL_BUDGET_RECONSTRUCTION\")` \u2014 producer-side whole re-materialisations through _document (a second 1.18 GB copy at full source), under the same MAX_ROSTER_BYTES" + ], + "refusal_codes": [ + "FINAL_BUDGET_VIEW_DOCUMENT", + "TRANSPORT_ROSTER_LIMIT", + "TRANSPORT_LIMIT", + "BUDGET_RECONSTRUCTION", + "FINANCIAL_BUDGET_RECONSTRUCTION" + ], + "encodes": "The decoded budget document (`document = json.loads(entry[1])`, :917) re-encoded by its consumer to prove it is the issued bytes \u2014 the whole roster document.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "117,809,350 (same document as the rows above)", + "bytes_at_full_source": "1,177,861,631 (0.274 of MAX_ROSTER_BYTES)", + "measurement_basis": "measure_budget_document.py; _json_matches walks the same _canonical_pieces stream whose total the replica measured (graph_survey_population.py:369).", + "binds_at_tenth": true, + "binds_at_full_source": true, + "base_value": "MAX_PAYLOAD_BYTES 64 MiB, materialised: `_require(_json(document) == entry[1], \"FINAL_BUDGET_VIEW_DOCUMENT\")` (git diff \u2212 side at :930) \u2014 a second whole encoding under 64 MiB: 1.76\u00d7 at 1/10, 17.55\u00d7 at full source", + "head_value": "MAX_ROSTER_BYTES 4 GiB, streamed token-by-token with no second copy. NOTE: this hunk (:927-937) is an UNCOMMITTED working-tree modification at the head read \u2014 git status shows `M survey_origin_budget.py` and `git diff` (vs HEAD 77bafbb3a) contains only this hunk; HEAD itself still has the base form.", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration \u2192 budget.checked_view (:535), verify_survey_origin_budget (:855), and every checked_view inside admit_survey_weight_only_population (:1362, :1376) and SamplingOriginSuccessor.checked_view (:1303, :1306) \u2192 :934-937.", + "classification": "consumer", + "notes": "The consumer takes the producer's total (maximum=MAX_ROSTER_BYTES). :917 `document = json.loads(entry[1])` still materialises the decoded object tree of the whole document before the streamed comparison; _check_view_constraint (:938) then walks it again (see the digest-payload row)." + }, + { + "constant": "survey_origin_budget.py:63 MAX_ROSTER_BYTES [selection_sha256 stream at :605-607]", + "value": "4,294,967,296 (head); the base used MAX_PAYLOAD_BYTES 67,108,864 here", + "enforced_at": [ + "survey_origin_budget.py:605-607 `\"selection_sha256\": graph._json_sha256(view.receipt[\"selection\"], maximum=MAX_ROSTER_BYTES)` \u2192 graph_survey_population.py:341-353: sha256 over _canonical_pieces(value, 64 MiB); :350 `_require(total + len(encoded) <= maximum, \"TRANSPORT_ROSTER_LIMIT\")`; each token \u2264 64 MiB at :303/:305 \"TRANSPORT_LIMIT\" \u2014 SurveyPopulationGraphError" + ], + "refusal_codes": [ + "TRANSPORT_ROSTER_LIMIT", + "TRANSPORT_LIMIT" + ], + "encodes": "The preparation receipt's `selection` block: `selected` \u2014 one row per selected household (key {native_id, source, source_year, survey_year}, domain, inclusion_probability pair, original_design_weight pair, share pair, statistical_unit) \u2014 plus fixed `cells` (5), `excluded` (65 at 1/1000), fraction, seed, supplied_households. Digested only; never held whole by this module (the same rows are held whole in the preparation receipt, another family's row).", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "39,643,296 = 250.0 B/ACS row (p=[4221,42211]) \u00d7 153,161 + 241.0 B/ASEC row (p=[5570,55697]) \u00d7 5,576 + 8,040 B invented preamble; byte-census-measurements.json selection_block.tenth_bytes_SCALED = 38,594,135 (scaled from 1/1000 widths)", + "bytes_at_full_source": "385,250,455 = 243.0 B/ACS row (p=[1,1]) \u00d7 1,531,614 + 234.0 B/ASEC row \u00d7 55,762 + preamble; byte-census-measurements.json selection_block.full_source_bytes_SCALED = 385,942,815. That is 5.74\u00d7 64 MiB and 0.090 of 4 GiB.", + "measurement_basis": "measure_constraint_digest_and_selection.py: selected rows shaped from the 1/1000 receipt at full-source and tenth widths (13-char ACS native ids, 5-digit ASEC, p reduced to [1,1] at full source), encoded through graph._bounded_json as a difference of 2 vs 2002 rows; the streamed _canonical_pieces total asserted equal to the materialised length (the bytes _json_sha256 digests).", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "MAX_PAYLOAD_BYTES 64 MiB, materialised: `\"selection_sha256\": _sha(_json(view.receipt[\"selection\"]))` (git diff \u2212 side at :595) \u2014 0.59\u00d7 at 1/10, 5.74\u00d7 at full source (276,017 rows admitted per byte-census-measurements.json)", + "head_value": "MAX_ROSTER_BYTES 4 GiB, streamed digest (0.090 of it at full source)", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration \u2192 freeze_survey_origin_budget \u2192 _document :605-607 (and again in _validate_budget :777 and _financial_budget_values :1116 on every checked_view).", + "classification": "segment", + "notes": "Comment at :603-604: 'One row per supplied household: digested from the stream, never held whole. The digest equals _sha(_json(...)) wherever _json admits.' The 1/1000 receipt's selection block is 393,395 B for 1,584 rows (243.13 B/row at those widths)." + }, + { + "constant": "survey_origin_budget.py:54 MAX_PAYLOAD_BYTES [weight-only successor document via _json at :1242 and :1316]", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "survey_origin_budget.py:1242 _checked_successor_document: `payload = _json(_successor_document(state, budget_view))` \u2192 graph._bounded_json(value, MAX_PAYLOAD_BYTES) \"TRANSPORT_LIMIT\" \u2014 SurveyPopulationGraphError", + "survey_origin_budget.py:1316 SamplingOriginSuccessor.checked_view: `_require(_json(document) == entry[1], \"FINAL_SUCCESSOR_VIEW_DOCUMENT\")` \u2014 consumer-side whole re-encoding under the same 64 MiB", + "survey_origin_budget.py:1305 `_require(expected == payload, \"SUCCESSOR_RECONSTRUCTION\")`, :1307-1310 and :1377-1379 \"FINAL_SUCCESSOR_SEAL\" \u2014 further whole re-encodings compared byte-for-byte" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT", + "FINAL_SUCCESSOR_VIEW_DOCUMENT", + "SUCCESSOR_RECONSTRUCTION", + "FINAL_SUCCESSOR_SEAL" + ], + "encodes": "The weight-only successor document (:1210-1222): fixed fields (protocol, budget_sha256, previous_binding_sha256, versions, two frame digests, constraint_digest, release_eligible), `mass_ledger` (one record per node transition, fixed) and `group_total_float64_hex` \u2014 `[float(v).hex() for v in totals]` (:1219), one 23-25 B token per group, i.e. per selected household.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "3,778,465 = 23.8 B/group \u00d7 158,737 + 1,080 B fixed", + "bytes_at_full_source": "37,775,073 = 23.8 B/group \u00d7 1,587,376 + fixed = 0.563 of 64 MiB (headroom 1.78\u00d7); worst case with every token at 25 B: 39,685,480 (0.59)", + "measurement_basis": "measure_successor_and_geography.py: the :1210-1222 dict with uniform(10, 5000) calibrated-looking totals (tokens measured 23-24 B) through survey_origin_budget._json, difference of 2 vs 2002 groups.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 MiB, whole document (unchanged: no diff at :1242 or :1316)", + "head_value": "64 MiB, whole document (unchanged)", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration observe() :616 budgets.admit_survey_weight_only_population \u2192 :1375 _checked_successor_document \u2192 :1242; then :1376-1379 (second encoding); verify_survey_weight_only_successor (:856) \u2192 to_bytes \u2192 SamplingOriginSuccessor.checked_view \u2192 :1304, :1308 and :1316 (three more whole encodings).", + "classification": "stays", + "notes": "Not segmented on this branch and not binding at full source. group_bounds.check (:1199-1202) and the per-row loop over `rows[int(group)][\"design_bound_float64_hex\"]` (:1204-1209) walk Python objects of the decoded budget, not bytes. At least six whole encodings of ~38 MB happen per run through these seals." + }, + { + "constant": "survey_origin_budget.py:54 MAX_PAYLOAD_BYTES [geography-binding semantic_population via _json at :390-392]", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "survey_origin_budget.py:390-392 `\"postclone_geography_population_semantic_sha256\": _sha(_json(semantic_population))` \u2192 graph._bounded_json(value, MAX_PAYLOAD_BYTES): graph_survey_population.py:281 `_require(len(piece) <= limit - len(result), \"TRANSPORT_LIMIT\")` on the single hex-string piece \u2014 SurveyPopulationGraphError", + "survey_origin_budget.py:383-401 the enclosing `_json({...})` binding document and :386 `_sha(_json(semantic_receipt))` \u2014 fixed-shape (digests and the decoded validation receipt), same bound" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT" + ], + "encodes": "`semantic_population` (:370-382): `\"design_weights\": [(entity, str(values.dtype), values.shape, values.tobytes().hex()) for entity, values in population.design_weights.items()]` \u2014 16 hex characters per float64 design weight, one per household row of `value.population`, the reconstructed postclone geography population (module docstring :7-8: 'independently reconstructs the complete postclone geography Frame'; :324-328 bind expanded_population to the COMBINED_CLONE_CLAIM_NODE stage), i.e. the clone household roster; plus fixed frame_sha256/version/owners/weight_kind/mass_ledger.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "5,088,502 = 16.0 B/clone household \u00d7 317,475 + 8,426 B invented preamble", + "bytes_at_full_source": "50,809,220 = 16.0 B \u00d7 3,174,752 + preamble = 0.757 of 64 MiB (headroom 1.32\u00d7)", + "measurement_basis": "measure_successor_and_geography.py: the :370-382 dict with np.full(n, 137.5).tobytes().hex() through survey_origin_budget._json, difference of 2 vs 2002 households.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 MiB (unchanged; no diff in _geography_binding)", + "head_value": "64 MiB (unchanged)", + "reachable_from": "age", + "reach_chain": "Only when the runner is given geography_config: survey_age_calibration.run_survey_age_calibration|run_survey_age_development(geography_config=...) \u2192 freeze_survey_origin_budget(geography_config=...) (:533, :665) \u2192 _initial :449-456 \u2192 _geography_binding; re-encoded at :495-501 (FINAL_GEOGRAPHY_RECONSTRUCTION_SEAL) and on every _validate_budget (:768-776, :796-804). With geography_config None, :448 sets geography_binding = None and the site never runs. Not reached from the 19/45 runners (they call _config_payload only).", + "classification": "stays", + "notes": "The tightest non-binding roster document in this family (1.32\u00d7 headroom at full source), and it is a single encoder piece \u2014 one Python str of 16\u00d7N chars \u2014 so segmenting would not help; the hex string itself is the document. The budget header embeds `json.loads(geography_binding)` (:618), which carries only the sha256 of this document, so the budget document does not grow by it." + }, + { + "constant": "survey_origin_budget.py:68 MAX_GROUPS", + "value": "7_000_000", + "enforced_at": [ + "survey_origin_budget.py:421 _initial: `_require(0 < len(instructions) <= MAX_GROUPS, \"GROUP_COUNT_BOUND\")` \u2014 SurveyOriginBudgetError(ValueError)", + "graph_survey_budget.py:75-86 numeric_survey_budget_payload: `and 0 < len(records) <= budgets.MAX_GROUPS` inside \"COMPLETE_CLONE_SHAPE\" \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_COMPLETE_CLONE_SHAPE\")" + ], + "refusal_codes": [ + "GROUP_COUNT_BOUND", + "SURVEY_BUDGET_TRANSPORT_COMPLETE_CLONE_SHAPE" + ], + "encodes": "A record cap: allocation instructions = groups = selected households (graph.allocation_instructions requires one instruction per selected household \u2014 docs/us-native-row-ceilings.md \u00a73a quotes `_require(len(household_origins) == len(plan.selected), \"ORIGIN_COUNT\")`).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "n/a \u2014 a row count: 158,737 groups at 1/10 (0.023 of the cap)", + "bytes_at_full_source": "n/a \u2014 1,587,376 groups at full source (0.227 of the cap; headroom 4.41\u00d7)", + "measurement_basis": "Counts from \u00a73 of the brief (roster-census.json); no bytes behind this bound.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "7_000_000 (unchanged on this branch; the row-ceilings lane had already moved it from 1_000_000 \u2014 docs/us-native-row-ceilings.md \u00a73a and the comment at :64-67)", + "head_value": "7_000_000", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration \u2192 freeze_survey_origin_budget \u2192 _initial :418-421; and \u2192 transport.numeric_survey_budget_payload :81.", + "classification": "not-a-transport", + "notes": "Row count wearing no byte name; the row-count rule is docs/us-native-row-ceilings.md \u00a73a (4\u00d7 the catalogue's selected-household count, rounded up to the next million). In the _live() contract tuple (:200). Test pins: test_us_native_row_ceilings.py:58 and :156." + }, + { + "constant": "survey_origin_budget.py:69 MAX_SCALAR_CHARS", + "value": "4096", + "enforced_at": [ + "survey_origin_budget.py:569-572 records(): `_require(type(key.native_id) is str and len(key.native_id) <= MAX_SCALAR_CHARS, \"NATIVE_KEY_BOUND\")` per origin record \u2014 SurveyOriginBudgetError" + ], + "refusal_codes": [ + "NATIVE_KEY_BOUND" + ], + "encodes": "A single token: the raw native id of one selected household (`raw_native_id`). Observed widths in the 1/1000 receipt: 13 chars for ACS (`2024GQ0000926`, `2024HU\u2026`), 4-5 chars for ASEC.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "\u2264 13 B per token; per-token width, independent of the fraction", + "bytes_at_full_source": "\u2264 13 B per token (0.3% of the cap)", + "measurement_basis": "Widths counted from preparation.json origins.households (1,529 ACS ids of 13 chars; 48 ASEC of 5, 7 of 4).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4096 (unchanged)", + "head_value": "4096", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration \u2192 freeze_survey_origin_budget \u2192 _document \u2192 records() :569.", + "classification": "must-not-move", + "notes": "Protects a token width; in the _live() contract tuple (:201)." + }, + { + "constant": "survey_origin_budget.py:99 128 (FRACTION_BOUND bit_length)", + "value": "value.numerator.bit_length() <= 128 and value.denominator.bit_length() <= 128 (each integer < 2**128, \u2264 39 decimal digits)", + "enforced_at": [ + "survey_origin_budget.py:96-102 _pair: `_require(value.numerator.bit_length() <= 128 and value.denominator.bit_length() <= 128, \"FRACTION_BOUND\")` \u2014 SurveyOriginBudgetError; reached from _reference :107-108 (d, p, s) and :129-133 (d, p, s, b, a) for every origin record" + ], + "refusal_codes": [ + "FRACTION_BOUND", + "EXACT_FRACTION_REQUIRED" + ], + "encodes": "A fixed-width field: each exact-fraction pair `[numerator, denominator]` in an origin record (original anchor d, inclusion probability p, share s, derived b and a). Observed widths at 1/1000: \u2264 5 digits (e.g. [42, 42211], [37601, 100]); at full source p = [1, 1].", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "\u2264 5 decimal digits per integer observed (e.g. p=[4221,42211]); independent of the roster", + "bytes_at_full_source": "\u2264 5 digits per integer (p reduces to [1,1] at full source); the bound admits 39", + "measurement_basis": "Read from preparation.json selection rows and origin records; the derived b, a pairs are ratios of those.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "128 (unchanged)", + "head_value": "128", + "reachable_from": "age", + "reach_chain": "\u2192 freeze_survey_origin_budget \u2192 _document \u2192 records() \u2192 _reference :561 \u2192 _pair.", + "classification": "must-not-move", + "notes": "Protects the exact-arithmetic token width, not a transport." + }, + { + "constant": "survey_origin_budget.py:871-875 json.dumps(digest_payload, sort_keys=True, separators=(\",\", \":\"), ensure_ascii=True).encode() (cap: none)", + "value": "none \u2014 the encoder takes no limit; the payload is a whole-roster document", + "enforced_at": [ + "survey_origin_budget.py:865-875 _check_view_constraint builds `digest_payload = {\"contract\": ..., \"household_ids\": ids (tuple of every clone row), \"group_indices\": groups (every clone row), \"absolute_bounds_hex\": bounds_hex (one per group)}` and hashes json.dumps(...).encode(); :876-898 `_require(... constraint.constraint_digest == expected_digest, \"FINAL_BUDGET_VIEW_CONSTRAINT\")` \u2014 SurveyOriginBudgetError" + ], + "refusal_codes": [ + "FINAL_BUDGET_VIEW_CONSTRAINT" + ], + "encodes": "A consumer-side re-encoding of the GroupedUpperBounds digest payload that microcosm.calibrate.group_bounds produced (group_bounds.py:71-81): two lists over the clone household roster plus one hex float per group.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "7,761,306 = 48.9 B/group \u00d7 158,737 + 99 B", + "bytes_at_full_source": "77,612,467 = 48.9 B/group \u00d7 1,587,376 + 99 B (1.16\u00d7 64 MiB, for scale; nothing refuses it)", + "measurement_basis": "measure_constraint_digest_and_selection.py: the exact json.dumps call over ids at 7-digit widths, group indices and float.hex() bounds, difference of 2 vs 2002 groups; the replica's sha256 asserted equal to GroupedUpperBounds(...).constraint_digest at both sizes.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (uncapped; unchanged)", + "head_value": "none (uncapped; unchanged)", + "reachable_from": "age", + "reach_chain": "\u2192 budget.checked_view (:535, :855, and inside every admit/verify successor call :1303/:1306/:1362/:1376) \u2192 :938 _check_view_constraint.", + "classification": "stays", + "notes": "No bound to meet; recorded because the brief asks for every consumer-side re-encoding of a roster-shaped document produced by another module. Also materialised at the same call: `immutable_array(...)` at :851-863 builds `np.asarray(expected, dtype).tobytes()` for the whole ids/groups/bounds (another ~50 MB at full source) and `buckets` (:845-849) a Python list per group." + }, + { + "constant": "graph_survey_calibration.py:44 MAX_BYTES [numeric-bounds document: one accumulation]", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "graph_survey_budget.py:146-159 `output = graph._segmented_json({...}, segment=numeric.MAX_BYTES, maximum=numeric.MAX_ROSTER_BYTES)` \u2192 graph_survey_population.py:328-331: segment close, then `_require(len(current) + len(encoded) <= segment, \"TRANSPORT_LIMIT\")`; per-token bound via _canonical_pieces :294/:303/:305 \"TRANSPORT_LIMIT\" \u2014 SurveyPopulationGraphError(ValueError)" + ], + "refusal_codes": [ + "TRANSPORT_LIMIT" + ], + "encodes": "The numeric-bounds document (BOUNDS_PROTOCOL, graph_survey_budget.py:147-156): four lists with one entry per clone household row (household_ids, group_indices, row_upper_hex, incoming_hex), one list with one entry per group (group_upper_hex), plus protocol, budget_sha256, population \u2014 151.97 B per group (75.99 B per clone row).", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "24,123,542 = 151.97 B/group \u00d7 158,737 + 281 B (byte-census-measurements.json numeric_bounds.tenth_bytes = 24,123,261 without the fixed part)", + "bytes_at_full_source": "241,233,811 = 151.97 \u00d7 1,587,376 + 281 (byte-census-measurements.json full_source_bytes = 241,233,530); 3.59\u00d7 64 MiB; 0.056 of MAX_ROSTER_BYTES", + "measurement_basis": "measure_numeric_bounds.py: the :147-156 document shape at full-source id widths through graph._segmented_json(segment=MAX_BYTES, maximum=MAX_ROSTER_BYTES) at this head, asserted byte-identical to graph._bounded_json(doc, MAX_BYTES); bytes at 2 and 2002 groups = 585 and 304,525, exactly the figures in byte-census-measurements.json (so that file's numbers are for this head's encoder); per group = (304,525 \u2212 585)/2000.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "64 MiB on the WHOLE document: graph_survey_budget.py \u2212 side `graph._bounded_json({...}, numeric.MAX_BYTES)` and `_require(len(output) <= numeric.MAX_BYTES, \"NUMERIC_LIMIT\")`; graph_survey_calibration.py:113 \u2212 side `0 < len(payload) <= MAX_BYTES` \"PAYLOAD\". At that scope: 0.36\u00d7 at 1/10, 3.59\u00d7 at full source (883,185 clone rows admitted).", + "head_value": "64 MiB on one segment only (graph_survey_budget.py:157 `segment=numeric.MAX_BYTES`); the total is under MAX_ROSTER_BYTES (:158, :160)", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration :537 transport.numeric_survey_budget_payload(budget_payload) \u2192 :146-159; and SurveySamplingBudgetKernel.run :235 inside run_graph (:697).", + "classification": "segment", + "notes": "Comment at :45-51 states 76 B/row and 241,233,530 B \u2014 reproduced. graph_survey_budget.py:161 immediately re-decodes the fresh output (`numeric.decode_numeric_survey_bounds(output)`), so the consumer re-encoding row below runs on every production." + }, + { + "constant": "graph_survey_calibration.py:52 MAX_ROSTER_BYTES [numeric-bounds document total and consumer PAYLOAD]", + "value": "64 * MAX_BYTES = 4,294,967,296", + "enforced_at": [ + "graph_survey_budget.py:158 `maximum=numeric.MAX_ROSTER_BYTES` \u2192 graph_survey_population.py:332 and :337 \"TRANSPORT_ROSTER_LIMIT\" \u2014 SurveyPopulationGraphError", + "graph_survey_budget.py:160 `_require(len(output) <= numeric.MAX_ROSTER_BYTES, \"NUMERIC_LIMIT\")` \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_NUMERIC_LIMIT\")", + "graph_survey_calibration.py:113 decode_numeric_survey_bounds: `_require(type(payload) is bytes and 0 < len(payload) <= MAX_ROSTER_BYTES, \"PAYLOAD\")` \u2192 ValueError(\"SURVEY_CALIBRATION_PAYLOAD\"), then json.loads at :115 \u2014 reached at graph_survey_budget.py:161 and :236, survey_age_calibration.py:538 and :830, graph_survey_calibration.py:357 and :417 (kernel run), survey_calibration_diagnostics.py:109" + ], + "refusal_codes": [ + "TRANSPORT_ROSTER_LIMIT", + "SURVEY_BUDGET_TRANSPORT_NUMERIC_LIMIT", + "SURVEY_CALIBRATION_PAYLOAD" + ], + "encodes": "The whole numeric-bounds document (same bytes as the accumulation row), joined once and decoded whole by every consumer.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "24,123,542 (see the MAX_BYTES row)", + "bytes_at_full_source": "241,233,811 = 0.056 of 4 GiB (headroom 17.8\u00d7)", + "measurement_basis": "measure_numeric_bounds.py (same document; the join length equals the streamed total by graph_survey_population.py:337).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none \u2014 added on this branch (git diff: `+MAX_ROSTER_BYTES = 64 * MAX_BYTES` at :52); the base's ceiling on this document was MAX_BYTES (that row)", + "head_value": "4,294,967,296", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration :537 \u2192 graph_survey_budget :158/:160; :538/:830 and the kernel/diagnostics decodes \u2192 graph_survey_calibration :113.", + "classification": "stays", + "notes": "MAX_ROWS = MAX_ROSTER_BYTES // 128 (:53) derives from it (its own row). The document is decoded whole at least seven times per run (the decode sites listed), each holding the 241 MB bytes plus the json.loads object tree plus the canonical_json re-encoding (next row)." + }, + { + "constant": "graph_survey_calibration.py:53 MAX_ROWS", + "value": "MAX_ROSTER_BYTES // 128 = 33,554,432 (base: MAX_BYTES // 128 = 524,288)", + "enforced_at": [ + "graph_survey_calibration.py:127 `_require(type(ids) is list and 0 < len(ids) <= MAX_ROWS, \"ROW_COUNT\")` \u2192 ValueError(\"SURVEY_CALIBRATION_ROW_COUNT\")", + "graph_survey_budget.py:75-86 `and 0 < len(ids) <= numeric.MAX_ROWS` inside \"COMPLETE_CLONE_SHAPE\" \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_COMPLETE_CLONE_SHAPE\")" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_ROW_COUNT", + "SURVEY_BUDGET_TRANSPORT_COMPLETE_CLONE_SHAPE" + ], + "encodes": "A record cap on `household_ids` = clone household rows of the numeric-bounds document.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "n/a \u2014 a row count: 317,475 clone rows at 1/10 (0.61 of the base 524,288; 0.0095 of the head value)", + "bytes_at_full_source": "n/a \u2014 3,174,752 clone rows at full source (6.06\u00d7 the base 524,288; 0.095 of the head 33,554,432)", + "measurement_basis": "Counts from \u00a73 of the brief; ratios in measure_numeric_bounds.receipt.json.", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "524,288 (MAX_BYTES // 128; git diff \u2212 side `MAX_ROWS = MAX_BYTES // 128`) \u2014 binds at full source (6.06\u00d7), not at 1/10", + "head_value": "33,554,432 (MAX_ROSTER_BYTES // 128) \u2014 binds at neither", + "reachable_from": "age", + "reach_chain": "\u2192 transport.numeric_survey_budget_payload :77 and every decode_numeric_survey_bounds :127 (survey_age_calibration.py:538/:830, kernel :357/:417, diagnostics :109, graph_survey_budget :161/:236).", + "classification": "not-a-transport", + "notes": "A row count in byte clothing: comment at :48-51 \u2014 'MAX_ROWS is the same //128 form the allocation pre-check uses (graph_survey_population.py); at 524,288 it refused a full-source clone of 3,174,752 rows before any byte did.' The row-count rule is the //128 structural pre-check of docs/us-native-scale-transport.md \u00a72d (ALLOCATION_ROSTER_BYTES // 128)." + }, + { + "constant": "graph_survey_calibration.py:119 canonical_json(document) == payload (cap: none in canonical_json; bounded upstream by :113 MAX_ROSTER_BYTES)", + "value": "none \u2014 microcosm.graph.canonical.canonical_json (canonical.py:43-58) is one json.dumps(...).encode() with no limit argument", + "enforced_at": [ + "graph_survey_calibration.py:119 decode_numeric_survey_bounds: `_require(canonical_json(document) == payload, \"CANONICAL\")` \u2192 ValueError(\"SURVEY_CALIBRATION_CANONICAL\"); the payload it compares against is \u2264 MAX_ROSTER_BYTES by :113" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_CANONICAL" + ], + "encodes": "A consumer-side whole re-encoding of the numeric-bounds document that graph_survey_budget produced in segments \u2014 a second whole copy of the roster document alongside the payload and its json.loads tree.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "24,123,542 (same bytes; canonical_json(json.loads(payload)) == payload verified for the invented 2002-group document)", + "bytes_at_full_source": "241,233,811 (a second copy at every decode; seven decodes per run)", + "measurement_basis": "measure_numeric_bounds.py: decode_numeric_survey_bounds run on the invented document (exercising :113, :119, :127, :143-146) and `canonical_json(json.loads(dec)) == dec` asserted.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (uncapped re-encoding; unchanged) \u2014 in the base the payload was capped at MAX_BYTES by :113, so the re-encoding was \u2264 64 MiB by construction", + "head_value": "none (uncapped); the payload cap is now MAX_ROSTER_BYTES 4 GiB, so this re-encoding may be up to 4 GiB", + "reachable_from": "age", + "reach_chain": "Every decode site: survey_age_calibration.py:538, :830; graph_survey_budget.py:161, :236; SurveyAgeCalibrationKernel.run graph_survey_calibration.py:357, :417; survey_calibration_diagnostics.py:109.", + "classification": "consumer", + "notes": "Takes the producer's total (MAX_ROSTER_BYTES). The analogous budget-document check at survey_origin_budget.py:934-937 was converted to a streamed _json_matches on this branch; this one was not." + }, + { + "constant": "graph_survey_calibration.py:88 32 (FLOAT_TOKEN)", + "value": "0 < len(v) <= 32 per hex-float token", + "enforced_at": [ + "graph_survey_calibration.py:86-98 _vector: `_require(all(type(v) is str and 0 < len(v) <= 32 for v in raw), \"FLOAT_TOKEN\")` \u2192 ValueError(\"SURVEY_CALIBRATION_FLOAT_TOKEN\"), then float.fromhex and the round-trip check :93-97 \"FLOAT_VALUES\"; called at :140-142 for group_upper_hex, incoming_hex, row_upper_hex and at graph_survey_budget.py:98-99 for each record's design_bound_float64_hex / upper_float64_hex" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_FLOAT_TOKEN", + "SURVEY_CALIBRATION_VECTOR_LENGTH", + "SURVEY_CALIBRATION_FLOAT_VALUES" + ], + "encodes": "A fixed-width token: one float64 as float.hex() (measured max 21 chars at these magnitudes; \u2264 24 for any finite float64).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "\u2264 24 B per token; independent of the roster", + "bytes_at_full_source": "\u2264 24 B per token", + "measurement_basis": "measure_numeric_bounds.receipt.json float_token_widths (max 21).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "32 (unchanged)", + "head_value": "32", + "reachable_from": "age", + "reach_chain": "\u2192 decode_numeric_survey_bounds :140-142 and numeric_survey_budget_payload :98-99.", + "classification": "must-not-move", + "notes": "Protects the encoder width of a float token." + }, + { + "constant": "graph_survey_calibration.py:123 256 (POPULATION)", + "value": "0 < len(document[\"population\"]) <= 256", + "enforced_at": [ + "graph_survey_calibration.py:122-125 `_require(type(document[\"population\"]) is str and 0 < len(document[\"population\"]) <= 256, \"POPULATION\")` \u2192 ValueError(\"SURVEY_CALIBRATION_POPULATION\")" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_POPULATION" + ], + "encodes": "A single scalar: the population version string (clone.COMBINED_CLONE_NODE, `combined_survey_puf_support_clone`, 33 chars).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "33 B at every fraction", + "bytes_at_full_source": "33 B", + "measurement_basis": "len(clone.COMBINED_CLONE_NODE) in measure_numeric_bounds.py's document.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "256 (unchanged)", + "head_value": "256", + "reachable_from": "age", + "reach_chain": "\u2192 decode_numeric_survey_bounds :123.", + "classification": "must-not-move", + "notes": "Token width." + }, + { + "constant": "graph_survey_calibration.py:80-82 64 (_digest)", + "value": "len(value) == 64 hex characters", + "enforced_at": [ + "graph_survey_calibration.py:78-83 _digest, used at :121 `_require(_digest(document[\"budget_sha256\"]), \"BUDGET_DIGEST\")`, graph_survey_budget.py:65 \"SOURCE_EDGE_DIGEST\" (preparation_sha256, allocation_sha256) and :166 \"BUDGET_DIGEST\"" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_BUDGET_DIGEST", + "SURVEY_BUDGET_TRANSPORT_SOURCE_EDGE_DIGEST", + "SURVEY_BUDGET_TRANSPORT_BUDGET_DIGEST" + ], + "encodes": "A single token: a sha256 hex digest.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 B at every fraction", + "bytes_at_full_source": "64 B", + "measurement_basis": "Fixed by sha256.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 (unchanged)", + "head_value": "64", + "reachable_from": "age", + "reach_chain": "\u2192 decode_numeric_survey_bounds :121; graph_survey_budget._document :64-65 and survey_sampling_budget_node :166.", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "graph_survey_calibration.py:274 65536 (ACTIVATION_TEXT)", + "value": "0 < len(value.encode()) <= 65536", + "enforced_at": [ + "graph_survey_calibration.py:273-274 calibration_activation_binding: `_require(type(value) is str and 0 < len(value.encode()) <= 65536, \"ACTIVATION_TEXT\")` \u2192 ValueError(\"SURVEY_CALIBRATION_ACTIVATION_TEXT\"); :277-280 `encoded = canonical_json(binding).decode()` / `_require(encoded == value, \"ACTIVATION_CANONICAL\")` (fixed-shape re-encoding equality)" + ], + "refusal_codes": [ + "SURVEY_CALIBRATION_ACTIVATION_TEXT", + "SURVEY_CALIBRATION_ACTIVATION_JSON", + "SURVEY_CALIBRATION_ACTIVATION_CANONICAL" + ], + "encodes": "A fixed-shape receipt: the survey age activation binding carried in node params \u2014 profile, the two fixed S0101 requests, two digests, 18 bands, convention text (survey_age_activation.py:47-75).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "3,766 B (canonical_json(activation_binding(SurveyAgeActivation('a'*64,'b'*64)))) at every fraction \u2014 5.7% of the cap", + "bytes_at_full_source": "3,766 B", + "measurement_basis": "Encoded through survey_age_activation.activation_binding and canonical_json in the final check step (see transcript); placeholder digests are the same width as real ones.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65536 (unchanged)", + "head_value": "65536", + "reachable_from": "age", + "reach_chain": "survey_age_calibration.run_survey_age_development \u2192 _run_survey_age_calibration :412 numerical.calibration_activation_binding(calibration.params); SurveyAgeCalibrationKernel.run :335-339. run_survey_age_calibration (invented registry) has no 'activation' param, so :274 is skipped (:271-272).", + "classification": "must-not-move", + "notes": "Does not move with the fraction; 18 bands are asserted by national_age_activation.py:309." + }, + { + "constant": "graph_survey_budget.py:132 16 (WEIGHT_BYTES)", + "value": "len(weight_bytes) == 16 hex characters (one float64)", + "enforced_at": [ + "graph_survey_budget.py:130-135 `_require(type(weight_bytes) is str and len(weight_bytes) == 16 and all(c in \"0123456789abcdef\" for c in weight_bytes), \"WEIGHT_BYTES\")` \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_WEIGHT_BYTES\"), then np.frombuffer(bytes.fromhex(...)) :136" + ], + "refusal_codes": [ + "SURVEY_BUDGET_TRANSPORT_WEIGHT_BYTES" + ], + "encodes": "A fixed-width field: each `incoming_clone_float64_bytes` entry of an origin record (survey_origin_budget.py:588-591 `clone_weights[...].tobytes().hex()`), two per group.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "16 B per token at every fraction", + "bytes_at_full_source": "16 B per token", + "measurement_basis": "Fixed by float64.tobytes().hex().", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "16 (unchanged)", + "head_value": "16", + "reachable_from": "age", + "reach_chain": "\u2192 transport.numeric_survey_budget_payload :112-138.", + "classification": "must-not-move", + "notes": "Encoder width; the producer's `original_design_float64_bytes` (survey_origin_budget.py:581-585) has the same 16-char shape but is not checked here." + }, + { + "constant": "graph_survey_budget.py:88 2**63 (HOUSEHOLD_IDS int64 range)", + "value": "-(2**63) <= i < 2**63 for every household id", + "enforced_at": [ + "graph_survey_budget.py:87-91 `_require(all(type(i) is int and -(2**63) <= i < 2**63 for i in ids) and len(set(ids)) == len(ids), \"HOUSEHOLD_IDS\")` \u2192 ValueError(\"SURVEY_BUDGET_TRANSPORT_HOUSEHOLD_IDS\")", + "graph_survey_calibration.py:128-132 the same range plus strict ascending order, \"ORDERED_IDS\" \u2192 ValueError(\"SURVEY_CALIBRATION_ORDERED_IDS\")", + "graph_national_age_counts.py:340-363 _ids: 'must fit signed int64 exactly' (ValueError), and graph_survey_age_artifact.py:79-84 dtype ' 256 (POPULATION)", + "value": "0 < len(header[\"population\"]) <= 256", + "enforced_at": [ + "graph_survey_age_artifact.py:72-75 _validate: `_require(type(header[\"population\"]) is str and 0 < len(header[\"population\"]) <= 256, \"POPULATION\")` \u2192 ValueError(\"SURVEY_AGE_ARTIFACT_POPULATION\")" + ], + "refusal_codes": [ + "SURVEY_AGE_ARTIFACT_POPULATION" + ], + "encodes": "A single scalar: the population version string in the header (33 chars).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "33 B at every fraction", + "bytes_at_full_source": "33 B", + "measurement_basis": "len(clone.COMBINED_CLONE_NODE).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "256 (unchanged)", + "head_value": "256", + "reachable_from": "age", + "reach_chain": "\u2192 _validate from the kernel :207 and decode :128.", + "classification": "must-not-move", + "notes": "Token width." + }, + { + "constant": "survey_financial_successor.py:26 MAX_PAYLOAD_BYTES", + "value": "256 * 1024 = 262,144", + "enforced_at": [ + "survey_financial_successor.py:211-215 admit_survey_financial_population: `_require(candidate is None or (type(candidate) is bytes and len(candidate) <= MAX_PAYLOAD_BYTES), \"CANDIDATE_TYPE_OR_BOUND\")` \u2014 SurveyFinancialSuccessorError(ValueError) (:30-36)", + "survey_financial_successor.py:232 `_require(len(payload) <= MAX_PAYLOAD_BYTES, \"PAYLOAD_BOUND\")` after `payload = _document(...)`", + "survey_financial_successor.py:176 checked_view: `_require(_document(state, budget_view, run_view) == entry[1], \"RECONSTRUCTION\")` \u2014 whole re-encoding equality (fixed shape)", + "the encoder: survey_financial_successor.py:116 `budget._json({...})` = graph._bounded_json(value, survey_origin_budget.MAX_PAYLOAD_BYTES 64 MiB)" + ], + "refusal_codes": [ + "SURVEY_FINANCIAL_SUCCESSOR_CANDIDATE_TYPE_OR_BOUND", + "SURVEY_FINANCIAL_SUCCESSOR_PAYLOAD_BOUND", + "SURVEY_FINANCIAL_SUCCESSOR_RECONSTRUCTION" + ], + "encodes": "A fixed-shape receipt (:116-131): protocol, budget_sha256, financial_run_sha256, the nested `financial_run` document (`json.loads(state.run_entry[1])`, the checked financial run's receipt from graph_atomic_survey_financial \u2014 outside this family), previous/current version strings, two frame digests, owned_columns (runner.values.OUTPUTS), three booleans. No per-household member.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "No measured number in the permitted inputs; the document is O(graph nodes \u00d7 frame columns) through the nested run receipt and does not move with the fraction", + "bytes_at_full_source": "same as at 1/10 (fixed shape)", + "measurement_basis": "Read :116-131; no per-row field. Not measured (needs a live financial run).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "262,144 (unchanged; no diff)", + "head_value": "262,144", + "reachable_from": "none", + "reach_chain": "No caller of admit_survey_financial_population or verify_survey_financial_successor exists outside packages/microcosm-build/tests (grep of packages/ and tools/). survey_origin_budget imports the module only for the _live() contract (:251-256: PROTOCOL, FINANCIAL_SUCCESSOR_TYPE, MAX_PAYLOAD_BYTES) and accepts a SamplingOriginFinancialSuccessor as an optional previous_binding (:1143-1165), which survey_age_calibration never passes (:616-620). graph_atomic_survey_financial and graph_survey_completion_host do not import it. The function that would reach it: survey_financial_successor.admit_survey_financial_population.", + "classification": "must-not-move", + "notes": "The nested `financial_run` at :121 is bounded by the runner's own cap on that receipt, not here. Listed in survey_origin_budget's _live() contract, so moving it changes the budget producer seal." + }, + { + "constant": "survey_calibration_diagnostics.py:24 MAX_DIAGNOSTIC_BYTES", + "value": "8 * 1024**2 = 8,388,608", + "enforced_at": [ + "survey_calibration_diagnostics.py:83-86 validate_survey_calibration_diagnostics: `_require(type(payload) is bytes and 0 < len(payload) <= MAX_DIAGNOSTIC_BYTES, \"PAYLOAD_LIMIT\")` \u2192 ValueError(\"SURVEY_DIAGNOSTICS_PAYLOAD_LIMIT\") (:33-35)", + "survey_age_calibration.py:141-145 _load_diagnostics: `limit = diagnostic_check.MAX_DIAGNOSTIC_BYTES`; `_require(stat.S_ISREG(before.st_mode) and 0 < before.st_size <= limit, \"DIAGNOSTIC_STORE_LIMIT\")` \u2192 ValueError(\"SURVEY_AGE_RUN_DIAGNOSTIC_STORE_LIMIT\")", + "survey_age_calibration.py:153 `payload = stream.read(limit + 1)` (read cap) and :160-166 `len(payload) == record[\"size\"] <= limit and _sha(payload) == record[\"sha256\"]` \"DIAGNOSTIC_STORE_CHANGED\"", + "consumer re-encodings (canonical_json, no cap): survey_calibration_diagnostics.py:92 `_require(canonical_json(document) == payload, \"CANONICAL\")`, :194 `_require(canonical_json(expected) == payload, \"RECOMPUTED_VALUES\")`, survey_age_calibration.py:933-936 `canonical_json(result.diagnostics) == diagnostic_identity` \"FINAL_DIAGNOSTICS_VALUES\"" + ], + "refusal_codes": [ + "SURVEY_DIAGNOSTICS_PAYLOAD_LIMIT", + "SURVEY_AGE_RUN_DIAGNOSTIC_STORE_LIMIT", + "SURVEY_AGE_RUN_DIAGNOSTIC_STORE_CHANGED", + "SURVEY_DIAGNOSTICS_CANONICAL", + "SURVEY_DIAGNOSTICS_RECOMPUTED_VALUES", + "SURVEY_AGE_RUN_FINAL_DIAGNOSTICS_VALUES" + ], + "encodes": "The schema-8 calibration diagnostics artifact (microcosm.calibrate.diagnostics.diagnostics_payload :343-431): one row per target (18 age bands), `loss_trajectory` (one float per epoch; epochs \u2264 1000 by graph_survey_calibration.py:218), options (grouped_upper_bounds diagnostics = four scalars, group_bounds.py:178-184; `target_records: None` :163), target_surface and past_cap_census (scalars; `n_records` is a scalar :409 / diagnostics.py:190), skipped, build anchors. No per-household row.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "No diagnostics artifact among the permitted inputs; grows with targets (18) and epochs (\u2264 1000 floats), not with the roster", + "bytes_at_full_source": "same shape as at 1/10", + "measurement_basis": "Read diagnostics_payload :343-431 and survey_calibration_diagnostics :138-187; not measured.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "8,388,608 (unchanged)", + "head_value": "8,388,608", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._run_survey_age_calibration :760 _load_diagnostics \u2192 :785 diagnostic_check.validate_survey_calibration_diagnostics \u2192 :83-86, :92, :194; :933-936.", + "classification": "must-not-move", + "notes": "The producer graph_survey_calibration.py:402-404 `payload = canonical_json(diagnostics_payload(...))` has no cap; the 8 MiB is enforced only on the consumer side. `_history` :50 `len(trajectory) == epochs` and graph_survey_calibration.py:218 `1 <= epochs <= 1000` cap the trajectory record count \u2014 a solver parameter, not recorded as a byte row." + }, + { + "constant": "demographic_calibration_graph.py:146 1_048_576 (registry declaration)", + "value": "len(text.encode()) > 1_048_576 refuses (1 MiB)", + "enforced_at": [ + "demographic_calibration_graph.py:145-147 _registry_from_json: `if not isinstance(text, str) or len(text.encode()) > 1_048_576: raise ValueError(\"A bounded canonical registry declaration is required.\")`; :160-163 `if text != _registry_json(registry): raise ValueError(\"Registry declaration must have its canonical typed representation.\")` (whole re-encoding equality, encoder _json :76-79 has no cap); reached from graph_survey_calibration.py:169 (survey_age_calibration_node), :332 (SurveyAgeCalibrationKernel.run), survey_age_calibration.py:784, survey_age_activation.py:101 (validate_survey_age_registry), demographic_calibration_graph.py:220" + ], + "refusal_codes": [ + "ValueError(\"A bounded canonical registry declaration is required.\")", + "ValueError(\"Registry declaration must have its canonical typed representation.\")" + ], + "encodes": "A fixed-shape declaration carried in node params: `{\"country\": ..., \"specs\": [asdict(spec) ...]}` for the 18 demographic targets (one per band, national_age_activation.py:309), each with metadata and a hierarchy dict.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "Not measured (needs a TargetRegistry; no I/O-free factory found in national_age_activation); 18 specs at every fraction", + "bytes_at_full_source": "same as at 1/10", + "measurement_basis": "Read :82-85 and :145-165; _validate_registry :88-142 fixes the spec shape.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1_048_576 (unchanged)", + "head_value": "1_048_576", + "reachable_from": "age", + "reach_chain": "survey_age_calibration.run_survey_age_calibration \u2192 numerical.survey_age_calibration_node :169 \u2192 demographic._registry_from_json; kernel run :332; :784.", + "classification": "must-not-move", + "notes": "Does not move with the fraction. DemographicCalibrationKernel.run (:278-351, with its uncapped `_json(diagnostics_payload(...))` at :330) is the national kernel and is not reached by survey_age_calibration (only its implementation_hash is, graph_survey_calibration.py:324)." + }, + { + "constant": "graph_national_age_counts.py:137 65_536 (age-band declaration)", + "value": "len(text.encode()) > 65_536 refuses", + "enforced_at": [ + "graph_national_age_counts.py:137-138 _bands_from_param: `if not isinstance(text, str) or len(text.encode()) > 65_536: raise ValueError(\"A bounded canonical age-band declaration is required.\")`; :165-166 canonical re-encoding equality `if _canonical([[c, lo, hi] for c, lo, hi in bands]) != text: raise ValueError(...)` (encoder _canonical :121-122 has no cap); reached via _node_from_schema :173 (graph_survey_age_artifact.survey_age_count_artifact_node :139 \u2192 national_age_count_node :209-211) and NationalAgeCountKernel.run :253 (called from SurveyAgeCountArtifactKernel.run graph_survey_age_artifact.py:181)" + ], + "refusal_codes": [ + "ValueError(\"A bounded canonical age-band declaration is required.\")", + "ValueError(\"The age-band declaration must be in canonical form.\")" + ], + "encodes": "A fixed-shape declaration: the canonical band schema `[[column, low, high], ...]` for the 18 activated bands.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "483 B (`_bands_param(NATIONAL_AGE_ACTIVATION.bands)`) at every fraction \u2014 0.74% of the cap", + "bytes_at_full_source": "483 B", + "measurement_basis": "Encoded through graph_national_age_counts._bands_param in the final check step.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65_536 (unchanged)", + "head_value": "65_536", + "reachable_from": "age", + "reach_chain": "survey_age_calibration._measurement :102 ages.survey_age_count_artifact_node \u2192 :139 \u2192 _node_from_schema :173; kernel run :253.", + "classification": "must-not-move", + "notes": "MAX_SUPPORTED_AGE = 120 (:95) is a value bound on ages, not a byte bound, and is not recorded as a row. The count body `np.bincount(..., minlength=len(bands) * len(index))` (:302-306) allocates 18 \u00d7 households \u00d7 8 B (457 MB at full-source clone households) with no ceiling \u2014 a memory allocation with no document behind it, not a bound." + }, + { + "constant": "survey_age_sources.py:19 MAX_RESPONSE_BYTES", + "value": "8 * 1024**2 = 8,388,608", + "enforced_at": [ + "survey_age_sources.py:128 capture_survey_age_response: `_require(type(payload) is bytes and 0 < len(payload) <= MAX_RESPONSE_BYTES, \"SIZE\")` \u2192 ValueError(\"SURVEY_AGE_SOURCE_SIZE\") (:36-38)", + "survey_age_sources.py:89-109 _read(path, limit): :98 `_require(0 < before.st_size <= limit, \"SIZE\")`, :99 `payload = stream.read(limit + 1)` (read cap), :101-106 \"CHANGED_FILE\" \u2014 with limit = MAX_RESPONSE_BYTES at :144 (_write \u2192 _read, \"IMMUTABLE\" :115) and :175 (the raw response read)", + "survey_age_sources.py:170-174 read_survey_age_response: `_require(type(descriptor[\"size_bytes\"]) is int and 0 < descriptor[\"size_bytes\"] <= MAX_RESPONSE_BYTES, \"SIZE\")`; :176-180 \"RESPONSE_PIN\"" + ], + "refusal_codes": [ + "SURVEY_AGE_SOURCE_SIZE", + "SURVEY_AGE_SOURCE_CHANGED_FILE", + "SURVEY_AGE_SOURCE_IMMUTABLE", + "SURVEY_AGE_SOURCE_RESPONSE_PIN", + "SURVEY_AGE_SOURCE_FILE" + ], + "encodes": "An upstream file: the captured Census API S0101 national response (`for=us:*`, one geography row of group(S0101); metadata and data kinds, :41-57) \u2014 identical at every survey fraction.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "No captured response among the permitted inputs; the request is one national row, so its size does not depend on the survey selection", + "bytes_at_full_source": "same file", + "measurement_basis": "Read :41-57 (the fixed request surface) and :149-181.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "8,388,608 (unchanged)", + "head_value": "8,388,608", + "reachable_from": "age", + "reach_chain": "survey_age_calibration.run_survey_age_development \u2192 age_activation.activate_survey_age_targets (survey_age_activation.py:117-125: read_survey_age_response \u00d72) and :860 age_activation.verify_survey_age_targets (survey_age_activation.py:209-219). run_survey_age_calibration (invented registry) never reaches it. capture_survey_age_response is capture-time only (no runner caller).", + "classification": "must-not-move", + "notes": "Protects an upstream file's real size. strict_json (cd_reference_sources.py:51-66) applied at :129/:158/:181 has no size bound of its own." + }, + { + "constant": "survey_age_sources.py:20 MAX_DESCRIPTOR_BYTES", + "value": "4096", + "enforced_at": [ + "survey_age_sources.py:143 `_require(_read(descriptor_path, MAX_DESCRIPTOR_BYTES) == encoded, \"IMMUTABLE\")`", + "survey_age_sources.py:145 `_write(descriptor_path, encoded, MAX_DESCRIPTOR_BYTES)` (\u2192 _read :98-99 SIZE / read(limit + 1))", + "survey_age_sources.py:158-159 `descriptor = strict_json(_read(root / \"requests\" / f\"{key}.json\", MAX_DESCRIPTOR_BYTES))`" + ], + "refusal_codes": [ + "SURVEY_AGE_SOURCE_SIZE", + "SURVEY_AGE_SOURCE_IMMUTABLE", + "SURVEY_AGE_SOURCE_DESCRIPTOR" + ], + "encodes": "A fixed-shape descriptor: the nine _DESCRIPTOR_FIELDS (table, year, dataset, geography, kind, url, sha256, size_bytes, path), canonical_json.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "a few hundred bytes (nine short fields; not measured) at every fraction", + "bytes_at_full_source": "same", + "measurement_basis": "Read :21-33 and :132-146.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "4096 (unchanged)", + "head_value": "4096", + "reachable_from": "age", + "reach_chain": "As for MAX_RESPONSE_BYTES (read_survey_age_response :158-159 via activate_survey_age_targets / verify_survey_age_targets).", + "classification": "must-not-move", + "notes": "Fixed-shape." + }, + { + "constant": "survey_age_sources.py:70-77 64 (_digest PIN)", + "value": "len(value) == 64 hex characters", + "enforced_at": [ + "survey_age_sources.py:70-77 _digest: `_require(type(value) is str and len(value) == 64 and all(c in \"0123456789abcdef\" for c in value), \"PIN\")`; used at :154 and by survey_age_activation.py:19/:43-44 (SurveyAgeActivation.__post_init__)" + ], + "refusal_codes": [ + "SURVEY_AGE_SOURCE_PIN" + ], + "encodes": "A single token: a sha256 content pin.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "64 B at every fraction", + "bytes_at_full_source": "64 B", + "measurement_basis": "Fixed by sha256.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "64 (unchanged)", + "head_value": "64", + "reachable_from": "age", + "reach_chain": "run_survey_age_development \u2192 activation_binding \u2192 SurveyAgeActivation.__post_init__ \u2192 _digest; read_survey_age_response :154.", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "cd_reference_sources.py:15 MAX_RESPONSE_BYTES", + "value": "32 * 1024 * 1024 = 33,554,432", + "enforced_at": [ + "cd_reference_sources.py:86 fetch_census: `payload = response.read(MAX_RESPONSE_BYTES + 1)` (read cap)", + "cd_reference_sources.py:91-92 `if len(payload) > MAX_RESPONSE_BYTES: raise ValueError(\"Census response exceeds size limit\")`" + ], + "refusal_codes": [ + "ValueError(\"Census response exceeds size limit\")" + ], + "encodes": "An upstream file: a Census API response for one of the eight declared requests (four tables \u00d7 metadata/data at congressional-district level, :18-48).", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "No captured response among the permitted inputs; independent of the survey fraction", + "bytes_at_full_source": "same file", + "measurement_basis": "Read :18-48 and :74-93.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "33,554,432 (unchanged)", + "head_value": "33,554,432", + "reachable_from": "none", + "reach_chain": "fetch_census is reached only through capture_request (:166), called from cd_reference.py:458 (a CLI capture tool); none of the three runners reaches it. The age lane imports only strict_json (:51-66) from this module (survey_age_sources.py:17; national_age_activation.py:58), which has no size bound. The function that would reach it: cd_reference_sources.capture_request / fetch_census.", + "classification": "must-not-move", + "notes": "The cached-replay branch of capture_request (:145-163) reads the raw file with `(root / expected_path).read_bytes()` (:156) and checks `len(payload) != descriptor[\"size_bytes\"]` \u2014 no cap on that read. write_json (:115-128) has no cap (fixed descriptor)." + }, + { + "constant": "group_bounds.py:77-81 json.dumps(payload, sort_keys=True, separators=(\",\", \":\"), ensure_ascii=True).encode() (cap: none)", + "value": "none \u2014 no limit argument; the digest payload is a whole-roster document", + "enforced_at": [ + "group_bounds.py:71-81 GroupedUpperBounds.__post_init__: `payload = {\"contract\": ..., \"household_ids\": ids, \"group_indices\": groups.tolist(), \"absolute_bounds_hex\": [float(x).hex() for x in bounds]}`; `digest = hashlib.sha256(json.dumps(...).encode()).hexdigest()` \u2014 materialised whole on every construction: survey_origin_budget.py:669 (every _document), :918-922 (every checked_view), graph_survey_calibration.py:143 (every decode); graph_fiscal_dense_calibration.py:140 is off-path" + ], + "refusal_codes": [ + "none at this site (ValueError from the shape checks at :41-56 only)" + ], + "encodes": "The producer-side encoding of the constraint digest payload: every clone household id (tuple), every group index, one hex float per group. survey_origin_budget.py:865-875 re-encodes the same payload as a consumer (its own row).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "7,761,306 = 48.9 B/group \u00d7 158,737 + 99 B", + "bytes_at_full_source": "77,612,467 = 48.9 B/group \u00d7 1,587,376 + 99 B (1.16\u00d7 64 MiB for scale; nothing refuses)", + "measurement_basis": "measure_constraint_digest_and_selection.py: the exact json.dumps call over the :71-76 payload at full-source widths, difference of 2 vs 2002 groups; the replica sha256 asserted equal to GroupedUpperBounds.constraint_digest at both sizes.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none (uncapped; unchanged \u2014 no diff)", + "head_value": "none (uncapped)", + "reachable_from": "age", + "reach_chain": "survey_age_calibration \u2192 freeze_survey_origin_budget \u2192 _document :669; checked_view :918; decode_numeric_survey_bounds graph_survey_calibration.py:143 (seven decodes per run); calibrate(... grouped_upper_bounds=bounds.grouped ...) graph_survey_calibration.py:383 consumes the object.", + "classification": "stays", + "notes": "Also materialised per construction: `groups.tolist()` (:74), one immutable numpy array per group in `_members` (:64-70; 1,587,376 small arrays at full source) and the `ids` tuple (:40). These are memory, not bounds. `check`/`totals` (:109-131) do a Python-level math.fsum per group." + } + ], + "open_questions": [ + "Row keying: the brief keys one row per constant definition, but survey_origin_budget.MAX_PAYLOAD_BYTES (:54) and MAX_ROSTER_BYTES (:63) each sit in front of several different documents with different shapes and verdicts (the budget accumulation, the successor document, the geography-binding population, the selection digest, the consumer re-encoding). I split them into per-document rows with the constant name repeated and a bracketed qualifier so each verification agent gets one document; a reviewer may prefer to merge.", + "Classification set: no bucket names a fixed-shape document cap (financial successor 256 KiB, diagnostics 8 MiB, registry 1 MiB, bands 64 KiB, activation 64 KiB, descriptor 4 KiB) or an uncapped roster re-encoding (the two constraint-digest json.dumps sites). I used must-not-move for the fixed-shape caps (with the reason) and stays with 'cap: none' for the uncapped re-encodings.", + "The FINAL_BUDGET_VIEW_DOCUMENT conversion to graph._json_matches (survey_origin_budget.py:927-937) is an UNCOMMITTED working-tree modification at the head I read (git status: M; git diff vs HEAD 77bafbb3a contains only that hunk). HEAD still has `_json(document) == entry[1]` under 64 MiB, which binds at 1/10 and at full source. The row records both; whoever commits should confirm it lands.", + "byte-census-measurements.json age_artifact encoded the header with age_convention 'integer_years_completed'; this head's graph_national_age_counts.SUPPORTED_AGE_CONVENTION is 'observed_interview_age_completed_years', so the header is 549 B not 534 B and the artifact 482,562,866 B not 482,562,851 B (15 B). Row and body figures are identical; the row uses the head figure.", + "The geography-binding semantic_population (50.8 MB at full source under 64 MiB, headroom 1.32\u00d7) is the tightest non-binding roster document in this family and is a single encoder piece; it is reachable only when a geography_config is supplied to the age runner. Its household count assumes value.population is the postclone (clone-roster) population per the module docstring (:7-8) and the stage bindings (:324-328); a verifier should confirm that against survey_atomic_geography.reconstruct_atomic_survey_geography, outside this family.", + "The registry declaration (demographic_calibration_graph.py:146, 1 MiB) and the diagnostics artifact (8 MiB) were not measured: no I/O-free registry factory exists in national_age_activation and no diagnostics artifact is among the permitted inputs. Both are fixed-shape by reading the code.", + "Household-id widths at full source are assumed 7-digit (as in the existing measurement); the 1/1000 receipt shows non-sequential ids (ACS from 1, ASEC up to 55,002 at 1/1000), so the clone-id scheme at full source is not observed. The measured average digit count over 0..1,587,375 is 6.30 and over 1..3,174,752 is 6.65, so the true per-group cost of the budget document is ~2 B below 742.", + "graph_survey_calibration.py:218 `1 <= epochs <= 1000` caps the loss_trajectory record count of the diagnostics artifact (survey_calibration_diagnostics.py:50); it is a solver parameter and is noted under MAX_DIAGNOSTIC_BYTES rather than given its own row.", + "survey_age_calibration.py defines no byte bound of its own; it appears only as enforcement sites (MAX_DIAGNOSTIC_BYTES at :141-166; CANDIDATE_LIMIT via :664) and as the reach chain for every row marked age.", + "The brief's `_recovered/pilot-runs/.../preparation.json` path is relative; the file lives at /Users/maxghenis/PolicyEngine/_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json (outside the worktree) and was read by absolute path as a \u00a74 input. No other recovered artifact was read." + ] + }, + { + "family": "ASEC native population, catalogue and prepared source", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py (full; no byte bound found: schema/identity validation only, HDF read via load_frame_checkpoint)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py (full; no byte bound found: _hash_file streams via hashlib.file_digest, _read_numeric_household reads selected int64 blocks whole with no cap)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py (full; no own constant; enforces asec_housing_status.HEADER_MAX_BYTES via _parse at :287)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py (full; no own constant; enforces asec_housing_universe.HEADER_MAX_BYTES via _parse at :143 and :221)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_original_household_weights.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py (full; no module-level byte constant; its canonical_json re-encodings at :189, :407-409, :651-652 are on digest-shaped documents)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py (full; no byte constant; re-encodings at :159 and :243 are on digest-shaped documents)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_coverage_fields.py (lines 150-320, for the nested field-record shape and its enforcement sites at :209, :265-266, :270, :403)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py (lines 495-520, 540-570, 1150-1185: _file_limits, _source_files enforcement, native receipt consumer)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py (lines 230-330: _Kernel.implementation_hash and _qualified)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py (lines 180-260: qualify_current_asec_demographics)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py (grep: :85 demographics call, :359 demographic_conditioning gate)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py (grep: :47-49 HEADER_MAX_BYTES/MAX_PERSONS/MAX_HOUSEHOLDS, :77-97 _json/_parse)", + "packages/microcosm-build/src/microcosm/build/us_runtime/education_assistance_source.py (grep: :112/:134/:156 member_size_bytes pins, :117/:139/:161 rows)", + "packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py (grep: :127 MAX_SOURCE_BYTES, :146/:156/:166 size_bytes pins)", + "packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py (grep: :183-186 origin_key is a sha256 hex)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py (lines 104-160: _row_identity and encode_us_frame_context are digest-shaped)", + "packages/microcosm-build/src/microcosm/build/frame_sampling.py (grep: :313 receipt carries selected_household_ids_sha256, not ids)", + "packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py (grep: :558-569 receipt carries digests only)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_domains.py (grep: dataclass constructors)", + "_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/graph.json (node ids; survey_predictors params demographic_conditioning=true)", + "_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/preparation.json (structure and id widths only)" + ], + "rows": [ + { + "constant": "asec_2024_native_population.py:41 _MAX_EVIDENCE_BYTES", + "value": "128 * 1024**2 = 134,217,728", + "enforced_at": [ + "asec_2024_native_population.py:65 (_encode: `_require(len(result) + len(part) <= _MAX_EVIDENCE_BYTES, \"EVIDENCE_SIZE\")` charged per iterencode part; AsecNativePopulationError)", + "asec_2024_native_population.py:141-145 (`len(candidate) <= _MAX_EVIDENCE_BYTES`, \"CANDIDATE\"; AsecNativePopulationError)", + "roster document encoded under it: asec_2024_native_population.py:760-795 (the population payload); fixed-shape documents under the same encoder: :223-230 _frame_identity, :409/:422/:528/:701 _implementation()" + ], + "refusal_codes": [ + "EVIDENCE_SIZE", + "CANDIDATE" + ], + "encodes": "The issued 2024 native-population payload: `households` = one row per selected 2024 ASEC household (:336-347), each nesting the whole hhpub anchor record as `source` (asec_original_household_weights._record_fields :274-316 plus the pins added at :364-373) and the whole household-fields record as `household_fields` (asec_household_coverage_fields._project :207-254 plus pins); `persons` = one 9-key dict per selected ASEC person (`projection.loc[mask].to_dict(orient=\"records\")` :348 over asec_coverage_authentication.COLUMNS); `selection.keys` = one [2024, H_SEQ] per household; plus a fixed preamble (coverage receipt header ~12.5 kB, parent identity, digests).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "12,256,009 (5,576 households + 14,212 persons)", + "bytes_at_full_source": "122,530,878 (55,762 households + 142,125 persons); headroom 1.095x", + "measurement_basis": "Encoded through asec_2024_native_population._encode on invented rows at full-source widths (6-digit parent receiving ids, 5-digit H_SEQ, 22-digit PERIDNUM, 64-hex digests, PRPERTYP \"2\"/\"observed_code\", a representative coverage header built as _reconstruct :673-702 with the real _implementation() = 12,454 B via _bounded_header, a 30-key parent-identity placeholder). Difference of two counts (1,000 hh/2,549 p -> 2,209,446 B; 2,000/5,097 -> 4,404,786 B) gives 2,195.34 B per household-with-2.5487-persons; a household row is 1,667 B incl. comma and selection key, a person row 205 B, preamble 16,118 B; then the whole document was encoded at both census counts (tenth 0.28 s, full 2.85 s, no refusal). Receipt: scratchpad/census/asec/asec-byte-census.json#native_population_payload.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "134,217,728", + "head_value": "134,217,728 (module unchanged vs origin/native-row-ceilings)", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial -> survey_population.create -> graph_survey_population create kernel -> survey_population_preparation.py:1965 asec_native.load_authenticated_asec_2024_native_population(**kwargs, selected_households=asec_keys) -> :760 _encode(payload); re-borrowed by replay/verification through _checked (:614-632, whole-payload equality) and consumed whole by survey_population_preparation.py:1160-1162 and current_asec_demographics.py:216/240 via json.loads (also reachable from 45, which contains the nineteen).", + "classification": "stays", + "notes": "Headroom at full source is only 9.5%. The width comes from :344-345 `\"source\": row, \"household_fields\": field_rows[key[1]]` nesting two full ~700 B records into every household row (1,667 B per household vs 205 B per person). The row-ceilings census estimated ~117 MB; this head measures 122.5 MB through the real encoder. The 1/10 figure is 12.3 MB. Three consumers materialise the whole payload again: `receipt` (:660-661 json.loads), survey_population_preparation.py:1162 (`asec_rows = {row[\"person_id\"]: row for row in native_receipts[\"asec\"][\"persons\"]}`), current_asec_demographics.py:240 (dict of all household rows). `_frame_identity` also copies the weights vector (`vector.values.tobytes()` :219) under the same encoder but that document is fixed-shape." + }, + { + "constant": "asec_2024_native_population.py:42 _MAX_CHECKPOINT_BYTES", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "asec_2024_native_population.py:162-165 (_file_identity: `stat.S_ISREG(before.st_mode) and 0 < before.st_size <= maximum`, \"SOURCE_FILE_BOUNDS\"), with this maximum supplied for P/H/T at :720-732 and rechecked on every borrow at :418-419", + "asec_2024_native_population.py:168-170 (read cap `os.read(descriptor, min(1024**2, before.st_size - count + 1))`, `count <= before.st_size` \"SOURCE_FILE_CHANGED\")", + "asec_population_catalogue.py:593-611 (`native._MAX_CHECKPOINT_BYTES` for the same three files; enforced via native._file_identity at :454)", + "survey_population_preparation.py:506 (_file_limits gives every _SOURCE_ROSTER entry this cap; enforced at :558 `_regular_reader(path, limits[name])` (:425 `0 <= before.st_size <= maximum`) and :561 `_require(size <= limits[name], \"SOURCE_FILE_LIMIT\")`, SurveyPopulationPreparationError)" + ], + "refusal_codes": [ + "SOURCE_FILE_BOUNDS", + "SOURCE_FILE_CHANGED", + "SOURCE_FILE_LIMIT" + ], + "encodes": "An upstream file: each of the three ASEC checkpoint HDFs (raw-stage parent P, household attachment H, restored person-income attachment T), hashed block by block.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: T = 666,333,922 (row-ceilings census, measured on the recovered restoration checkpoint); P and H not established", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Not re-measured (gated data); the T size is cited from experiments/native-row-ceilings/census.json (`_ATTACHMENT_MAX_BYTES` row). Fraction-independent by construction: the files are the whole three-cohort source.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "8,589,934,592", + "head_value": "8,589,934,592 (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> survey_population_preparation:1965 -> load_authenticated_asec_2024_native_population -> _validate_state :418-419 -> _file_identity; and :1942 issue_asec_source_catalogue -> _validate_state :453-454.", + "classification": "must-not-move", + "notes": "Protects an upstream file's real size; the 1 MiB `os.read` chunk at :168 is a chunk width, not a document bound." + }, + { + "constant": "education_assistance_source.py:112/134/156 member_size_bytes (pppub23 281,065,733; pppub24 277,250,415; pppub25 277,882,549), carried as _MEMBER_PINS in asec_coverage_authentication.py:34-37, asec_person_income_source.py:35-38, asec_demographic_source.py:388-391, asec_student_controls.py:46-49, asec_income_observations.py:53-56", + "value": "exact pins: 281,065,733 / 277,250,415 / 277,882,549", + "enforced_at": [ + "asec_coverage_authentication.py:323 (`before.st_size == size`, \"COVERAGE_SOURCE_SIZE\"), :329-331 (read cap `src.read(min(_CHUNK, size - count + 1))`, `count <= size` \"COVERAGE_SOURCE_SIZE\"), :337-340 (\"COVERAGE_SOURCE_CHANGED\"); AsecCoverageAuthenticationError", + "asec_2024_native_population.py:734-742 (`source[\"member_bytes\"]` passed as both maximum and size to _file_identity -> :163 \"SOURCE_FILE_BOUNDS\", :166 \"SOURCE_FILE_SIZE\")", + "asec_population_catalogue.py:613-622 (same, via native._file_identity)", + "asec_person_income_source.py:83 (`path.stat().st_size == size`, \"MEMBER_SIZE\", MoneyRefusalError)", + "asec_student_controls.py:103 (_snapshot `before.st_size == size` \"STUDENT_SOURCE_SIZE\"), :109 (read cap `src.read(min(1024 * 1024, size - count + 1))`), :111, :116-120 (\"STUDENT_SOURCE_CHANGED\"); reused by asec_demographic_source.py:1189 (\"DEMOGRAPHIC_SOURCE_BYTES\") and :537 (\"MEMBER_SIZE\"), asec_income_observations.py:196/:403 (_capture), asec_student_controls.py:254" + ], + "refusal_codes": [ + "COVERAGE_SOURCE_SIZE", + "COVERAGE_SOURCE_CHANGED", + "SOURCE_FILE_BOUNDS", + "SOURCE_FILE_SIZE", + "MEMBER_SIZE", + "STUDENT_SOURCE_SIZE", + "STUDENT_SOURCE_CHANGED", + "DEMOGRAPHIC_SOURCE_BYTES" + ], + "encodes": "An upstream file: one official pppub CSV member per cohort, captured whole to a private temp copy and hashed.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 281,065,733 / 277,250,415 / 277,882,549 (pinned)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "The pins themselves (education_assistance_source.py:112/134/156); no gated read.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "exact pins (unchanged)", + "head_value": "exact pins (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> load_authenticated_asec_2024_native_population -> restoration.load_authenticated_restored_current_money_source (:702) and coverage_owner.authenticate_asec_coverage (:706) -> _reconstruct -> _capture; and survey_predictors.* -> current_asec_demographics:222 -> asec_demographic_source._reconstruct :1186-1192 -> _snapshot.", + "classification": "must-not-move", + "notes": "Definition lives outside this family's module list (education_assistance_source.py); recorded here so the family's enforcement sites are not lost \u2014 merge on the definition key if another family lists it. Every capture writes a whole private copy of the ~278 MB member to a temp directory (three members per reconstruction, repeated by the coverage, restoration, student, income and demographic owners)." + }, + { + "constant": "native_household_origin.py:146/156/166 size_bytes (hhpub23 30,259,450; hhpub24 30,448,089; hhpub25 33,298,479), the _ASEC_MEMBER_PINS read by asec_original_household_weights._registry :166-185", + "value": "exact pins: 30,259,450 / 30,448,089 / 33,298,479", + "enforced_at": [ + "asec_original_household_weights.py:172-177 (`pin.size_bytes <= _MAX_MEMBER_BYTES`, \"MEMBER_REGISTRY\")", + "asec_original_household_weights.py:336 (`before.st_size == pin.size_bytes`, \"CAPTURE_SIZE\"), :246-250 (_DigestReader.readinto cap `min(len(buffer), self.size - self.count + 1)`, `self.count <= self.size` \"CAPTURE_SIZE\"), :376-380 (\"CAPTURE_CHANGED\"); HouseholdWeightSourceError", + "asec_original_household_weights.py:485-486 (`_capture_owner._snapshot(paths[...], capture, size=pin.size_bytes)` -> asec_student_controls.py:103/109/111)", + "asec_household_coverage_fields.py:265-266, :306, :425 (same pattern for the field owner)", + "asec_2024_native_population.py:743-746 (`pin[\"size_bytes\"]` as maximum and size to _file_identity :163/:166); asec_population_catalogue.py:623-626", + "native_household_origin.py:126-127 (pin dataclass: `0 < self.size_bytes <= MAX_SOURCE_BYTES`; outside this family)" + ], + "refusal_codes": [ + "MEMBER_REGISTRY", + "CAPTURE_SIZE", + "CAPTURE_CHANGED", + "STUDENT_SOURCE_SIZE", + "SOURCE_FILE_BOUNDS", + "SOURCE_FILE_SIZE" + ], + "encodes": "An upstream file: the hhpub household CSV member; the 19-path reads only hhpub25 (`{2024: member}` at asec_2024_native_population.py:709-712).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 33,298,479 (hhpub25)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "The pins themselves (native_household_origin.py:166-167); no gated read.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "exact pins (unchanged)", + "head_value": "exact pins (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> load_authenticated_asec_2024_native_population :709 -> anchor_owner.load_authenticated_asec_household_weights -> _registry / _snapshot / _read_capture; also :710 field_owner.", + "classification": "must-not-move", + "notes": "Definition outside the family list (native_household_origin.py); merge on the definition key." + }, + { + "constant": "asec_population_catalogue.py:26 _MAX_ENVELOPE_BYTES", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "asec_population_catalogue.py:53 (_encode: `_require(len(result) + len(part) <= _MAX_ENVELOPE_BYTES, \"ENVELOPE_SIZE\")` per iterencode part; AsecSourceCatalogueError)", + "asec_population_catalogue.py:556-560 (`len(candidate) <= _MAX_ENVELOPE_BYTES`, \"CANDIDATE_SIZE_OR_TYPE\")", + "documents under this encoder: :172 one catalogue record `_encode([kind, _record(row)])`; :333 one bound record `_encode([kind, row])`; :82-89 _source_authority(); :440/:455/:573 _implementation(); :644-679 the catalogue receipt payload" + ], + "refusal_codes": [ + "ENVELOPE_SIZE", + "CANDIDATE_SIZE_OR_TYPE" + ], + "encodes": "A single record (one ASEC household with its nested persons, one ledger record, one native household row nesting the anchor and field records, or one person row) or the fixed catalogue receipt; never a whole roster (:45-46: \"The complete catalogue is hashed incrementally, never encoded as another body\").", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: largest household catalogue record with 16 persons 1,476 B (:172 form); largest bound household row 1,671 B (:333 form); catalogue receipt ~4,539 B", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Measured through asec_population_catalogue._encode on a 16-person invented household in both record forms and on a receipt shaped as :644-679 (receipt #catalogue_records_stream).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,048,576", + "head_value": "1,048,576 (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> survey_population_preparation.py:1942 issue_asec_source_catalogue -> :589 _records_identity / :659 _bound_records_identity / :644 payload.", + "classification": "must-not-move", + "notes": "A per-record envelope width; grows only with persons per household (~76 B per nested person), not with the selection fraction." + }, + { + "constant": "asec_population_catalogue.py:27 _MAX_RECORD_BYTES", + "value": "128 * 1024**2 = 134,217,728", + "enforced_at": [ + "asec_population_catalogue.py:174 (_records_identity: `size += len(data); _require(size <= _MAX_RECORD_BYTES, \"RECORD_SIZE\")` over every catalogue record; AsecSourceCatalogueError)", + "asec_population_catalogue.py:335 (_bound_records_identity: same accumulation over every native household row and person row of the complete 2024 cohort)" + ], + "refusal_codes": [ + "RECORD_SIZE" + ], + "encodes": "Two whole-cohort canonical streams, both fraction-independent because :587 calls `native._roster(parent, coverage, anchors, fields, None)` (the complete 2024 cohort, `selected=None`): (a) :159-194 the catalogue \u2014 55,762 household records with 142,125 nested persons plus 33,170 unrepresented-household ledger records; (b) :328-338 the native binding \u2014 the same 55,762 household rows (each nesting the anchor and field records) and 142,125 person rows that the population payload carries at full source, as `_encode([kind, row]) + b\"\\n\"`.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: (a) 26,535,585; (b) 124,133,900 (92.5% of the cap, headroom 1.081x)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "(a) asec_population_catalogue._records_identity run on 55,762 invented domains.AsecHousehold records nesting 142,125 domains.AsecPerson records (22-digit PERIDNUM, string A_LINENO/A_AGE, \"observed_code\") plus 33,170 UnrepresentedAsecHousehold records: identity[1] = 26,535,585 (389 B per 1,000 household-with-persons bytes-per-1000 389,000; 209,000 per 1,000 ledger records). (b) asec_population_catalogue._bound_records_identity run on the full-source rows/roster of the native payload measurement: canonical_bytes = 124,133,900 over 197,887 records. Receipt #catalogue_records_stream and #catalogue_bound_records_stream.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "134,217,728", + "head_value": "134,217,728 (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> survey_population_preparation.py:1942 issue_asec_source_catalogue -> :589 `_records_identity(households, ledger, memo=True)` and :659 `_bound_records_identity(rows, roster)`; every later borrow (`_checked` -> _validate_state :447/:462 -> _current_records_identity) re-streams (a) unless the leaf memo :299-325 hits.", + "classification": "stays", + "notes": "The tightest bound in the family and it does not shrink with the fraction: stream (b) is the same 124 MB at 1/1000 as at full source, so the recovered 1/1000 run's success (preparation.json /catalogues/asec) is the proof it is under in reality; the exact real figure is the catalogue receipt's `native_binding.canonical_bytes` (:659), which the recovered run kept only as a sha256. Widths here are invented (6-digit parent ids, 5-digit H_SEQ). A lift of _MAX_EVIDENCE_BYTES for the population payload would not help this stream, and vice versa." + }, + { + "constant": "asec_coverage_authentication.py:38 _HEADER_MAX", + "value": "262_144", + "enforced_at": [ + "asec_coverage_authentication.py:455-460 (_bounded_header: `remaining = _HEADER_MAX` charged per visited token, `_require(remaining >= 0, \"COVERAGE_HEADER_BYTES\")`; depth cap :463 \"COVERAGE_HEADER_DEPTH\")", + "asec_coverage_authentication.py:514-517 (validate: `0 < len(self._header) <= _HEADER_MAX`, \"COVERAGE_HEADER_BYTES\"); AsecCoverageAuthenticationError" + ], + "refusal_codes": [ + "COVERAGE_HEADER_BYTES", + "COVERAGE_HEADER_DEPTH" + ], + "encodes": "The coverage receipt header (:673-702): fixed keys, three `sources`, partition counts, the literal reader receipt (contract + three cohorts + two digests) and the `implementation` dict (7,931 B measured).", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 12,454 B representative", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "coverage._bounded_header on a header shaped exactly as :673-702 with the real _implementation() and literal.coverage_field_contract(), digests as placeholders (receipt #coverage_body.header_bytes_representative).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "262,144", + "head_value": "262,144 (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_population.create -> load_authenticated_asec_2024_native_population :706 -> authenticate_asec_coverage -> _reconstruct :703 `_bounded_header(header)`.", + "classification": "must-not-move", + "notes": "Charged token-at-a-time before the single `_json(header)` materialisation at :497; row-independent (`rows` and `compared_rows` are integers)." + }, + { + "constant": "asec_coverage_authentication.py:39 _BODY_MAX", + "value": "268_435_456", + "enforced_at": [ + "asec_coverage_authentication.py:281-286 (_CsvBounds.feed: per CSV record `self.budget[0] -= 4 + _NUMBERS.size + 12 + 22 + 32` then `-= 1` per PRPERTYP token byte, `_require(self.budget[0] >= 0, \"COVERAGE_BODY_BYTES\")`; one budget list `[_BODY_MAX]` seeded at :632 and shared by the three cohorts at :643-645)", + "asec_coverage_authentication.py:414 (_body: `_require(len(result) + _LENGTH.size + size <= _BODY_MAX, \"COVERAGE_BODY_BYTES\")`)", + "asec_coverage_authentication.py:424 (_decode_rows: `len(body) <= _BODY_MAX`)", + "asec_coverage_authentication.py:518-521 (validate: `len(self._body) <= _BODY_MAX`)", + "out-of-module callers seeding their own `budget=[..._BODY_MAX]` for _capture over the same members: current_asec_income_routing_source.py:1319, current_asec_dividend_source.py:353, current_asec_interest_source.py:329, current_asec_retirement_detail_source.py:408, current_social_security_source.py:239, current_survey_health_source.py:270, current_survey_housing.py:294, puf55_survey_recipients.py:156 (their reachability belongs to those families)" + ], + "refusal_codes": [ + "COVERAGE_BODY_BYTES" + ], + "encodes": "The coverage body: one length-prefixed record per ASEC person of all three cohorts (432,523): ` load_authenticated_asec_2024_native_population :706 -> authenticate_asec_coverage -> _reconstruct :632-668 (_capture x3, then _body).", + "classification": "stays", + "notes": "Three-cohort whole source, so no selection fraction moves it. The body is retained in `_attached_evidence` (asec_2024_native_population.py:399-400) and compared whole (`current[1:] == state.attached_evidence[1:]` :436-440) on every borrow; `table()` :552-559 materialises a 432,523-row DataFrame from it each call (`_roster` :234)." + }, + { + "constant": "asec_coverage_authentication.py:40 _ROW_MAX", + "value": "1_048_576", + "enforced_at": [ + "asec_coverage_authentication.py:272-275 (`self.row <= (_CSV_HEADER_MAX if self.header else _ROW_MAX)`, \"COVERAGE_CSV_RECORD_BYTES\" \u2014 raw CSV record bytes incl. quotes and delimiters)", + "asec_coverage_authentication.py:413 (_body: `size <= _ROW_MAX`, \"COVERAGE_ROW_BYTES\")", + "asec_coverage_authentication.py:432-435 (_decode_rows: `_NUMBERS.size + 12 <= size <= _ROW_MAX`, \"COVERAGE_ROW_BYTES\")", + "puf55_survey_recipients.py:162-164 (`stream.readline(owner._ROW_MAX + 1)`, `len(chunk) <= owner._ROW_MAX`; another family's module, not on these paths)" + ], + "refusal_codes": [ + "COVERAGE_CSV_RECORD_BYTES", + "COVERAGE_ROW_BYTES" + ], + "encodes": "A width: one pppub CSV record on capture, one encoded coverage record on encode/decode.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: ~1,955 B per pppub25 CSV record (277,882,549 / 142,125 from the pins); 100 B per encoded record", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Pins arithmetic and the _body measurement; no gated read.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,048,576", + "head_value": "1,048,576 (unchanged)", + "reachable_from": "19", + "reach_chain": "authenticate_asec_coverage -> _reconstruct -> _capture (feed) and _body.", + "classification": "must-not-move", + "notes": "The ` _reconstruct -> _capture / _body.", + "classification": "must-not-move", + "notes": "Token width; not a document." + }, + { + "constant": "asec_coverage_authentication.py:42 _CSV_HEADER_MAX", + "value": "65_536", + "enforced_at": [ + "asec_coverage_authentication.py:272-275 (the `self.header` branch of `self.row <= (_CSV_HEADER_MAX if self.header else _ROW_MAX)`, \"COVERAGE_CSV_RECORD_BYTES\"); the header is buffered at :278 and decoded once at :256-262 to locate PRPERTYP" + ], + "refusal_codes": [ + "COVERAGE_CSV_RECORD_BYTES" + ], + "encodes": "A width: the pppub CSV header line.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: not measured (gated CSV; a header line of several hundred short column names, a few kB)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Not measured; the value is a property of the upstream CSV header, not of any roster.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "19", + "reach_chain": "authenticate_asec_coverage -> _reconstruct -> _capture -> _CsvBounds.feed.", + "classification": "must-not-move", + "notes": "The header bytes are the only part of the CSV decoded inside _CsvBounds (:258-262)." + }, + { + "constant": "asec_coverage_authentication.py:44 _CHUNK", + "value": "65_536", + "enforced_at": [ + "asec_coverage_authentication.py:329 (`src.read(min(_CHUNK, size - count + 1))` \u2014 the cap is the pin `size`, the chunk is this constant)", + "asec_coverage_authentication.py:715 (_compare_candidate: `handle.read(min(_CHUNK, len(payload) - offset + 1))`)" + ], + "refusal_codes": [ + "none of its own; the surrounding checks raise COVERAGE_SOURCE_SIZE (:331) and COVERAGE_CANONICAL_BYTES (:716-719)" + ], + "encodes": "A read-chunk width for the streaming capture; no document is bounded by it.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 65,536 per read", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "The constant itself.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "19", + "reach_chain": "authenticate_asec_coverage -> _reconstruct -> _capture :329.", + "classification": "not-a-transport", + "notes": "A memory chunk size with no document behind it; listed in the implementation `limits` (:204-212) so it is code identity, not a bound." + }, + { + "constant": "asec_person_coverage_source.py:33 MAX_MEMBER_BYTES", + "value": "1_000_000_000", + "enforced_at": [ + "asec_person_coverage_source.py:168 (_regular_member_opener: `snapshot.st_size <= MAX_MEMBER_BYTES`, \"MEMBER_BYTES\"; CoverageSourceRefusalError)", + "asec_person_coverage_source.py:181 and :189-191 (_BoundedMember: `_remaining = MAX_MEMBER_BYTES`, `readinto(memoryview(buffer)[: self._remaining + 1])`, `_require(self._remaining >= 0, \"MEMBER_BYTES\")`)", + "asec_coverage_authentication.py:315-318 (_capture: `0 < size <= literal.MAX_MEMBER_BYTES`, \"COVERAGE_MEMBER_BYTES\"), :219 (recorded in the implementation identity)", + "survey_population_preparation.py:508 (_file_limits cap for asec/pppub23-25.csv -> :558 _regular_reader and :561 \"SOURCE_FILE_LIMIT\")" + ], + "refusal_codes": [ + "MEMBER_BYTES", + "COVERAGE_MEMBER_BYTES", + "SOURCE_FILE_LIMIT" + ], + "encodes": "An upstream file: one pppub CSV member (largest pinned 281,065,733).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 281,065,733 (largest member)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "The pins (education_assistance_source.py:112/134/156); no gated read.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1,000,000,000", + "head_value": "1,000,000,000 (unchanged)", + "reachable_from": "19", + "reach_chain": "authenticate_asec_coverage -> _reconstruct :658 read_asec_person_coverage_source -> _regular_member_opener / _BoundedMember.", + "classification": "must-not-move", + "notes": "`_digest` (:150-158) streams `json.dumps(row)` per person into sha256 with no cap and no whole copy (two passes over 432,523 rows); it is a stream, not a bound." + }, + { + "constant": "asec_person_income_source.py:39 _ATTACHMENT_MAX_BYTES", + "value": "2_000_000_000", + "enforced_at": [ + "asec_person_income_source.py:272-275 (_candidate_snapshot: `0 < before.st_size == expected_size <= _ATTACHMENT_MAX_BYTES`, \"RESTORATION_SIZE\"; MoneyRefusalError)", + "asec_person_income_source.py:277-279 (`src.read(1024 * 1024)` chunks, `count <= expected_size` \"RESTORATION_SIZE\"), :283 (`count == expected_size`), :284-300 (\"RESTORATION_BYTES_CHANGED\")" + ], + "refusal_codes": [ + "RESTORATION_SIZE", + "RESTORATION_BYTES_CHANGED" + ], + "encodes": "An upstream file: the candidate restored person-income checkpoint (asec_person_income_source.checkpoint.h5), whose expected size is the freshly written canonical checkpoint's (:323-333).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 666,333,922 (row-ceilings census, measured on the recovered restoration)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Cited from experiments/native-row-ceilings/census.json; not re-measured (gated).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "2,000,000,000", + "head_value": "2,000,000,000 (unchanged)", + "reachable_from": "19", + "reach_chain": "load_authenticated_asec_2024_native_population :702 -> load_authenticated_restored_current_money_source :455 -> verify_asec_person_income_source -> _verify :330 _candidate_snapshot (after write_frame_checkpoint :324 writes the expected copy).", + "classification": "must-not-move", + "notes": "Each verification rewrites the whole 666 MB canonical checkpoint to a temp directory and copies the candidate beside it (:320-337). `_read_member` :83 enforces the pppub pin exactly (see the member_size_bytes row)." + }, + { + "constant": "asec_prepared_source.py:91 RESTORATION_RECEIPT_MAX_BYTES", + "value": "256 * 1024 = 262,144", + "enforced_at": [ + "asec_prepared_source.py:217-219 (`0 < info.st_size <= RESTORATION_RECEIPT_MAX_BYTES`, \"RESTORATION_RECEIPT\"; PreparedSourceRefusalError)", + "asec_prepared_source.py:221 (`stream.read(RESTORATION_RECEIPT_MAX_BYTES + 1)`), :224 (`0 < len(payload) <= RESTORATION_RECEIPT_MAX_BYTES`)", + "asec_prepared_source.py:227 (`_parse(payload[:-1], RESTORATION_RECEIPT_MAX_BYTES)` -> asec_current_money.py:84 \"JSON_SIZE_OR_TYPE\" MoneyRefusalError, re-raised at :228-229 as \"RESTORATION_RECEIPT\")", + "asec_prepared_source.py:230 (consumer re-encoding `_json(receipt) == payload[:-1]`, \"RESTORATION_RECEIPT_CANONICAL\")" + ], + "refusal_codes": [ + "RESTORATION_RECEIPT", + "RESTORATION_RECEIPT_CANONICAL" + ], + "encodes": "restoration.receipt.json \u2014 a fixed receipt (schema keys :126-150, three `sources` :151-165).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: 2,424 B (row-ceilings census, recovered artifact)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Cited from experiments/native-row-ceilings/census.json.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "262,144", + "head_value": "262,144 (unchanged)", + "reachable_from": "none", + "reach_chain": "prepare_asec_current_money_population :421 is called only by graph_asec_prepared.USAsecPreparedCreateKernel.run :309 (node asec_prepared.create) and graph_composed_population.compose_from_sources :406 (node composed_population.create); neither is a node of the 19- or 45-node graphs nor of survey_age_calibration.", + "classification": "must-not-move", + "notes": "The other prepared-source encoders (`receipt_payload` :400, `encode_current_money` :471 from _asec_current_money_codec \u2014 the money family) carry no cap of this module's; the prepared receipt is digest-shaped (`entity_rows`, `dtype_transitions` per column)." + }, + { + "constant": "asec_demographic_source.py:66 _HEADER_MAX", + "value": "262_144", + "enforced_at": [ + "asec_demographic_source.py:1106-1109 (validate: `0 < len(self._header) <= _HEADER_MAX`, \"DEMOGRAPHIC_HEADER_SIZE\"; DemographicSourceRefusalError)", + "asec_demographic_source.py:1118-1119 (consumer re-encodings of fixed sub-documents: `_json(header[\"contract\"]) == _json(contract_document())`, `_json(header[\"implementation\"]) == _json(_implementation())`, \"DEMOGRAPHIC_CONTENT_CHANGED\")" + ], + "refusal_codes": [ + "DEMOGRAPHIC_HEADER_SIZE", + "DEMOGRAPHIC_CONTENT_CHANGED" + ], + "encodes": "The demographic header (:1302-1324): contract document, implementation, household-membership verification, per-cohort diagnostics, three `sources`, column digests \u2014 all fixed-shape.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: ~30-40 kB (contract 9,159 B and implementation 10,154 B measured; membership/diagnostics/sources fixed-shape, not measured)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "`len(demo._json(demo.contract_document()))` and `len(demo._json(demo._implementation()))` offline (receipt #demographic_source).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "262,144", + "head_value": "262,144 (unchanged)", + "reachable_from": "19", + "reach_chain": "survey_predictors.source_projection / .asec_design_donor (recovered graph.json params `demographic_conditioning: true`) -> graph_current_survey_predictors._Kernel._qualified :301 values.qualify_current_survey_predictors(..., demographic_conditioning=...) -> current_survey_predictors.py:359 `_demographic_features(preparation, entry) if demographic_conditioning` -> :85 observed_geography.demographics.qualify_current_asec_demographics -> current_asec_demographics.py:222 load_authenticated_asec_demographic_source(parent, member_paths=...) -> _reconstruct :1325.", + "classification": "must-not-move", + "notes": "Reached only because the pilot conditioned on demographics; with `demographic_conditioning=False` (the default at current_survey_predictors.py:315) the loader is never called and the module is only hashed for code identity (graph_current_survey_predictors.py:275)." + }, + { + "constant": "asec_demographic_source.py:75 _MAX_PERSONS (as the row factor of the width identity at :1115 `len(self._body) == rows * len(COLUMNS) * 8`)", + "value": "14_000_000 rows x 14 columns x 8 = 1,568,000,000 implicit body ceiling", + "enforced_at": [ + "asec_demographic_source.py:1112 (validate: `0 < rows <= _MAX_PERSONS`, \"DEMOGRAPHIC_ROWS\")", + "asec_demographic_source.py:1113-1121 (`len(self._body) == rows * len(COLUMNS) * 8`, \"DEMOGRAPHIC_CONTENT_CHANGED\")", + "asec_demographic_source.py:1153 (_reconstruct: `0 < rows <= _MAX_PERSONS`, \"DEMOGRAPHIC_ROWS\"); row-count-only sites :660 (\"MEMBERSHIP_ROWS\"), :803 (\"CLASSIFY_ROWS\"), :1024 (\"ACS_ROWS\", reached only from current_survey_household_roles \u2014 49/51-node variants)" + ], + "refusal_codes": [ + "DEMOGRAPHIC_ROWS", + "DEMOGRAPHIC_CONTENT_CHANGED" + ], + "encodes": "The demographic body: 14 little-endian int64 columns (COORDINATES + ALIASES + DERIVED, :378) over the three-cohort ASEC persons of the restored money parent (432,523).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: 48,442,576 (432,523 x 112 B)", + "bytes_at_full_source": "same as at tenth (headroom 32x against the implicit ceiling)", + "measurement_basis": "`len(demo.COLUMNS)` = 14 imported, times 8, times 432,523 (146,133 + 144,265 + 142,125 pinned rows); the body is `b\"\".join(buffers)` at :1300-1301 (receipt #demographic_source).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "14,000,000 (already lifted on origin/native-row-ceilings; experiments/native-row-ceilings/census.json predates that lift and still says 600,000)", + "head_value": "14,000,000 (unchanged on this branch)", + "reachable_from": "19", + "reach_chain": "As the _HEADER_MAX row: survey_predictors.* -> current_asec_demographics.py:222 -> _reconstruct :1300-1301.", + "classification": "stays", + "notes": "Not a byte-named constant: the body ceiling exists only through the width identity. `array()` :1123-1128 is a zero-copy frombuffer view; `attach_asec_demographic_source` :1447-1448 copies the nine alias/derived columns into the frame. The three pppub captures (`_snapshot`, ~278 MB each) are written whole to a temp directory per reconstruction." + }, + { + "constant": "asec_engine_evaluation.py:42 EVALUATION_HEADER_MAX_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "asec_engine_evaluation.py:391 (_evaluation_parts: `0 < len(header) <= EVALUATION_HEADER_MAX_BYTES`, \"HEADER_SIZE\"; EngineAdmissionError)", + "asec_engine_evaluation.py:424-427 (decode: `0 < size <= EVALUATION_HEADER_MAX_BYTES and start + size <= len(payload) - 32`, \"HEADER_SIZE\")", + "asec_engine_evaluation.py:431-433 (consumer re-encoding `canonical_json(header) == payload[start : start + size]`, \"HEADER_CANONICAL\")" + ], + "refusal_codes": [ + "HEADER_SIZE", + "HEADER_CANONICAL" + ], + "encodes": "The engine-evaluation header (:356-384): four closures, aggregates, baseline runtime identity, bindings \u2014 fixed-shape.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: not measured (fixed, a few kB)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Row-independent by the key roster at :356-384 (`person_rows` is an integer).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "none", + "reach_chain": "Would be reached by graph_asec_prepared.USAsecPreparedEngineKernel.run :848 encode_engine_evaluation (node asec_prepared.engine_current_money); decode_engine_evaluation has no production caller in packages/*/src.", + "classification": "must-not-move", + "notes": "Header width." + }, + { + "constant": "asec_engine_evaluation.py:43 EVALUATION_MAX_ROWS (times 8 x len(roots) through the length identity at :439 `len(payload) == start + size + 8 * rows * len(roots) + 32`)", + "value": "1_000_000 rows x 32 B = 32,000,000 implicit body ceiling (4 admitted roots x 8)", + "enforced_at": [ + "asec_engine_evaluation.py:345 (materialize_engine_outputs: `0 < rows <= EVALUATION_MAX_ROWS`, \"ENGINE_ROW_BOUND\")", + "asec_engine_evaluation.py:437 (decode: `0 < rows <= EVALUATION_MAX_ROWS`, \"ENGINE_ROW_BOUND\"), :438-440 (\"PAYLOAD_LENGTH\")", + "asec_engine_evaluation.py:398-401 (`array.shape == (rows,)`, \"ENGINE_RESULT_SHAPE\")" + ], + "refusal_codes": [ + "ENGINE_ROW_BOUND", + "PAYLOAD_LENGTH", + "ENGINE_RESULT_SHAPE" + ], + "encodes": "Four float64 arrays (` :558 _regular_reader and :561 \"SOURCE_FILE_LIMIT\")" + ], + "refusal_codes": [ + "MEMBER_REGISTRY", + "SOURCE_FILE_LIMIT" + ], + "encodes": "An upstream file: an hhpub household CSV member (largest pinned 33,298,479).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: 33,298,479 (hhpub25)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "The pins (native_household_origin.py:146/156/166).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "67,108,864", + "head_value": "67,108,864 (unchanged)", + "reachable_from": "19", + "reach_chain": "load_authenticated_asec_2024_native_population :709 -> load_authenticated_asec_household_weights :469 _registry.", + "classification": "must-not-move", + "notes": "Registry-level ceiling on the pins; the exact size is enforced by the pin rows." + }, + { + "constant": "asec_original_household_weights.py:43 _MAX_RECORD_CHARS", + "value": "65_536", + "enforced_at": [ + "asec_original_household_weights.py:266 (_RecordLines.__next__: `self.text.readline(_MAX_RECORD_CHARS - self.characters + 1)`), :270 (`self.characters <= _MAX_RECORD_CHARS`, \"MEMBER_RECORD_SIZE\")", + "asec_household_coverage_fields.py:270 (`shared._RecordLines(text)` \u2014 the same class for the field owner)" + ], + "refusal_codes": [ + "MEMBER_RECORD_SIZE" + ], + "encodes": "A width: one logical hhpub CSV record including embedded newlines.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: ~374 chars mean (33,298,479 / 88,932 from the pins)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Pins arithmetic.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "19", + "reach_chain": "load_authenticated_asec_household_weights :492 _read_capture -> _RecordLines.", + "classification": "must-not-move", + "notes": "Record width." + }, + { + "constant": "asec_original_household_weights.py:44 _MAX_FIELD_CHARS", + "value": "128", + "enforced_at": [ + "asec_original_household_weights.py:276 (_record_fields: `all(len(t) <= _MAX_FIELD_CHARS for t in tokens)`, \"MEMBER_FIELD_SIZE\"; HouseholdWeightSourceError)", + "asec_household_coverage_fields.py:208-210 (`len(token) <= shared._MAX_FIELD_CHARS`, \"MEMBER_FIELD_SIZE\"; HouseholdCoverageSourceError)", + "current_asec_demographics.py:60 (`len(token) <= household._MAX_FIELD_CHARS` on the current-state tokens; on the 19 path via qualify_current_asec_demographics :236 _load_current_state)" + ], + "refusal_codes": [ + "MEMBER_FIELD_SIZE" + ], + "encodes": "A width: one H_SEQ / H_HHTYPE / HSUP_WGT (and field/state) token.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "identical at every fraction: <= 8 (HSUP_WGT printed_length 8, _FIELD_CONTRACT :53-77)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "_FIELD_CONTRACT literals.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "128", + "head_value": "128 (unchanged)", + "reachable_from": "19", + "reach_chain": "load_authenticated_asec_household_weights -> _read_capture :360 _record_fields; and survey_predictors.* -> current_asec_demographics :236.", + "classification": "must-not-move", + "notes": "Token width." + }, + { + "constant": "asec_original_household_weights.py:45 _MAX_PAYLOAD_BYTES", + "value": "512 * 1024**2 = 536,870,912", + "enforced_at": [ + "asec_original_household_weights.py:95-97 (_issue_capsule: `len(payload) <= _MAX_PAYLOAD_BYTES`, \"SOURCE_BYTES\"; HouseholdWeightSourceError)", + "asec_original_household_weights.py:116-121 (_checked_capsule_payload: `len(payload) <= _MAX_PAYLOAD_BYTES and _sha(payload) == entry[1]`, \"SOURCE_MUTATION\")", + "asec_original_household_weights.py:161 (_encode: `len(result) + len(encoded) <= _MAX_PAYLOAD_BYTES` per iterencode fragment, \"ENCODING_SIZE\")", + "asec_original_household_weights.py:471-475 (`len(candidate) <= _MAX_PAYLOAD_BYTES`, \"CANDIDATE_SIZE\")", + "asec_household_coverage_fields.py:403 (`len(candidate) <= shared._MAX_PAYLOAD_BYTES` for the field owner's candidate)" + ], + "refusal_codes": [ + "SOURCE_BYTES", + "SOURCE_MUTATION", + "ENCODING_SIZE", + "CANDIDATE_SIZE" + ], + "encodes": "The household-weights payload (:498-525) with `records` = one record per hhpub25 row (88,932; `_record_fields` :274-316 plus the pins added at :364-373), and two further whole materialisations under the same cap: `_encode(records)` for `projection_sha256` (:518) and `_encode([[income_year, native_household_id] ...])` for `native_roster_sha256` (:515-517).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: payload 45,437,948; records-only 45,433,147; roster list 1,156,117 (510 B per record)", + "bytes_at_full_source": "same as at tenth (headroom 11.8x)", + "measurement_basis": "anchors._encode on 88,932 invented records built by anchors._record_fields([str(H_SEQ), \"1\", \"199603\"]) plus pins, with the real _implementation() as `producer`; per-record width from the 1,000 -> 2,000 difference (receipt #household_weights_payload).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "536,870,912", + "head_value": "536,870,912 (unchanged)", + "reachable_from": "19", + "reach_chain": "load_authenticated_asec_2024_native_population :709 -> load_authenticated_asec_household_weights :498 _encode(payload).", + "classification": "stays", + "notes": "Whole member at every fraction. Consumers re-parse the 45 MB payload on every `.document` access (`verify_asec_household_weights_source` :454 `json.loads(payload)`): asec_2024_native_population.py:241, :321 (original_weight_fractions :419), asec_population_catalogue.py:373, and every borrow via :416/:451; `_encode(doc[\"producer\"]) == _encode(_implementation())` at :455 re-encodes only the fixed producer sub-document." + }, + { + "constant": "graph_asec_income.py:81 ACCOUNTING_HEADER_MAX", + "value": "65536", + "enforced_at": [ + "graph_asec_income.py:482-483 (derive_reported_income: `encoded = _json(header); _require(len(encoded) <= ACCOUNTING_HEADER_MAX, \"ACCOUNTING_HEADER_SIZE\")`; MoneyRefusalError)", + "factor of ACCOUNTING_PAYLOAD_MAX_BYTES at graph_asec_income.py:86" + ], + "refusal_codes": [ + "ACCOUNTING_HEADER_SIZE" + ], + "encodes": "The reported-income accounting header (:450-481): 28 column descriptors (name, dtype, offset, bytes, sha256), input digests, source digests \u2014 fixed-shape.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: not measured (28 descriptors with a sha256 each, roughly 3-4 kB)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Row-independent by the key roster.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "none", + "reach_chain": "USAsecReportedIncomeKernel.run :628 / USComposedAsecReportedIncomeKernel.run :455 -> derive_reported_income \u2014 nodes asec_prepared.reported_income / composed_population.asec_reported_income, not in the 19/45 graphs.", + "classification": "must-not-move", + "notes": "Header width." + }, + { + "constant": "graph_asec_income.py:82 ACCOUNTING_ROW_BYTES", + "value": "8 * 9 + len(EVIDENCE_COLUMNS) = 72 + 19 = 91", + "enforced_at": [ + "graph_asec_income.py:445-449 (`cursor == rows * ACCOUNTING_ROW_BYTES`, \"ACCOUNTING_LENGTH\")", + "factor of ACCOUNTING_PAYLOAD_MAX_BYTES at graph_asec_income.py:87" + ], + "refusal_codes": [ + "ACCOUNTING_LENGTH" + ], + "encodes": "A width: bytes per accounting row (3 int64 + 6 float64 + 19 uint8 columns, ACCOUNTING_DTYPES :78).", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: 91", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Imported constant.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "91", + "head_value": "91 (unchanged)", + "reachable_from": "none", + "reach_chain": "As ACCOUNTING_HEADER_MAX.", + "classification": "must-not-move", + "notes": "Encoder width." + }, + { + "constant": "graph_asec_income.py:83-89 ACCOUNTING_PAYLOAD_MAX_BYTES", + "value": "len(ACCOUNTING_MAGIC) + 4 + ACCOUNTING_HEADER_MAX + source._MAX_PERSONS * ACCOUNTING_ROW_BYTES + 32 = 9 + 4 + 65,536 + 600,000 * 91 + 32 = 54,665,581", + "enforced_at": [ + "graph_asec_income.py:495 (encode_reported_income: `len(payload) + 32 <= ACCOUNTING_PAYLOAD_MAX_BYTES`, \"ACCOUNTING_SIZE\"; MoneyRefusalError)", + "graph_asec_income.py:501-504 (read_reported_income: `len(payload) <= ACCOUNTING_PAYLOAD_MAX_BYTES`, \"ACCOUNTING_SIZE\")", + "graph_asec_income.py:506 (consumer re-encoding `payload == encode_reported_income(expected)`, \"ACCOUNTING_RECONSTRUCTION\")" + ], + "refusal_codes": [ + "ACCOUNTING_SIZE", + "ACCOUNTING_RECONSTRUCTION" + ], + "encodes": "The reported-income accounting payload: header + 28 buffers, 91 B per selected ASEC person (the prepared slice's sample, or the composed population's ASEC arm).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "3,935,932 at 1/10 of the prepared population (43,252 persons x 91) \u2014 the prepared slice's own fraction", + "bytes_at_full_source": "39,359,593 over the whole prepared population (432,523 x 91) + header + 45 framing; headroom 1.39x", + "measurement_basis": "Width x count (receipt #fixed_width_bodies.reported_income_accounting).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "54,665,581", + "head_value": "54,665,581 (unchanged)", + "reachable_from": "none", + "reach_chain": "USAsecReportedIncomeKernel.run :650 encode_reported_income; USComposedAsecReportedIncomeKernel.run :479 \u2014 not in the 19/45 graphs or the age runner.", + "classification": "stays", + "notes": "The row ceiling is source._MAX_PERSONS = 600,000, 1.39x the whole three-cohort roster; `read_reported_income` has no production caller in packages/*/src." + }, + { + "constant": "asec_current_money.py:47 HEADER_MAX_BYTES (imported at graph_composed_asec_binding.py:78; definition owned by the current-money family)", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "graph_composed_asec_binding.py:1016 (_arm_rows_parts: `len(header) <= HEADER_MAX_BYTES`, \"ARM_ROWS_HEADER_SIZE\"; PreparedGraphError)", + "graph_composed_asec_binding.py:1056-1059 (bind_composed_asec_arm_rows: `0 < size <= HEADER_MAX_BYTES and start + size < len(payload) - 32`, \"ARM_ROWS_HEADER_SIZE\")", + "graph_composed_asec_binding.py:1061 (`_parse(header, HEADER_MAX_BYTES)` -> asec_current_money.py:84 \"JSON_SIZE_OR_TYPE\")", + "graph_composed_asec_binding.py:1063 (consumer re-encoding `_json(data) == header`, \"ARM_ROWS_HEADER_CANONICAL\")", + "term of ARM_ROWS_PAYLOAD_MAX_BYTES at graph_composed_asec_binding.py:959" + ], + "refusal_codes": [ + "ARM_ROWS_HEADER_SIZE", + "JSON_SIZE_OR_TYPE", + "ARM_ROWS_HEADER_CANONICAL" + ], + "encodes": "The arm-rows header (:995-1015): seven keys, a four-entry buffer roster with byte counts and digests \u2014 fixed-shape.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: roughly 600 B (four buffer descriptors with a sha256 each)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "Row-independent by the key roster (`rows` are two integers).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "65,536", + "head_value": "65,536 (unchanged)", + "reachable_from": "none", + "reach_chain": "USComposedAsecBindKernel.run :1268 encode_composed_asec_arm_rows; the measure kernels' bind_composed_asec_arm_rows \u2014 node composed_population.asec_bind only.", + "classification": "must-not-move", + "notes": "Only this family's enforcement sites are listed; merge on the definition key with the current-money family's row (asec_current_money.py:47), which also carries MAX_PERSONS :48 and MAX_HOUSEHOLDS :49." + }, + { + "constant": "graph_composed_asec_binding.py:956-962 ARM_ROWS_PAYLOAD_MAX_BYTES", + "value": "len(ARM_ROWS_MAGIC) + 4 + HEADER_MAX_BYTES + 16 * (MAX_PERSONS + MAX_HOUSEHOLDS) + 32 = 9 + 4 + 65,536 + 16 * (1,000,000 + 400,000) + 32 = 22,465,581 (row factors from asec_current_money.py:48-49)", + "enforced_at": [ + "graph_composed_asec_binding.py:1033 (encode_composed_asec_arm_rows: `len(payload) + 32 <= ARM_ROWS_PAYLOAD_MAX_BYTES`, \"ARM_ROWS_SIZE\"; PreparedGraphError)", + "graph_composed_asec_binding.py:1047-1051 (bind_composed_asec_arm_rows: `len(ARM_ROWS_MAGIC) + 4 + 32 < len(payload) <= ARM_ROWS_PAYLOAD_MAX_BYTES`, \"ARM_ROWS_SIZE\")", + "graph_composed_asec_binding.py:1093-1106 (width identity `width = rows[entity] * 8` per buffer, \"ARM_ROWS_BUFFER_HEADER:...\" / \"ARM_ROWS_LENGTH\")" + ], + "refusal_codes": [ + "ARM_ROWS_SIZE", + "ARM_ROWS_BUFFER_HEADER", + "ARM_ROWS_LENGTH" + ], + "encodes": "Four int64 buffers (source positions and original ids for the ASEC-arm persons and households of the composed population), 16 B per arm person row + 16 B per arm household row (:984-1017, :1030-1031).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "963,536 at 1/10 of the prepared arm (43,252 persons + 16,969 households) \u2014 the composed graph's own sample_fraction", + "bytes_at_full_source": "9,635,552 for the whole prepared arm (432,523 persons + 169,699 households, the latter from the row-ceilings derivation); headroom 2.33x", + "measurement_basis": "Width x count (receipt #fixed_width_bodies.arm_rows).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "22,465,581", + "head_value": "22,465,581 (unchanged)", + "reachable_from": "none", + "reach_chain": "USComposedAsecBindKernel.run :1268 (composed_population.asec_bind); bound again by graph_composed_asec_measures :325/:446 \u2014 not in the 19/45 graphs or the age runner.", + "classification": "stays", + "notes": "The arm can never exceed the prepared population (`ARM_OVERFLOW` :636-639), so the bound is fixed at 2.33x headroom whatever the composed fraction." + }, + { + "constant": "graph_composed_population.py:322-333 canonical_json(channels) / canonical_json(spine_ids) / canonical_json([[channel, spine_id, source_id, output_id] ...]) in _source_origin (cap: none)", + "value": "none (no byte cap; each list is materialised whole by microcosm.graph.canonical.canonical_json before `_sha`)", + "enforced_at": [ + "producer graph_composed_population.py:311-334 (_source_origin: three whole-roster encodings per entity for all six US entities, hashed into `ordered_channels_sha256`, `ordered_spine_source_ids_sha256`, `ordered_source_to_output_sha256`)", + "consumer graph_composed_asec_binding.py:448-470 (_origin_entity_digests recomputes the same three encodings per entity from the composed tables) compared at :613-616 (`_origin_entity_digests(tables) == origin[\"assembled\"]`, \"ORIGIN_DOES_NOT_DESCRIBE_THIS_FRAME\"; PreparedGraphError)" + ], + "refusal_codes": [ + "ORIGIN_DOES_NOT_DESCRIBE_THIS_FRAME" + ], + "encodes": "Whole-roster provenance lists over every row of the composed (stacked) population: per entity a list of channel strings, a list of pre-remap spine ids and a list of four-element rows; also `.tolist()` copies at :311-316 / :448-453.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "four-tuple list (largest of the three): households 5,085,952; persons 11,422,301 (158,737 / 356,501 rows); channels 6.04 B/row, spine ids 8.0 B/row, tuples 32.04 B/row", + "bytes_at_full_source": "four-tuple list: households 50,859,527; persons 114,223,016 (1,587,376 / 3,565,013 rows); channels 9,587,751 / 21,532,678; spine ids 12,699,008 / 28,520,104; the four other entities scale by the 1/1000 roster ratios (SCALED)", + "measurement_basis": "canonical_json on invented lists at 100,000 and 200,000 rows with 7-digit ids and a 4% \"asec\" share, per-row width by difference, times the stacked counts (receipt #composed_origin_rosters).", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none", + "head_value": "none (unchanged)", + "reachable_from": "none", + "reach_chain": "graph_composed_population.compose_from_sources :438 / USComposedPopulationCreateKernel.run :487 (composed_population.create) and USComposedAsecBindKernel.run :1238 -> resolve_composed_asec_binding :614 \u2014 not in the 19/45 graphs or the age runner.", + "classification": "segment", + "notes": "No cap, so it cannot bind, but each of the eighteen lists exists whole before hashing (114 MB for the persons' four-tuple list at full source) and the consumer builds all of them again; a per-row digest stream keeps the bytes and removes the whole copies. Both `binds` flags are false only because there is nothing to exceed." + }, + { + "constant": "graph_composed_population.py:252-259 canonical_json([None if value is None or value is pd.NA else str(value) for value in series.tolist()]) in _arm_origin (cap: none)", + "value": "none (no byte cap; one whole-column list per native identity column, materialised then `_sha`)", + "enforced_at": [ + "graph_composed_population.py:241-260 (_arm_origin, called at :382-385 for both arms inside _source_origin); the ACS arm's SERIALNO / source_household_id / source_person_id and the ASEC arm's asec_H_SEQ / PERIDNUM / PH_SEQ per NATIVE_IDENTITY_COLUMNS :151-162" + ], + "refusal_codes": [ + "none (ARM_NATIVE_IDENTITY:* only refuses a missing column at :246)" + ], + "encodes": "Whole-arm identity lists at every fraction: the sources are read whole (:406 prepare_asec_current_money_population, :419 load_graph_acs) before prepare_stacked_spine samples (:428-430), so the lists cover the whole prepared ASEC arm and the whole ACS source.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "identical at every fraction: ASEC PERIDNUM 10,813,075 (432,523 x 25); PH_SEQ 3,460,184; asec_H_SEQ 1,357,592 (169,699 households, DERIVED); ACS SERIALNO 24,505,824 (1,531,614 x 16); ACS source_household_id and source_person_id 34,228,880 each (3,422,888 x 10)", + "bytes_at_full_source": "same as at tenth", + "measurement_basis": "canonical_json on invented string lists at two sizes (22-digit PERIDNUM, 5-digit H_SEQ, 13-char SERIALNO, 7-digit ints as str), per-element width by difference, times the whole-source counts (receipt #composed_origin_rosters.arm_origin). The SERIALNO figure agrees with byte-census-measurements.json acs_household_archive.serialno_list_bytes_full_source = 24,505,825 measured by the ACS family through its own encoder.", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "none", + "head_value": "none (unchanged)", + "reachable_from": "none", + "reach_chain": "graph_composed_population.compose_from_sources :438 -> _source_origin :382-385 -> _arm_origin (composed_population.create only).", + "classification": "segment", + "notes": "Whole ACS-source identity lists (24.5 MB + 2 x 34.2 MB) are built even at 1/1000 because the arm is read whole before sampling; the `.tolist()` at :256 and the str() conversion double the transient copy." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/asec/measure_asec.py", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/asec/asec-byte-census.json" + ], + "open_questions": [ + "Native population payload headroom is 9.5% at full source (122,530,878 vs 134,217,728) on invented widths: 6-digit parent receiving ids, 5-digit H_SEQ, 1-digit A_LINENO, \"observed_code\" states, and a 30-key placeholder for `parent.source.identity` in the fixed preamble (16,118 B total). A real full-source encode could move the figure by roughly +/-1%; the verdict `stays` holds unless real rows are ~10% wider than these.", + "The catalogue's native-binding stream (asec_population_catalogue.py:328-338) measures 124,133,900 B against 134,217,728 at every fraction (92.5%). The only exact figure is the catalogue receipt's `native_binding.canonical_bytes` (:659), which the recovered 1/1000 run kept only as `catalogues.asec.receipt_sha256`; reading `asec.to_bytes()` from any live 1/1000 preparation would settle the real margin. Because it is fraction-independent, no selection fraction can relieve it; only the nesting of `source`/`household_fields` into each household row (asec_2024_native_population.py:344-345) drives it.", + "The three-cohort household count 169,699 (housing status / universe / arm-rows / asec_H_SEQ figures) is the row-ceilings lane's derivation, not re-measured (gated data); those figures scale linearly with it.", + "Rows keyed on definitions outside this family's module list \u2014 education_assistance_source.py:112/134/156 member_size_bytes, native_household_origin.py:146/156/166 size_bytes, asec_current_money.py:47 HEADER_MAX_BYTES \u2014 carry only this family's enforcement sites and should be merged with the owning family's rows on the definition key.", + "asec_coverage_authentication._BODY_MAX is seeded as a fresh capture budget by eight modules outside this family (current_asec_income_routing_source.py:1319, current_asec_dividend_source.py:353, current_asec_interest_source.py:329, current_asec_retirement_detail_source.py:408, current_social_security_source.py:239, current_survey_health_source.py:270, current_survey_housing.py:294, puf55_survey_recipients.py:156) and _ROW_MAX by puf55_survey_recipients.py:162-164; whether the 45-node property-income nodes reach the dividend/interest/retirement/routing callers is the property family's call chain to confirm.", + "The demographic-source rows are reachable from the nineteen only because the recovered 19-node graph.json carries `demographic_conditioning: true` on the survey_predictors nodes; the 45-node run's graph.json is not under _recovered/pilot-runs/native45-v5/required or /probe, so the same flag on the 45 is inferred from \u00a72 (\"the nineteen above\"), not read.", + "Consumer-side canonical_json re-encodings in graph_asec_prepared (:189, :407-409, :651-652), graph_asec_income (:588-589), graph_composed_asec_binding (:413, :422, :579, :811), graph_composed_asec_measures (:159, :243) and graph_composed_population (:577) were judged not roster-shaped after reading graph_context.py:104-160 (per-entity `columns` list + `rows` + `ordered_ids_sha256`), frame_sampling.py:313 (`selected_household_ids_sha256`) and stacked_spine.py:558-569 (digests only); if any of those documents carries per-row content these checks become rows with cap `none`.", + "asec_checkpoint.py and asec_household_observations.py contain no byte bound (no rows): asec_household_observations._read_numeric_household reads the selected int64 HDF block whole (`values[:, wanted]` :177) with no cap, and both are on the raw-stage assembly path rather than the native 19/45 path.", + "Row-count-only constants without a byte name and without a width product (asec_2024_native_population._MAX_HOUSEHOLDS :39 and _MAX_PERSONS :40, asec_coverage_authentication._MAX_PERSONS :43, asec_person_coverage_source.MAX_PERSONS :32, asec_original_household_weights._MAX_ROWS :42, asec_housing_status/universe.MAX_HOUSEHOLDS as factors) are covered by experiments/native-row-ceilings/census.json and were not given separate rows here; where they multiply a width (demographic, student, engine, income, housing, accounting, arm rows) the product row above names them." + ] + }, + { + "family": "ACS coverage and housing source", + "modules_read": [ + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py (full, 832 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py (full, 749 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py (full, 301 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py (full, 933 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py (full, 874 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_sources.py (full, 309 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py (full, 613 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py (full, 365 lines; no byte bound)", + "packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py (full, 31 lines; no byte bound)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py (full, 203 lines; no byte bound)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py (full, 230 lines)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py (full, 698 lines; reachability note only)", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_1yr_sources.json (full)", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py (lines 15-50, the constants graph_housing_universe enforces)", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py (partial: 500-560, 590-612, 1157-1200, 1430-1460, 1695-1720, 1760-1800, 1880-1990, 2035-2050, def inventory)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py (partial: 1275-1360)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py (partial: 137-175)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py (partial: 949-1010)", + "packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py (partial: 230-300, 355-385)", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_sources.py (partial: 280-310)", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py (partial: 90-105)", + "packages/microcosm-graph/src/microcosm/graph/canonical.py (partial: 1-80, canonical_json)", + "packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py, test_us_acs_housing_source.py, test_us_acs_population_catalogue.py, test_us_acs_pums.py (partial; fixture builders reused by the measurement script, not run under pytest)", + "experiments/native-byte-transports/census-brief.md (full)", + "experiments/native-byte-transports/byte-census-measurements.json (full)", + "experiments/native-byte-transports/byte_census_measure.py (full)", + "experiments/native-row-ceilings/census.json (this family's 38 rows and verdicts)", + "docs/us-native-scale-transport.md and docs/us-native-row-ceilings.md (ACS-related sections)", + "_recovered/pilot-runs/native19-required-20260912/run/financial-artifacts/graph.json and preparation.json (nodes/sources; /catalogues/acs, /native/acs, /producer/acs, /selection counts)" + ], + "rows": [ + { + "constant": "acs_person_coverage_authentication.py:36 MAX_HEADER_BYTES", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_person_coverage_authentication.py:84 default cap of _json_size -> :93 ACSCoverageAuthenticationError(\"CANONICAL_SIZE\")", + "acs_person_coverage_authentication.py:132/:141/:147 default cap of _json_chunks -> :147 \"CANONICAL_SIZE\"", + "acs_person_coverage_authentication.py:151 default cap of _json (no runner-path call relies on the default; every site passes a cap)", + "acs_person_coverage_authentication.py:676-684 native_binding receipt, explicit MAX_HEADER_BYTES", + "acs_person_coverage_authentication.py:804 `raw_header = _json(header, MAX_HEADER_BYTES)` (coverage envelope header)", + "recorded, not enforced: acs_person_coverage_authentication.py:268 producer ceilings" + ], + "refusal_codes": [ + "CANONICAL_SIZE (ACSCoverageAuthenticationError, :93 pre-charge pass and :147 encoder pass)" + ], + "encodes": "Fixed-shape header of the coverage envelope (protocol, producer closure 1,490 B, archives, six-member inventory, field_contract, original_literal_receipt whose row-dependent parts are integers and digests, columns, body_bytes/body_sha256/rows integers, native_consistency) plus the 589-byte UngrantedACSNativeBinding receipt; O(members) and O(columns), never O(rows).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "5,915 (header with the real six-member inventory; 5,138 with the invented three); native_binding receipt 589", + "bytes_at_full_source": "5,915 (identical: row counts enter as integers)", + "measurement_basis": "measured through auth._json at this head by driving the real issuance on invented archives (scratchpad measure_acs_family.py part 2: coverage_header_bytes_invented, coverage_header_bytes_REALISTIC_members with the pilot's 3+3 member names/sizes, native_binding_receipt_bytes)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1275) -> run_atomic_survey_population (:1355; graph_atomic_survey_population.py:137) -> run_authenticated_survey_population (:158; graph_survey_population.py:949) -> survey_population_preparation.prepare_authenticated_survey_population (:1892) -> acs_native.issue_acs_native_coverage(root/'acs', serialnos=acs_keys) (:1962) -> binding.issue_acs_native_coverage (:558) -> coverage.load_authenticated_acs_person_coverage (:614; auth.py:712) -> :804; native_binding at binding.py:627 and auth.py:695 on every verify", + "classification": "stays", + "notes": "Fixed-shape, 177x headroom. Base and head identical (no diff at :36). The default cap only matters for `_json(value)` calls without a cap and there are none on the runner path (auth :597,:676,:760,:804; binding :477; catalogue :205,:234,:443,:452,:486,:516,:590 all pass one)." + }, + { + "constant": "acs_person_coverage_authentication.py:37 MAX_BODY_BYTES [coverage body: one NDJSON line per selected ACS person]", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "HEAD acs_person_coverage_authentication.py:763-766: `if current and len(current) + len(raw) > MAX_BODY_BYTES:` start a new segment; `_require(len(current) + len(raw) <= MAX_BODY_BYTES, \"BODY_SIZE\")` (one line larger than one accumulation)", + "HEAD acs_person_coverage_authentication.py:767 `_require(size + len(raw) <= MAX_ROSTER_BYTES, \"BODY_LIMIT\")` and :772 `_require(len(body) == size, \"BODY_LIMIT\")` (the total, under MAX_ROSTER_BYTES)", + "HEAD acs_person_coverage_authentication.py:163/:178 `_json_roster(segment=MAX_BODY_BYTES)` -> \"CANONICAL_SIZE\" when one token exceeds a segment (used by binding.py:724)", + "BASE acs_person_coverage_authentication.py:661 `_require(size <= MAX_BODY_BYTES, \"BODY_SIZE\")` on the whole body (removed by this branch)", + "recorded: :269 producer ceilings" + ], + "refusal_codes": [ + "BODY_SIZE (ACSCoverageAuthenticationError, :766)", + "BODY_LIMIT (:767, :772)", + "CANONICAL_SIZE (:178 through _json_roster)" + ], + "encodes": "Per-row roster: ACS persons (selected). Each line is `_json([SERIALNO, SPORDER, AGEP, MIL, ESR, MIL_state, ESR_state, source_member, source_row_ordinal], MAX_RECORD_BYTES - 1) + b\"\\n\"` (auth.py:759-762).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "30,894,098 (342,288 lines x measured mean 90.26 B; scaled from the measured full-source mean)", + "bytes_at_full_source": "308,941,698 MEASURED (byte-census-measurements.json acs_person_archive.body_bytes, 3,422,888 lines, longest 102) = 4.60 x 64 MiB, 5 segments; 0.072 x MAX_ROSTER_BYTES", + "measurement_basis": "cited byte-census-measurements.json (whole captured archive through auth._json); cross-checked on invented rows at full-source widths through auth._json at this head: 89 B (observed_code, 1-digit SPORDER, 7-digit ordinal), 100 B (outside_age_universe), 90 B (2-digit SPORDER)", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "64 MiB on the whole body (base auth.py:661 `_require(size <= MAX_BODY_BYTES, \"BODY_SIZE\")`): binds at full source (4.60x), not at 1/10 (0.46x)", + "head_value": "64 MiB per accumulation segment (:763-766); total under MAX_ROSTER_BYTES 4 GiB (:767) = 0.072x at full source", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1275) -> run_atomic_survey_population (:1355; graph_atomic_survey_population.py:137) -> run_authenticated_survey_population (:158; graph_survey_population.py:949) -> survey_population_preparation.prepare_authenticated_survey_population (:1892) -> acs_native.issue_acs_native_coverage(root/'acs', serialnos=acs_keys) (:1962) -> binding.issue_acs_native_coverage (:558) -> coverage.load_authenticated_acs_person_coverage (:614; auth.py:712) -> literal.read_acs_person_coverage_columns (:747) -> body loop :757-771", + "classification": "segment", + "notes": "Already segmented on this branch (brief section 5). The bytes are still one string: `body = b\"\".join(segments)` (:771) and `payload = _MAGIC + len(raw_header).to_bytes(4, \"big\") + raw_header + body` (:805); the payload is retained in `_Owned.literal.payload` and digested at binding.py:515/:720. Consumer-side re-parse with no cap: `AuthenticatedACSPersonCoverage.table` (auth.py:664-671, `rows = [json.loads(row) for row in self._parts()[1].splitlines()]` into a DataFrame) runs at issuance (auth.py:702 via binding.py:522 `_verify`) and at every verify_acs_native_coverage (preparation.py:1789, current_survey_predictors.py:99)." + }, + { + "constant": "acs_person_coverage_authentication.py:47 MAX_ROSTER_BYTES [pre-allocation charge SELECTED_BODY_BUDGET]", + "value": "64 * MAX_BODY_BYTES = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "HEAD acs_person_coverage_authentication.py:475-479 `selected_budget += _body_row_bound(cells, member.filename, row_count)`; `_require(selected_budget <= MAX_ROSTER_BYTES, \"SELECTED_BODY_BUDGET\")`, per selected row, fresh per _inventory call (role household and role person)", + "BASE acs_person_coverage_authentication.py:377-382 `selected_budget += 6 * len(raw) + 1024`; `_require(selected_budget <= MAX_BODY_BYTES, \"SELECTED_BODY_BUDGET\")`", + "callers: acs_native_coverage_binding.py:337 and :351 (_preflight, both roles) and acs_person_coverage_authentication.py:733 and :736 (load_authenticated, both roles): four charges per issuance; acs_population_catalogue.py:155 passes frozenset() so its charge is 0", + "recorded: :270 producer ceilings" + ], + "refusal_codes": [ + "SELECTED_BODY_BUDGET (ACSCoverageAuthenticationError, :479)" + ], + "encodes": "A running upper bound on the coverage body's bytes, per selected row: `len(json.dumps([*cells, member, ordinal], sort_keys, separators, ensure_ascii=True)) + _STATE_BYTES + 1` (:201-213). Person role: roster ACS persons; household role: roster ACS households (cells [SERIALNO, NP]). No document is allocated; the sum guards the DataFrame the low-level reader allocates.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "person role 38,652,211 (scaled); household role ~15,142,084 (ESTIMATE). Base charge: person 1,793,778,148 (26.7 x 64 MiB), household 709,514,338 (10.6x)", + "bytes_at_full_source": "person role 386,523,011 (DERIVED exactly: measured cells_json_bytes 198,264,171 + 54 x 3,422,888 + 3,422,888) = 0.090 x 4 GiB; household role ~151,421,240 (ESTIMATE, 93 B + ordinal digits per household). Base charge: person 17,937,823,406 MEASURED (267 x 64 MiB), household occupied 7,095,161,910 MEASURED (106x)", + "measurement_basis": "auth._body_row_bound on invented rows at full-source widths (113 B per person row; 93 B + ordinal digits per household row) and the measured components in byte-census-measurements.json (cells_json_bytes, old_charge_bytes, old_charge_bytes_occupied); household-role total sums ordinal digits exactly over the two member files (827,133 + 804,836 rows) scaled to the non-vacant share", + "binds_at_tenth": true, + "binds_at_full_source": true, + "base_value": "MAX_BODY_BYTES 64 MiB with charge 6*len(raw)+1024 (base auth.py:377-382): binds at 1/10 (person 26.7x, household 10.6x) and at full (267x, 106x)", + "head_value": "MAX_ROSTER_BYTES 4 GiB with the exact line bound: 0.090x (person), 0.035x (household) at full source", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1275) -> run_atomic_survey_population (:1355) -> run_authenticated_survey_population (graph_survey_population.py:949) -> prepare_authenticated_survey_population (survey_population_preparation.py:1892) -> acs_native.issue_acs_native_coverage (:1962) -> binding._preflight (:585 -> :337/:351 coverage._inventory) and coverage.load_authenticated_acs_person_coverage (:614 -> auth.py:733/:736 _inventory) -> :475-479", + "classification": "stays", + "notes": "Moved on this branch: both the charge formula and its ceiling. At head it is a counter that takes the body's ceiling and needs no transport of its own; 11.1x headroom (person role) and 28x (household role). The charge is at least the body line it guards: SPORDER is charged as the record's string (2 bytes more than the int the line carries) and both states as 27 bytes each (`_STATE_BYTES = 2 * (2 + 24 + 1)`, :187) against the 16/23 the measured file needs." + }, + { + "constant": "acs_person_coverage_authentication.py:47 MAX_ROSTER_BYTES [selected SERIALNO list pre-check, binding.py:578]", + "value": "64 * MAX_BODY_BYTES = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "HEAD acs_native_coverage_binding.py:578 `coverage._json_size(serialnos, coverage.MAX_ROSTER_BYTES)` -> acs_person_coverage_authentication.py:93 \"CANONICAL_SIZE\" (sized, never encoded)", + "BASE acs_native_coverage_binding.py:573-575 `coverage._json(serialnos, min(MAX_EVIDENCE_BYTES, housing.ACS_HU_RECEIPT_MAX_BYTES))` = 1 MiB, materialised" + ], + "refusal_codes": [ + "CANONICAL_SIZE (ACSCoverageAuthenticationError :93, surfaced by binding.py:749 as ACSNativeCoverageBindingError(\"NATIVE_ISSUANCE_REFUSED\") with the cause chained)" + ], + "encodes": "Per-row roster: selected ACS households, the tuple of 13-character SERIALNO keys the runner passes (`serialnos=acs_keys`, survey_population_preparation.py:1946-1963); 16 canonical bytes per key (`\"2024HU0000001\",`) + 1.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "2,450,577 (= 16 x 153,161 + 1)", + "bytes_at_full_source": "24,505,825 (= 16 x 1,531,614 + 1; matches byte-census-measurements.json serialno_list_bytes_full_source) = 0.0057 x 4 GiB", + "measurement_basis": "auth._json_size on invented 13-char keys at 1,000 and 2,000 rows (per entry 16.0, fixed 1.0) at this head; full-source figure cited from byte-census-measurements.json", + "binds_at_tenth": true, + "binds_at_full_source": true, + "base_value": "min(MAX_EVIDENCE_BYTES 2 MiB, ACS_HU_RECEIPT_MAX_BYTES 1 MiB) = 1,048,576 (base binding.py:573-575): binds at 1/10 (2.34x) and at full (23.4x); 65,535 keys was the most it admitted", + "head_value": "MAX_ROSTER_BYTES 4 GiB, sized without encoding: 0.0057x at full source", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial -> ... -> prepare_authenticated_survey_population (survey_population_preparation.py:1892) -> acs_native.issue_acs_native_coverage(root/'acs', serialnos=acs_keys) (:1962) -> acs_native_coverage_binding.py:578 (before any capture or producer work)", + "classification": "segment", + "notes": "The list is the roster part of the issuance receipt (row below); this pre-check takes that receipt's ceiling (`coverage.MAX_ROSTER_BYTES`, comment :574-577). The same bytes are encoded whole and uncapped at housing.py:605 `_sha(_json(chosen))` (separate row)." + }, + { + "constant": "acs_person_coverage_authentication.py:47 MAX_ROSTER_BYTES [raw_person_keys_sha256 digest, binding.py:656-659]", + "value": "64 * MAX_BODY_BYTES = 4,294,967,296 (4 GiB)", + "enforced_at": [ + "HEAD acs_native_coverage_binding.py:656-659 `coverage._json_sha256(sorted(zip(keys.SERIALNO, map(int, keys.SPORDER), strict=True)), coverage.MAX_ROSTER_BYTES)` -> acs_person_coverage_authentication.py:155-160 (_json_sha256) -> :93/:147 \"CANONICAL_SIZE\"", + "BASE acs_native_coverage_binding.py:641-647 `coverage._sha(coverage._json(sorted(...), coverage.MAX_BODY_BYTES))` (materialised, 64 MiB)" + ], + "refusal_codes": [ + "CANONICAL_SIZE (ACSCoverageAuthenticationError :93/:147, surfaced as NATIVE_ISSUANCE_REFUSED at binding.py:749)" + ], + "encodes": "Per-row roster: selected ACS persons, one `[\"2024HU0000001\",1]` pair per person (20 bytes with a 1-digit SPORDER, 21 with two digits).", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "6,846,175 (20 x 342,288 + 414 two-digit rows + 1)", + "bytes_at_full_source": "68,461,900 MEASURED (byte-census-measurements.json acs_person_archive.key_list_bytes; 4,139 two-digit SPORDER rows) = 1.02 x 64 MiB, 0.016 x 4 GiB", + "measurement_basis": "auth._json_size on invented pairs at 1,000 and 2,000 rows (20.0 per pair, fixed 1.0) at this head; full-source figure cited from byte-census-measurements.json", + "binds_at_tenth": false, + "binds_at_full_source": true, + "base_value": "MAX_BODY_BYTES 64 MiB, whole list materialised then hashed (base binding.py:641-647): binds at full (1.020x), not at 1/10 (0.10x)", + "head_value": "MAX_ROSTER_BYTES 4 GiB, digested token-at-a-time (`for part in _json_chunks(value, cap): digest.update(part)`, auth.py:158-159): 0.016x", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial -> ... -> prepare_authenticated_survey_population (survey_population_preparation.py:1892) -> acs_native.issue_acs_native_coverage (:1962) -> receipt construction acs_native_coverage_binding.py:644-721 -> :656-659", + "classification": "segment", + "notes": "Already streamed on this branch (a digest; no whole copy). `keys` comes from coverage._native_roster(prepared.frame) (binding.py:604), which itself streams one `_json(row, MAX_RECORD_BYTES - 1) + b\"\\n\"` per person into a sha256 with no total cap (auth.py:584-598)." + }, + { + "constant": "acs_person_coverage_authentication.py:47 MAX_ROSTER_BYTES [native issuance receipt, binding.py:724 _json_roster]", + "value": "64 * MAX_BODY_BYTES = 4,294,967,296 (4 GiB); segments of MAX_BODY_BYTES 64 MiB", + "enforced_at": [ + "HEAD acs_native_coverage_binding.py:724 `payload = coverage._json_roster(receipt)` -> acs_person_coverage_authentication.py:163-181: :174 `_json_chunks(value, maximum)` (\"CANONICAL_SIZE\" above 4 GiB, :93/:147) and :178 `_require(len(current) + len(part) <= segment, \"CANONICAL_SIZE\")` (one token larger than 64 MiB)", + "BASE acs_native_coverage_binding.py:711 `payload = coverage._json(receipt, MAX_EVIDENCE_BYTES)` (2 MiB, materialised)", + "consumers (no cap): acs_native_coverage_binding.py:388 `json.loads(_owned(self).payload)` (.receipt), :491 and :503 (`_verify_frame`, `_verify`), :411 payload equality; survey_population_preparation.py:1983 `native_receipts = {\"acs\": actual_acs.receipt, ...}` (parsed whole, only the asec side is read by _origins :1157-1162); current_survey_predictors.py:100-101 `json.loads(acs_payload)[\"vintage\"]`; candidate copy binding.py:728-737 (candidate_path is never passed by the runner)" + ], + "refusal_codes": [ + "CANONICAL_SIZE (ACSCoverageAuthenticationError, auth.py:93/:147/:178; surfaced as NATIVE_ISSUANCE_REFUSED at binding.py:749 with the cause chained)" + ], + "encodes": "Per-row roster: selected ACS households. The receipt (binding.py:644-721) embeds `\"requested_serialnos\": serialnos` (:651, 16 B per key) and `\"vacant_serialnos\"` (:660-662, `[]` on the runner path because the catalogue selection draws only non-vacant records) inside a fixed preamble of 20,265 bytes at this head (producer closure 17,532 of it).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "2,470,841 (= 20,265 + 16 x 153,161)", + "bytes_at_full_source": "24,526,089 (= 20,265 + 16 x 1,531,614) = 11.7 x 2 MiB, 0.0057 x 4 GiB; one segment", + "measurement_basis": "the real issuance receipt issued on invented archives at this head, re-encoded through auth._json_roster with the requested list swapped for 1,000 and 2,000 invented keys (per key 16.0; fixed preamble 20,265) -- scratchpad measure_acs_family.py part 2 `issuance_receipt`", + "binds_at_tenth": true, + "binds_at_full_source": true, + "base_value": "MAX_EVIDENCE_BYTES 2 MiB (base binding.py:711): binds at 1/10 (1.18x) and at full (11.7x); 65,535 selected households (4.28% of source) was the most the issuer admitted", + "head_value": "MAX_ROSTER_BYTES 4 GiB in 64 MiB segments: 0.0057x at full source", + "reachable_from": "19", + "reach_chain": "run_atomic_survey_financial (graph_atomic_survey_financial.py:1275) -> run_atomic_survey_population (:1355) -> run_authenticated_survey_population (graph_survey_population.py:949) -> prepare_authenticated_survey_population (survey_population_preparation.py:1892) -> acs_native.issue_acs_native_coverage (:1962) -> acs_native_coverage_binding.py:724; parsed whole again at preparation.py:1983 and, at graph time, current_survey_predictors.py:99-101 (survey_predictors.source_projection node)", + "classification": "segment", + "notes": "Already segmented on this branch; the bytes are one string (`return b\"\".join(segments)`, auth.py:181) retained as `_Owned.payload` and as `AuthenticatedACSNativeCoverage.payload`, and `json.loads` of the whole 24.5 MB string happens on every borrow that reads `.receipt`. Base-time preamble differed slightly (the 1/1000 artifact's producer closure was 17,610 B; this head's is 17,532 B)." + }, + { + "constant": "acs_person_coverage_authentication.py:48 MAX_RECORD_BYTES", + "value": "400_000", + "enforced_at": [ + "acs_person_coverage_authentication.py:290 `cap = MAX_CSV_HEADER_BYTES if first else MAX_RECORD_BYTES` -> :298 and :302 `_require(length < cap, \"CSV_RECORD_BYTES\")` (raw logical CSV record, replayed only above :291's fast path)", + "acs_person_coverage_authentication.py:597 `_json(row, MAX_RECORD_BYTES - 1)` per native-roster row (streamed digest) -> :93/:147 \"CANONICAL_SIZE\"", + "acs_person_coverage_authentication.py:760 `_json([*row, *lineage[...]], MAX_RECORD_BYTES - 1)` per coverage body line -> \"CANONICAL_SIZE\"", + "recorded: :271" + ], + "refusal_codes": [ + "CSV_RECORD_BYTES (ACSCoverageAuthenticationError, :298/:302)", + "CANONICAL_SIZE (:93/:147 for the :597 and :760 sites)" + ], + "encodes": "Width: one raw logical CSV record of an ACS member, and separately one canonical row (native roster row ~40 B; body line <= 102 B).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "longest raw record 877 (measured raw_max); longest body line 102", + "bytes_at_full_source": "877 / 102, unchanged by fraction (per-record)", + "measurement_basis": "cited byte-census-measurements.json (acs_person_archive.raw_max 877, body_row_max 102); the row-count lane measured the longest logical record in either archive as the 1,882-byte person header (first record, under MAX_CSV_HEADER_BYTES)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding._preflight (:585) and coverage.load_authenticated_acs_person_coverage (:614) -> auth._inventory (:426) -> _records (:447) -> _fence (:366/:373) -> :290-302; body line :760; native roster :597 via binding.py:604/:823 and auth.py:724", + "classification": "must-not-move", + "notes": "Per-record encoder width; does not scale with the roster. `_fence` replays only records longer than `min(cap, MAX_TOKEN_BYTES)` (:291), so with 877-byte records the byte loop never runs." + }, + { + "constant": "acs_person_coverage_authentication.py:49 MAX_CSV_HEADER_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "acs_person_coverage_authentication.py:290 (`cap` for the first record of a member) -> :298/:302 \"CSV_RECORD_BYTES\"", + "recorded: :272" + ], + "refusal_codes": [ + "CSV_RECORD_BYTES (ACSCoverageAuthenticationError, :298/:302)" + ], + "encodes": "Width: the CSV header line of one archive member (1,882 B person, 1,570 B household per the row-count lane's measurement).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "1,882", + "bytes_at_full_source": "1,882, unchanged by fraction (upstream header)", + "measurement_basis": "cited experiments/native-row-ceilings/census.json row acs_person_coverage_authentication.py:39 (measured header widths 1,882 / 1,570)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth._inventory (:426) -> _records (:447) -> _fence(record, quoted, first=True) (:366) -> :290", + "classification": "must-not-move", + "notes": "Protects an upstream file's header width." + }, + { + "constant": "acs_person_coverage_authentication.py:50 MAX_TOKEN_BYTES", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "acs_person_coverage_authentication.py:291 fast path `if len(record) <= min(cap, MAX_TOKEN_BYTES): return`", + "acs_person_coverage_authentication.py:307 `_require(token_bytes <= MAX_TOKEN_BYTES, \"CSV_TOKEN_BYTES\")`", + "recorded: :273" + ], + "refusal_codes": [ + "CSV_TOKEN_BYTES (ACSCoverageAuthenticationError, :307)" + ], + "encodes": "Width: one raw CSV field including quoting (longest field in either archive is the 13-byte SERIALNO).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13", + "bytes_at_full_source": "13, unchanged by fraction", + "measurement_basis": "cited row-count census (longest field 13; zero quote characters in either archive); serialno_widths {13: 1,631,969} in byte-census-measurements.json", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth._inventory -> _records -> _fence -> :291/:307", + "classification": "must-not-move", + "notes": "Token/field width." + }, + { + "constant": "acs_person_coverage_authentication.py:51 _SCAN_BLOCK", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_person_coverage_authentication.py:332 `while block := stream.read(_SCAN_BLOCK):` (read block size in _records; no refusal)" + ], + "refusal_codes": [ + "none (a read block, not a check)" + ], + "encodes": "Memory: the block size the byte fence scans; a record spans at most two blocks (docstring :326-329). No document behind it.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "1,048,576 per read, unchanged", + "bytes_at_full_source": "1,048,576 per read, unchanged", + "measurement_basis": "code read only", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth._inventory (:426) -> _records (:447) -> :332", + "classification": "not-a-transport", + "notes": "A memory ceiling with no document behind it; listed for completeness." + }, + { + "constant": "acs_person_coverage_authentication.py:187 _STATE_BYTES", + "value": "2 * (2 + 24 + 1) = 54", + "enforced_at": [ + "acs_person_coverage_authentication.py:211 (`+ _STATE_BYTES` inside _body_row_bound, the per-row pre-allocation charge)" + ], + "refusal_codes": [ + "none directly; contributes to SELECTED_BODY_BUDGET (:479)" + ], + "encodes": "Width: the two derived state cells of one body line, charged at the longest state `value_below_age_universe` (24 chars) plus quotes and separator; the measured file's cells are `observed_code` (16 B) and `outside_age_universe` (23 B).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "54 per row, unchanged", + "bytes_at_full_source": "54 per row, unchanged (184,835,952 summed over 3,422,888 rows inside the charge)", + "measurement_basis": "code read; state vocabulary from acs_person_coverage_columns._field_state (:119-126); measured state counts in byte-census-measurements.json (observed_code 5,704,812; outside_age_universe 1,140,964)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth._inventory (:426) -> _body_row_bound (:475) -> :211", + "classification": "must-not-move", + "notes": "Must stay at least the widest state's encoded width or the charge stops bounding the body (test test_body_row_bound_covers_every_body_line)." + }, + { + "constant": "acs_person_coverage_authentication.py:442 member.file_size read cap", + "value": "`stream.read(min(4096, member.file_size - count + 1))` (non-applicable members) and `count <= member.file_size` after every record (applicable members); member.file_size from the ZIP central directory (largest 1,226,543,489)", + "enforced_at": [ + "acs_person_coverage_authentication.py:442-444 `_require(count <= member.file_size, \"ZIP_MEMBER_GREW\")`", + "acs_person_coverage_authentication.py:448-449 (applicable members, per raw record)", + "acs_person_coverage_authentication.py:497 `_require(count == member.file_size, \"ZIP_MEMBER_SIZE_MISMATCH\")`", + "acs_person_coverage_authentication.py:415 `0 <= member.file_size <= custody._MEMBER_MAX` (\"ZIP_MEMBER_SIZE\", see the _MEMBER_MAX row)" + ], + "refusal_codes": [ + "ZIP_MEMBER_GREW (ACSCoverageAuthenticationError, :444/:449)", + "ZIP_MEMBER_SIZE_MISMATCH (:497)" + ], + "encodes": "Upstream: the declared uncompressed size of each ZIP member (psam_pusa 1,226,543,489; psam_pusb 1,178,924,624; psam_husa 495,752,055; psam_husb 481,732,465; README 103,658).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "3,383,159,949 across the six members, unchanged", + "bytes_at_full_source": "3,383,159,949, unchanged", + "measurement_basis": "cited row-count census (member sizes read from the captured archives' central directory); byte-census-measurements.json raw_bytes 977,481,380 + 2,405,464,349 for the four data members", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding._preflight (:585) and coverage.load_authenticated_acs_person_coverage (:614) -> auth._inventory (:426) -> :440-449, :497", + "classification": "must-not-move", + "notes": "The read(N+1) pattern: a member that grows past its declared size refuses. The 4096 is a chunk size for non-applicable members only." + }, + { + "constant": "acs_person_coverage_authentication.py:805 4-byte header length", + "value": "`len(raw_header).to_bytes(4, \"big\")` -> at most 2**32 - 1 = 4,294,967,295", + "enforced_at": [ + "acs_person_coverage_authentication.py:805 (OverflowError from int.to_bytes above 2**32-1; MAX_HEADER_BYTES 1 MiB at :804 precludes it)", + "consumer acs_person_coverage_authentication.py:654-657 `_parts`: `size = int.from_bytes(self.payload[len(_MAGIC) : start], \"big\")`" + ], + "refusal_codes": [ + "none of this module's; an OverflowError would be caught by :831 and re-raised as SOURCE_RECONSTRUCTION_REFUSED" + ], + "encodes": "Width: the length prefix of the coverage envelope header.", + "shape": "width", + "site_kind": "materialises", + "bytes_at_tenth": "4 (field); header 5,915", + "bytes_at_full_source": "4; header 5,915, unchanged", + "measurement_basis": "code read; header size from part 2 of the measurement script", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> coverage.load_authenticated_acs_person_coverage (:614; auth.py:712) -> :805; read back by `.receipt`/`.table` (:659-671)", + "classification": "must-not-move", + "notes": "Encoder width; the header is fixed-shape (MAX_HEADER_BYTES row)." + }, + { + "constant": "acs_native_coverage_binding.py:28 MAX_ARCHIVE_BYTES", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "acs_native_coverage_binding.py:581 `_require(sum(p[3] for p in pins) <= MAX_ARCHIVE_BYTES, \"ARCHIVE_BUDGET\")`", + "recorded: :274 (producer limits)" + ], + "refusal_codes": [ + "ARCHIVE_BUDGET (ACSNativeCoverageBindingError, :581)" + ], + "encodes": "Upstream: the two pinned archives' declared size_bytes (csv_hus.zip 251,500,587 + csv_pus.zip 602,847,146).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "854,347,733, unchanged", + "bytes_at_full_source": "854,347,733, unchanged (0.099x)", + "measurement_basis": "acs_2024_1yr_sources.json size_bytes, read this session (housing._pins())", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (survey_population_preparation.py:1962) -> :579 housing._pins() -> :581", + "classification": "must-not-move", + "notes": "Protects a real file size; the catalogue has its own twin (_ARCHIVE_BYTES) and survey_population_preparation.py:551 checks the on-disk sizes against that twin." + }, + { + "constant": "acs_native_coverage_binding.py:29 MAX_EXPANDED_BYTES", + "value": "16 * 1024**3 = 17,179,869,184", + "enforced_at": [ + "acs_native_coverage_binding.py:314-319 `expanded += sum(member.file_size for member in members)`; `_require(expanded <= MAX_EXPANDED_BYTES, \"EXPANDED_BUDGET\")` (combined, both archives, before any member is opened)", + "recorded: :275" + ], + "refusal_codes": [ + "EXPANDED_BUDGET (ACSNativeCoverageBindingError, :319)" + ], + "encodes": "Upstream: total declared uncompressed member bytes of both archives.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "3,383,159,949, unchanged", + "bytes_at_full_source": "3,383,159,949, unchanged (0.197x)", + "measurement_basis": "cited row-count census (member sizes from the captured archives' central directories)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (:1962) -> binding._preflight (:585) -> :314-319", + "classification": "must-not-move", + "notes": "Decompression-bomb gate on real upstream sizes." + }, + { + "constant": "acs_native_coverage_binding.py:30 MAX_SOURCE_ROWS", + "value": "6_000_000 (a row count)", + "enforced_at": [ + "acs_native_coverage_binding.py:325-328, :340-343, :354-357 `sum(m[\"rows\"] for m in inventories[role]) <= MAX_SOURCE_ROWS` -> \"SOURCE_ROW_BUDGET\"", + "recorded: :276" + ], + "refusal_codes": [ + "SOURCE_ROW_BUDGET (ACSNativeCoverageBindingError)" + ], + "encodes": "Row count of the upstream archive per role (household 1,631,969; person 3,422,888), not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a (rows: 1,631,969 / 3,422,888, unchanged)", + "bytes_at_full_source": "n/a (rows: 1,631,969 / 3,422,888, unchanged)", + "measurement_basis": "preparation.json /catalogues/acs/counts (households 1631969, people 3422888)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding._preflight (:585) -> :326/:341/:355", + "classification": "not-a-transport", + "notes": "A row count; the row-count lane's upstream-file-size rule applies (census.json row acs_native_coverage_binding.py:30). Same number as acs_person_coverage_columns.MAX_ROWS and acs_population_catalogue._SOURCE_ROWS." + }, + { + "constant": "acs_native_coverage_binding.py:39 MAX_EVIDENCE_BYTES", + "value": "2 * 1024**2 = 2,097,152", + "enforced_at": [ + "HEAD acs_native_coverage_binding.py:477 `return coverage._sha(coverage._json(details, MAX_EVIDENCE_BYTES))` (_frame_sha256) -> auth.py:93/:147 \"CANONICAL_SIZE\"", + "BASE acs_native_coverage_binding.py:573-575 (serialno pre-check, min with ACS_HU_RECEIPT_MAX_BYTES) and :711 (issuance receipt) -- both moved to coverage.MAX_ROSTER_BYTES on this branch", + "recorded: :277" + ], + "refusal_codes": [ + "CANONICAL_SIZE (ACSCoverageAuthenticationError auth.py:93/:147; surfaced as NATIVE_ISSUANCE_REFUSED :749 or NATIVE_VERIFICATION_REFUSED :555)" + ], + "encodes": "Fixed-shape: the frame details document of _frame_sha256 (per entity: index type/dtype, column axis, dtypes list, attrs, allows_duplicate_labels, weight kind/dtype/shape/sha256; strata attrs). O(columns), never O(rows).", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "3,745 (this head, the real column set)", + "bytes_at_full_source": "3,745 (weight_shape digits aside), unchanged (0.0018x)", + "measurement_basis": "captured the len of coverage._json(details, MAX_EVIDENCE_BYTES) by wrapping auth._json around binding._frame_sha256 on the invented frame after issuance (measure_acs_family.py part 2 frame_sha256_details_bytes); the frame's columns are the same 43+18+2+2+1+1 the real builder produces", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "2 MiB fronting the issuance receipt (:711) and, through min(), the 1 MiB serialno pre-check (:573-575): those documents bound at 1/10 and at full (recorded on their own rows)", + "head_value": "2 MiB fronting only the fixed _frame_sha256 details: 560x headroom", + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (:1962) -> :716 `\"exact_frame_sha256\": _frame_sha256(prepared.frame)`, :738 _verify -> _verify_frame (:494); survey_population_preparation.py:1970 and :1774 `acs_native._frame_sha256(frame)` (source seals); every verify_acs_native_coverage (:534 -> :548/:550)", + "classification": "stays", + "notes": "Fixed-shape at head; the binding rows carry the base-time bindings. `_require(not frame._link_tables, \"UNEXPECTED_LINK_TABLES\")` (:476) precedes the encode." + }, + { + "constant": "acs_native_coverage_binding.py:54 _COMPILE_CACHE_MAX_SOURCE_BYTES", + "value": "16 * 1024**2 = 16,777,216", + "enforced_at": [ + "acs_native_coverage_binding.py:107 eligibility (`len(source) <= _COMPILE_CACHE_MAX_SOURCE_BYTES`, bypass only)", + "acs_native_coverage_binding.py:152-157 FIFO eviction while retained source bytes + this source exceed it" + ], + "refusal_codes": [ + "none (silent cache bypass / eviction)" + ], + "encodes": "Python module source bytes retained as cache keys for the producer drift check; not survey data.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "1,844,965 retained across 69 entries after one issuance at this head", + "bytes_at_full_source": "1,844,965, unchanged", + "measurement_basis": "read binding._COMPILE_CACHE after the invented issuance (measure_acs_family.py part 2 compile_cache)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding._producer (:582/:739) -> _live_code (:268) -> _compile_source (:215) -> :107/:152-157", + "classification": "not-a-transport", + "notes": "A memory ceiling with no document behind it. Siblings: _COMPILE_CACHE_MAX_ENTRIES = 128 (:53; :108 eligibility, :153 eviction; 69 entries observed) and _COMPILE_CACHE_MAX_ENTRY_BYTES = 1 MiB (:55; :106 eligibility only; largest source 135,278 B observed). All three: no refusal code." + }, + { + "constant": "acs_person_coverage_columns.py:39 MAX_ROWS", + "value": "6_000_000 (a row count)", + "enforced_at": [ + "acs_person_coverage_columns.py:198-199 `if rows > MAX_ROWS: raise ValueError(\"ACS coverage source row bound exceeded\")`", + "acs_person_coverage_authentication.py:465 `_require(rows <= literal.MAX_ROWS, \"SOURCE_ROWS\")`", + "recorded: acs_person_coverage_authentication.py:264" + ], + "refusal_codes": [ + "ValueError(\"ACS coverage source row bound exceeded\") (columns.py:199)", + "SOURCE_ROWS (ACSCoverageAuthenticationError, auth.py:465)" + ], + "encodes": "Upstream person-file rows (3,422,888) and, at auth.py:465, either role's rows; not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a (rows 3,422,888 person / 1,631,969 household, unchanged)", + "bytes_at_full_source": "n/a (same)", + "measurement_basis": "preparation.json /catalogues/acs/counts", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth._inventory (:465) via binding._preflight and load_authenticated; literal._scan_acs_person_coverage (:198) via auth.py:747 read_acs_person_coverage_columns and acs_population_catalogue.py:300", + "classification": "not-a-transport", + "notes": "Row count wearing no byte name; the row-count lane's upstream-file-size rule applies." + }, + { + "constant": "acs_person_coverage_columns.py:40 MAX_SELECTED_ROWS", + "value": "14_000_000 (a row count)", + "enforced_at": [ + "acs_person_coverage_columns.py:239-240 `if not 0 < len(expected) <= MAX_SELECTED_ROWS: raise ValueError(\"ACS coverage selected person count is outside the bound\")`", + "acs_person_coverage_authentication.py:494 `_require(len(selected) < literal.MAX_SELECTED_ROWS, \"SELECTED_ROWS\")` (strict)", + "acs_person_coverage_authentication.py:517 `_require(0 < len(h) <= len(p) <= literal.MAX_SELECTED_ROWS, \"NATIVE_ROWS\")`", + "acs_native_coverage_binding.py:330-333 (\"NATIVE_ROW_BUDGET\", serialnos-is-None route only) and :348 `_require(expected_rows <= coverage.literal.MAX_SELECTED_ROWS, \"NATIVE_ROW_BUDGET\")`", + "acs_population_catalogue.py:315-318 `count <= min(_BATCH_PEOPLE, literal.MAX_SELECTED_ROWS)` (\"BATCH_PERSON_LIMIT\")", + "recorded: acs_person_coverage_authentication.py:265" + ], + "refusal_codes": [ + "ValueError (columns.py:240)", + "SELECTED_ROWS (auth.py:494)", + "NATIVE_ROWS (auth.py:517)", + "NATIVE_ROW_BUDGET (binding.py:333/:348)", + "BATCH_PERSON_LIMIT (catalogue.py:318)" + ], + "encodes": "Selected ACS person rows (3,422,888 at full source); not bytes.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "n/a (rows 342,288)", + "bytes_at_full_source": "n/a (rows 3,422,888; 4.09x headroom)", + "measurement_basis": "brief section 3 counts", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding._preflight (:348) ; auth._inventory (:494) ; auth._native_roster (:517) via binding.py:604/:823 and auth.py:724 ; literal.read_acs_person_coverage_columns (:239) via auth.py:747", + "classification": "not-a-transport", + "notes": "Lifted 1,000,000 -> 14,000,000 on native-row-ceilings (docs/us-native-row-ceilings.md section 3); unchanged on this branch." + }, + { + "constant": "acs_person_coverage_columns.py:41 MAX_CSV_RECORD_CHARS", + "value": "100_000", + "enforced_at": [ + "acs_person_coverage_columns.py:138 `stream.readline(MAX_CSV_RECORD_CHARS - record_chars + 1)` (read cap) and :140-141 `raise ValueError(\"ACS coverage CSV record character bound exceeded\")`", + "recorded: acs_person_coverage_authentication.py:263" + ], + "refusal_codes": [ + "ValueError(\"ACS coverage CSV record character bound exceeded\") (columns.py:141); inside the owners it surfaces as SOURCE_RECONSTRUCTION_REFUSED (auth.py:832) / CATALOGUE_ISSUANCE_REFUSED (catalogue.py:607)" + ], + "encodes": "Width: decoded characters of one logical CSV record including its line ending (record_chars resets per record, :151).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "1,882 worst case (the header)", + "bytes_at_full_source": "1,882, unchanged", + "measurement_basis": "cited row-count census (longest logical record in either archive)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth.load_authenticated_acs_person_coverage (:747) -> literal.read_acs_person_coverage_columns -> _scan_acs_person_coverage (:256) -> _literal_csv_records (:187) -> :138-141; also auth._decode_record (:383) per raw record, and catalogue._collect (:300)", + "classification": "must-not-move", + "notes": "Per-record width." + }, + { + "constant": "acs_person_coverage_columns.py:84 _json [native_roster_sha256, :292-294] (cap: none)", + "value": "none -- `json.dumps(value, sort_keys=True, separators=(\",\", \":\"), allow_nan=False).encode()` with no limit", + "enforced_at": [ + "acs_person_coverage_columns.py:292-294 `\"native_roster_sha256\": hashlib.sha256(_json(expected.to_dict(\"records\"))).hexdigest()` (whole list materialised, then hashed)" + ], + "refusal_codes": [ + "none" + ], + "encodes": "Per-row roster: selected ACS persons as records `{\"SERIALNO\":\"2024HU0000001\",\"SPORDER\":1}` (41 bytes each with a 1-digit SPORDER).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "14,034,223 (41 x 342,288 + 414 + 1)", + "bytes_at_full_source": "140,342,548 (41 x 3,422,888 + 4,139 + 1)", + "measurement_basis": "literal._json on invented records at 1,000 and 2,000 rows (41.0 per row) at this head; two-digit SPORDER share from byte-census-measurements.json", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding.issue_acs_native_coverage (:558) -> coverage.load_authenticated_acs_person_coverage (:614) -> literal.read_acs_person_coverage_columns (auth.py:747) -> columns.py:292-294; the receipt lands in the coverage header as original_literal_receipt (auth.py:791)", + "classification": "segment", + "notes": "No bound exists at all; the whole 140 MB string exists before the digest. Classified segment because the brief has no 'uncapped' class: a digest over iterencode (the shape auth._json_sha256 already has) would remove the copy. Recorded because the brief asks for every whole-roster encoding; the orchestrator may prefer to carry it as a note." + }, + { + "constant": "acs_person_coverage_columns.py:84 _json [projection_sha256, :295-297] (cap: none)", + "value": "none -- uncapped json.dumps", + "enforced_at": [ + "acs_person_coverage_columns.py:295-297 `\"projection_sha256\": hashlib.sha256(_json(result.to_dict(\"records\"))).hexdigest()`" + ], + "refusal_codes": [ + "none" + ], + "encodes": "Per-row roster: selected ACS persons as 7-key records (SERIALNO, SPORDER, AGEP, MIL, ESR, MIL_state, ESR_state); 129 bytes each at observed_code widths, +7 per outside_age_universe cell.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "~44,155,152 ESTIMATE (129 x 342,288; the measured 1/6 outside_age_universe cells add ~0.8 MB)", + "bytes_at_full_source": "~441,552,552 ESTIMATE (129 x 3,422,888; +~8 MB for the wider state cells)", + "measurement_basis": "literal._json on invented 7-key records at 1,000 and 2,000 rows (129.0 per row) at this head", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> auth.load_authenticated_acs_person_coverage (:747) -> literal.read_acs_person_coverage_columns -> columns.py:295-297", + "classification": "segment", + "notes": "The largest uncapped whole copy in the family (~440 MB at full source), alongside the 3.4M-row `observed` DataFrame (:261) and `result` (:268). Same caveat as the previous row." + }, + { + "constant": "acs_housing_universe_source.py:42 ACS_HU_SOURCE_MAX_BYTES [full lexical projection, whole file]", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "acs_housing_universe_source.py:355-364 streamed sum `selected_bytes += len(_json(row))`; `_require(selected_bytes <= ACS_HU_SOURCE_MAX_BYTES, \"PROJECTION_TOO_LARGE\")` -- every applicable row of one archive, whatever the selection", + "acs_housing_universe_source.py:573-574 `full_bytes = _json(full)`; `_write(private / \"full-projection.json\", full_bytes, ACS_HU_SOURCE_MAX_BYTES)` -> :180 `_require(len(data) <= cap, \"OUTPUT_SIZE\")`", + "acs_population_catalogue.py:497-499 `projection_raw = stream.read(housing.ACS_HU_SOURCE_MAX_BYTES + 1)`; `_require(len(projection_raw) <= housing.ACS_HU_SOURCE_MAX_BYTES, \"PROJECTION_SIZE\")` then `json.loads(projection_raw)` (:508)", + "acs_housing_universe_source.py:203 doubled into the disk demand" + ], + "refusal_codes": [ + "PROJECTION_TOO_LARGE (ACSHousingSourceError, :363)", + "OUTPUT_SIZE (:180)", + "PROJECTION_SIZE (ACSSourceCatalogueError, catalogue.py:499)" + ], + "encodes": "The whole-source lexical projection: one row per household record (1,631,969, incl. vacancies) `[SERIALNO, TYPEHUGQ, NP, TEN, WGTP, PUMA, ST, source_member, source_row_ordinal]` and one per person (3,422,888) `[SERIALNO, SPORDER, PWGTP, source_member, source_row_ordinal]`; identical at every fraction.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "292,484,053 (unchanged: _reconstruct always builds the whole source before selecting)", + "bytes_at_full_source": "292,484,053 MEASURED by the row-count lane on the pilot's full-projection.json = 0.034 x 8 GiB", + "measurement_basis": "cited experiments/native-row-ceilings/census.json row acs_housing_universe_source.py:42; reproduced through housing._json (canonical_json) on invented rows at full-source widths: 65 B + ordinal digits per occupied household row, 62 GQ, 64 vacant, 44 B + digits per person row at 3-digit WGTP/PWGTP -> 292,401,960 materialised (0.03% under the measured file), preamble 270", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> prepare_authenticated_survey_population -> acs_catalogue.issue_acs_source_catalogue (:1939) -> housing._reconstruct(private, paths, pins, None, ...) (catalogue.py:490) -> _archive (:564-565) and :573-574; catalogue reads it back whole at :495-508. Also -> acs_native.issue_acs_native_coverage (:1962) -> housing.prepare_acs_housing_population (binding.py:591) -> _reconstruct (housing.py:833)", + "classification": "stays", + "notes": "Built twice per preparation (catalogue path and native path) and written to each private capture; the catalogue then reads the 292 MB file whole and json.loads it (`del projection_raw` only after parsing, :509). 29x headroom; does not move with the fraction. `_tables(projection)` (:398-494) then materialises pandas frames of the whole source." + }, + { + "constant": "acs_housing_universe_source.py:42 ACS_HU_SOURCE_MAX_BYTES [selected projection]", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "acs_housing_universe_source.py:576-577 `projection = _json(selected)`; `_require(len(projection) <= ACS_HU_SOURCE_MAX_BYTES, \"PROJECTION_TOO_LARGE\")`", + "acs_housing_universe_source.py:641-673 (_run_source readback/publish caps for projection.json: \"RECONSTRUCTION_SIZE\"/\"OUTPUT_SIZE\") -- produce/load_acs_housing_source only, not on a runner path", + "consumers (no cap): acs_native_coverage_binding.py:519-521 `housing.verify_acs_frame_projection(frame, json.loads(prepared.source.projection_json))` on every _verify (issuance :738 and every verify_acs_native_coverage); acs_housing_universe_source.py:834 `json.loads(source.projection_json)` and :852/:889 verify_acs_frame_projection; :551-560 properties re-parse per access; digested at binding.py:513-514/:719", + "graph_acs_housing_universe.py:205 and :309 re-encode it under ACS_HU_PAYLOAD_MAX_BYTES 64 MiB (unreachable; see the _parse_json row)" + ], + "refusal_codes": [ + "PROJECTION_TOO_LARGE (ACSHousingSourceError, :577)" + ], + "encodes": "Per-row roster: selected ACS households (1,531,614 at full source: the full projection minus the 100,355 vacant rows) plus their ACS persons (3,422,888), same row shapes as the full projection, sorted by SERIALNO (:511-528).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "~28,529,285 ESTIMATE (preamble 270 + a tenth of the calibrated selected rows)", + "bytes_at_full_source": "~285,290,419 ESTIMATE (292,484,053 measured full projection minus the vacant household rows at calibrated widths) = 0.033 x 8 GiB", + "measurement_basis": "housing._json on invented rows at 3-digit WGTP/PWGTP widths (the variant that reproduces the measured full projection to 0.03%), vacant rows removed; see measure_acs_family.py part 1 lexical_projection.variants", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (:1962) -> housing.prepare_acs_housing_population (binding.py:591) -> _reconstruct (housing.py:833) -> _select (:575) -> :576-577; parsed whole at binding.py:520 on every verify", + "classification": "stays", + "notes": "30x headroom. Held whole in AuthenticatedACSHousingSource.projection_json and re-parsed into DataFrames (`_tables`, :398-494) at every verification: `json.loads(prepared.source.projection_json)` at binding.py:520." + }, + { + "constant": "acs_housing_universe_source.py:43 ACS_HU_RECEIPT_MAX_BYTES", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_housing_universe_source.py:615-616 `receipt_bytes = _json(receipt)`; `_require(len(receipt_bytes) <= ACS_HU_RECEIPT_MAX_BYTES, \"RECEIPT_TOO_LARGE\")` (source receipt)", + "acs_housing_universe_source.py:232 `_write(private / \"failure.json\", failure, ACS_HU_RECEIPT_MAX_BYTES)` -> :180 \"OUTPUT_SIZE\" (best effort on refusal)", + "acs_housing_universe_source.py:647 and :678 (_run_source candidate/publish caps; not on a runner path)", + "acs_housing_universe_source.py:204 doubled into the disk demand", + "graph_acs_housing_universe.py:202/:218/:294/:308/:310 (`_parse_json(..., ACS_HU_RECEIPT_MAX_BYTES)`; unreachable)", + "BASE acs_native_coverage_binding.py:573-575 `min(MAX_EVIDENCE_BYTES, housing.ACS_HU_RECEIPT_MAX_BYTES)` on the serialno tuple (moved to coverage.MAX_ROSTER_BYTES on this branch)" + ], + "refusal_codes": [ + "RECEIPT_TOO_LARGE (ACSHousingSourceError, :616)", + "OUTPUT_SIZE (:180)", + "JSON_SIZE / CANONICAL_JSON (graph sites, :82/:85)" + ], + "encodes": "Fixed-shape source receipt: parser profile, archives, six-member inventory including each data member's header_hex (3,140 / 3,764 hex chars) and header name list (241 / 286 names), full_counts, selected_counts, digests. No per-row data.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "26,076 (with the real header widths and six members; 3,258 on the invented archives)", + "bytes_at_full_source": "26,076, unchanged (0.025x)", + "measurement_basis": "the real source receipt issued on invented archives at this head, re-encoded through housing._json with the members inventory replaced by six entries carrying synthetic 1,570-byte/241-name and 1,882-byte/286-name headers (measure_acs_family.py part 2 source_receipt_bytes_REALISTIC_headers)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "base_value": "1 MiB, also fronting (through min()) the selected serialno list at base binding.py:573-575: bound at 1/10 (2.34x) and full (23.4x); recorded on that row", + "head_value": "1 MiB fronting only fixed-shape receipts", + "reachable_from": "19", + "reach_chain": "... -> housing._reconstruct (via catalogue.py:490 and binding.py:591 -> housing.py:833) -> :615-616; parsed by binding.py:618 `json.loads(prepared.source.receipt_json)` and digested :511/:718", + "classification": "stays", + "notes": "40x headroom. The graph's prepared-receipt parse at graph_acs_housing_universe.py:202 shares this constant and would refuse the prepared receipt above 65,535 selected households (see the housing.py:913 and _parse_json rows) -- unreachable today." + }, + { + "constant": "acs_housing_universe_source.py:44 _CHUNK", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_housing_universe_source.py:142 `stream.read(min(_CHUNK, cap - count + 1))` (_copy) with :144 \"FILE_GREW\"", + "acs_housing_universe_source.py:168 `stream.read(min(_CHUNK, size - count + 1))` (_persisted_sha) with :170 \"SNAPSHOT_GREW\"", + "acs_housing_universe_source.py:318 `stream.read(min(_CHUNK, member.file_size - count + 1))` (non-applicable members) with :323 \"ZIP_MEMBER_GREW\"" + ], + "refusal_codes": [ + "none from _CHUNK itself; the caps are the pinned sizes (see the size_bytes row) and member sizes" + ], + "encodes": "Memory: the read chunk of the read(N+1) loops; no document.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "1,048,576 per read, unchanged", + "bytes_at_full_source": "1,048,576 per read, unchanged", + "measurement_basis": "code read", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._capture (:190) -> _copy (:213) / _persisted_sha (:215); _archive (:311-318)", + "classification": "not-a-transport", + "notes": "Chunk size only." + }, + { + "constant": "acs_housing_universe_source.py:45 _MEMBER_MAX", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "acs_housing_universe_source.py:293 `_require(0 <= member.file_size <= _MEMBER_MAX, \"ZIP_MEMBER_SIZE\")`", + "acs_person_coverage_authentication.py:415 `_require(0 <= member.file_size <= custody._MEMBER_MAX, \"ZIP_MEMBER_SIZE\")`", + "recorded: acs_person_coverage_authentication.py:274" + ], + "refusal_codes": [ + "ZIP_MEMBER_SIZE (ACSHousingSourceError :293; ACSCoverageAuthenticationError auth.py:415)" + ], + "encodes": "Upstream: one ZIP member's declared uncompressed size (largest psam_pusa.csv 1,226,543,489).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "1,226,543,489, unchanged", + "bytes_at_full_source": "1,226,543,489, unchanged (0.143x)", + "measurement_basis": "cited row-count census (central directory of the captured archives)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._archive (:262, via _reconstruct) and auth._members (:389, via _inventory from binding._preflight/load_authenticated/catalogue._preflight :148)", + "classification": "must-not-move", + "notes": "Real upstream size." + }, + { + "constant": "acs_housing_universe_source.py:46 _EXPANDED_MAX", + "value": "16 * 1024**3 = 17,179,869,184", + "enforced_at": [ + "acs_housing_universe_source.py:294-295 `total_bytes += member.file_size`; `_require(total_bytes <= _EXPANDED_MAX, \"ZIP_EXPANDED_SIZE\")` (per archive)", + "acs_person_coverage_authentication.py:416-417 (per archive)", + "recorded: acs_person_coverage_authentication.py:275" + ], + "refusal_codes": [ + "ZIP_EXPANDED_SIZE (ACSHousingSourceError :295; ACSCoverageAuthenticationError auth.py:417)" + ], + "encodes": "Upstream: one archive's total declared member bytes (person 2,405,571,771; household 977,588,178).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "2,405,571,771 (larger archive), unchanged", + "bytes_at_full_source": "2,405,571,771, unchanged (0.14x)", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._archive (:262) and auth._members (:389)", + "classification": "must-not-move", + "notes": "Real upstream size; the binding's MAX_EXPANDED_BYTES and the catalogue's _EXPANDED_BYTES are the combined twins." + }, + { + "constant": "acs_housing_universe_source.py:47 _MEMBER_COUNT_MAX", + "value": "64 (a count)", + "enforced_at": [ + "acs_housing_universe_source.py:269 `_require(0 < len(members) <= _MEMBER_COUNT_MAX, \"ZIP_MEMBER_COUNT\")`", + "acs_person_coverage_authentication.py:395 (same, \"ZIP_MEMBER_COUNT\")", + "recorded: acs_person_coverage_authentication.py:276" + ], + "refusal_codes": [ + "ZIP_MEMBER_COUNT (ACSHousingSourceError :269; ACSCoverageAuthenticationError auth.py:395)" + ], + "encodes": "Count of ZIP central-directory entries per archive (3), not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a (3 members per archive, unchanged)", + "bytes_at_full_source": "n/a (3, unchanged)", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._archive (:262) and auth._members (:389)", + "classification": "not-a-transport", + "notes": "A count wearing the _MAX_ name; the row-count lane's rule applies." + }, + { + "constant": "acs_housing_universe_source.py:48 _RECORD_MAX", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_housing_universe_source.py:239 `_require(0 < len(raw) <= _RECORD_MAX, \"CSV_RECORD_SIZE\")`", + "acs_housing_universe_source.py:313-316 `stream.readline(min(_RECORD_MAX + 1, member.file_size - count + 1))` (read cap)", + "recorded in the receipt: :596 parser_profile.record_max_bytes" + ], + "refusal_codes": [ + "CSV_RECORD_SIZE (ACSHousingSourceError, :239)" + ], + "encodes": "Width: one physical CSV record including terminator (longest 1,882).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "1,882", + "bytes_at_full_source": "1,882, unchanged", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._reconstruct -> _archive (:564-565) -> :313-316 / _csv_record (:327 -> :239)", + "classification": "must-not-move", + "notes": "Per-record width; also recorded in the receipt's parser_profile (:589-597), so moving it changes issued receipts." + }, + { + "constant": "acs_housing_universe_source.py:49 _FIELD_MAX", + "value": "64 * 1024 = 65,536", + "enforced_at": [ + "acs_housing_universe_source.py:256-258 `_require(all(len(v.encode(\"utf-8\")) <= _FIELD_MAX for v in values), \"CSV_FIELD_SIZE\")`", + "recorded in the receipt: :595 parser_profile.field_max_bytes" + ], + "refusal_codes": [ + "CSV_FIELD_SIZE (ACSHousingSourceError, :257)" + ], + "encodes": "Width: one parsed CSV field (longest 13).", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13", + "bytes_at_full_source": "13, unchanged", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._archive -> _csv_record (:327) -> :256-258", + "classification": "must-not-move", + "notes": "Field width; part of the receipt's parser_profile." + }, + { + "constant": "acs_2024_1yr_sources.json size_bytes (pinned archive sizes; loaded by acs_sources.py:88 load_acs_source_manifest -> acs_housing_universe_source.py:101-107 _pins)", + "value": "csv_hus.zip 251,500,587; csv_pus.zip 602,847,146 (sum 854,347,733)", + "enforced_at": [ + "acs_housing_universe_source.py:213 `_copy(source / name, target, size, exact_size=size)` -> :137 \"FILE_TOO_LARGE\", :139 \"SOURCE_SIZE\", :142-144 read(N+1) \"FILE_GREW\", :149-154 \"FILE_CHANGED\"; :214 \"SOURCE_SHA256\"", + "acs_housing_universe_source.py:215 `_persisted_sha(target, size)` -> :165-167 \"SNAPSHOT_SIZE\", :168-170 read(N+1) \"SNAPSHOT_GREW\", :172-175 \"SNAPSHOT_CHANGED\"; :218 \"SOURCE_AUTHORITY_CHANGED\"", + "acs_person_coverage_authentication.py:817-821 `custody._persisted_sha(paths[role], size) == digest` -> \"SNAPSHOT_CHANGED\"", + "acs_native_coverage_binding.py:447 `housing._persisted_sha(path, size) == digest` -> \"SOURCE_CHANGED\" (every verify, every snapshot path)", + "acs_population_catalogue.py:201 -> \"SOURCE_CHANGED\"", + "acs_sources.py:198-202 (download exceeds pinned size), :273-276 (_file_matches), :300-304 (_verify_download), :266-269 (manifest validation) -- fetch path, not on a runner path", + "survey_population_preparation.py:551 compares the on-disk sizes to acs_catalogue._ARCHIVE_BYTES" + ], + "refusal_codes": [ + "FILE_TOO_LARGE, SOURCE_SIZE, FILE_GREW, FILE_CHANGED, SOURCE_SHA256, SNAPSHOT_SIZE, SNAPSHOT_GREW, SNAPSHOT_CHANGED, SOURCE_AUTHORITY_CHANGED (ACSHousingSourceError)", + "SNAPSHOT_CHANGED (ACSCoverageAuthenticationError auth.py:817-821)", + "SOURCE_CHANGED (ACSNativeCoverageBindingError binding.py:447; ACSSourceCatalogueError catalogue.py:201)", + "ValueError (acs_sources.py fetch path)" + ], + "encodes": "Upstream: the two Census archives' exact byte sizes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "854,347,733, unchanged", + "bytes_at_full_source": "854,347,733, unchanged", + "measurement_basis": "manifest read this session; equal to the sum the row-count lane confirmed on disk", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._capture (catalogue.py:487, binding.py:583, housing.py:832 via prepare, auth.py:726 via load: four captures per preparation) -> :211-215; re-hashed on every verify (binding._verify_sources :442-448 over every snapshot)", + "classification": "must-not-move", + "notes": "Exact-size read(N+1) caps on real files; every capture copies both archives (854 MB) into a fresh private directory." + }, + { + "constant": "acs_housing_universe_source.py:201-207 disk demand", + "value": "sum(pins) + 2 * ACS_HU_SOURCE_MAX_BYTES + 2 * ACS_HU_RECEIPT_MAX_BYTES + 1024**3 = 854,347,733 + 17,179,869,184 + 2,097,152 + 1,073,741,824 = 19,110,055,893 (17.8 GiB free)", + "enforced_at": [ + "acs_housing_universe_source.py:207 `_require(shutil.disk_usage(root).free >= required, \"INSUFFICIENT_DISK\")`" + ], + "refusal_codes": [ + "INSUFFICIENT_DISK (ACSHousingSourceError, :207)" + ], + "encodes": "An operational free-disk ceiling for one private capture; no document (comment :198-200: 'This bound is operational, not a source-size prediction').", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "19,110,055,893 required, unchanged", + "bytes_at_full_source": "19,110,055,893 required, unchanged", + "measurement_basis": "computed from the constants at this head (measure_acs_family.py part 1 disk_demand)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._capture (:190) at each of the four captures per preparation -> :201-207", + "classification": "not-a-transport", + "notes": "Tests patch `shutil.disk_usage` (test_us_acs_housing_source.py:514-520); the measurement script did the same. Not a byte transport; listed because it is an inline byte literal." + }, + { + "constant": "acs_housing_universe_source.py:386-395 `len(token) <= 5` (_integer)", + "value": "5 ASCII digits for TYPEHUGQ, NP, WGTP, TEN, SPORDER, PWGTP, PUMA tokens; also :456 `len(v) <= 2` (SPORDER) and :473-474 `len(v) == 5` (PUMA)", + "enforced_at": [ + "acs_housing_universe_source.py:387-392 `_require(type(token) is str and re.fullmatch(r\"[0-9]+\", token, flags=re.ASCII) is not None and len(token) <= 5, label)`", + "acs_housing_universe_source.py:441-464 (per field, inside _tables)", + "acs_housing_universe_source.py:473-475 PUMA" + ], + "refusal_codes": [ + "the field label as the code: TYPEHUGQ, NP, WGTP, TEN, SPORDER, PWGTP, PUMA (ACSHousingSourceError)" + ], + "encodes": "Width: one numeric source token.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "<= 5 per token, unchanged", + "bytes_at_full_source": "<= 5, unchanged", + "measurement_basis": "code read", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._tables (via _select :498, verify_acs_frame_projection :709 on every verify, properties :551-560)", + "classification": "must-not-move", + "notes": "Fixed-width field checks. SERIALNO's 13-character width is checked separately (acs_pums row)." + }, + { + "constant": "acs_housing_universe_source.py:605 `_sha(_json(chosen))` (cap: none)", + "value": "none -- canonical_json of the whole selected-serialno tuple, then sha256", + "enforced_at": [ + "acs_housing_universe_source.py:605 `\"selected_serialnos_sha256\": _sha(_json(chosen))`", + "consumer re-encodings: graph_acs_housing_universe.py:174 `receipt[\"selected_serialnos_sha256\"] == source._sha(canonical_json(chosen))` and :609 (unreachable)" + ], + "refusal_codes": [ + "none" + ], + "encodes": "Per-row roster: `chosen` = sorted selected ACS household SERIALNOs (binding path: 1,531,614 at full source) or every household record including vacancies (catalogue path, serialnos=None: 1,631,969 at every fraction); 16 B per key + 1.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "2,450,577 (binding path); 26,111,505 (catalogue path, unchanged)", + "bytes_at_full_source": "24,505,825 (binding path); 26,111,505 (catalogue path)", + "measurement_basis": "housing._json (canonical_json) gives the same 16 B per 13-char key as auth._json_size (measured at 1,000/2,000 rows); counts from brief section 3 and preparation.json", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing._reconstruct (:563) via catalogue.py:490 (serialnos None) and via binding.py:591 -> prepare_acs_housing_population -> housing.py:833 (serialnos=acs_keys) -> :605", + "classification": "segment", + "notes": "Uncapped whole copy of the same bytes the binding sizes without encoding at :578; a streamed digest would remove it. Same caveat as the other uncapped rows." + }, + { + "constant": "acs_housing_universe_source.py:913 `_json(receipt)` prepared receipt (cap: none)", + "value": "none -- `PreparedACSHousingPopulation(frame, source, _json(receipt))`, canonical_json with no limit", + "enforced_at": [ + "acs_housing_universe_source.py:891-913 (producer; no cap)", + "consumers: acs_native_coverage_binding.py:481 `json.loads(prepared.receipt_json)[\"frame_sha256\"]` (_verify_prepared_frame, at :594 and inside every _verify_frame :497), :509 `coverage._sha(prepared.receipt_json)` (EVIDENCE_CHANGED), :600 `prepared.receipt[\"requested_serialnos\"] == (None if serialnos is None else list(serialnos))` (parses whole and compares the list), :717; acs_housing_universe_source.py:823-824 `.receipt` property; survey_population_preparation.py:1710 retains `acs.prepared.receipt_json` in the nested seals", + "graph_acs_housing_universe.py:202 `source._parse_json(prepared.receipt_json, source.ACS_HU_RECEIPT_MAX_BYTES)` (1 MiB; JSON_SIZE / CANONICAL_JSON) -- the only capped consumer, unreachable" + ], + "refusal_codes": [ + "none on the reachable path; JSON_SIZE / CANONICAL_JSON (ACSHousingSourceError :82/:85) at the unreachable graph site" + ], + "encodes": "Per-row roster: selected ACS households through `\"requested_serialnos\": serialnos` (:904), inside a fixed preamble of 2,518 bytes at this head (dtype_transitions list of 10 columns, entity_rows, digests).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "2,453,094 (= 2,518 + 16 x 153,161)", + "bytes_at_full_source": "24,508,342 (= 2,518 + 16 x 1,531,614)", + "measurement_basis": "the real prepared receipt issued on invented archives at this head, re-encoded through housing._json with requested_serialnos swapped for 1,000 and 2,000 keys (per key 16.0; preamble 2,518) -- measure_acs_family.py part 2 prepared_receipt", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (survey_population_preparation.py:1962) -> housing.prepare_acs_housing_population (binding.py:591) -> housing.py:913; parsed whole at binding.py:481 on every verify and at :600 at issuance", + "classification": "segment", + "notes": "The document the brief says the row-count lane missed. No bound at head; a whole 24.5 MB string parsed on every verify. If the ACS housing graph were run on a selection, graph_acs_housing_universe.py:202 would refuse it above 65,535 keys (2.34x over 1 MiB at 1/10)." + }, + { + "constant": "acs_pums.py:55 MAX_EXACT_HOUSEHOLDS", + "value": "7_000_000 (a row count)", + "enforced_at": [ + "acs_pums.py:192-208 `snapshot_serialnos`: `0 < len(serialnos) <= MAX_EXACT_HOUSEHOLDS` else ValueError(\"ACS exact selection requires bounded unique raw native keys.\")", + "callers: acs_native_coverage_binding.py:568, acs_housing_universe_source.py:830, acs_pums.py:224", + "recorded: acs_native_coverage_binding.py:278" + ], + "refusal_codes": [ + "ValueError (acs_pums.py:206-208); surfaces as NATIVE_ISSUANCE_REFUSED (binding.py:749) / FRAME_PREPARATION_REFUSED (housing.py:925)" + ], + "encodes": "Count of requested ACS household keys (1,531,614 at full source), not bytes.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "n/a (153,161 keys)", + "bytes_at_full_source": "n/a (1,531,614 keys; 4.57x headroom)", + "measurement_basis": "brief section 3", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_native.issue_acs_native_coverage (:1962) -> binding.py:568 -> acs_pums.py:192", + "classification": "not-a-transport", + "notes": "Lifted on native-row-ceilings; unchanged here." + }, + { + "constant": "acs_pums.py:56 MAX_EXACT_PERSON_ROWS", + "value": "14_000_000 (a row count)", + "enforced_at": [ + "acs_pums.py:270-273 `if household.NP.sum() > MAX_EXACT_PERSON_ROWS: raise ValueError(\"ACS selected complete roster exceeds native person budget.\")` (serialnos route only)", + "recorded: acs_native_coverage_binding.py:279" + ], + "refusal_codes": [ + "ValueError (acs_pums.py:271-273); surfaces as FRAME_PREPARATION_REFUSED / NATIVE_ISSUANCE_REFUSED" + ], + "encodes": "NP summed over the selected households (3,422,888 at full source), not bytes.", + "shape": "roster", + "site_kind": "streams", + "bytes_at_tenth": "n/a (342,288)", + "bytes_at_full_source": "n/a (3,422,888; 4.09x headroom)", + "measurement_basis": "brief section 3", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing.prepare_acs_housing_population (binding.py:591) -> build_acs_pums_unit_frame (housing.py:836) -> load_acs_pums_tables (acs_pums.py:333) -> :270", + "classification": "not-a-transport", + "notes": "Row count." + }, + { + "constant": "acs_pums.py:197 SERIALNO width 13", + "value": "`len(key) != 13` refuses; equivalently the regex `2024(?:HU|GQ)[0-9]{7}` at acs_person_coverage_columns.py:104, acs_housing_universe_source.py:432 and acs_population_catalogue.py:264", + "enforced_at": [ + "acs_pums.py:192-208 snapshot_serialnos (ValueError)", + "acs_person_coverage_columns.py:101-107 _keys (ValueError \"ACS coverage native serial is not a 2024 HU/GQ identity\")", + "acs_housing_universe_source.py:429-437 (\"SERIALNO\")", + "acs_population_catalogue.py:262-268 (\"LITERAL_PERSON_KEY\")" + ], + "refusal_codes": [ + "ValueError (acs_pums.py, columns.py)", + "SERIALNO (ACSHousingSourceError)", + "LITERAL_PERSON_KEY (ACSSourceCatalogueError)" + ], + "encodes": "Width: every household key is exactly 13 characters (measured serialno_widths {13: 1,631,969}); this fixes the 16 canonical bytes per key that every serialno roster in this family carries.", + "shape": "width", + "site_kind": "streams", + "bytes_at_tenth": "13 per key, unchanged", + "bytes_at_full_source": "13 per key, unchanged", + "measurement_basis": "byte-census-measurements.json acs_household_archive.serialno_widths", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> binding.py:568 snapshot_serialnos; literal._keys via auth.py:579 and columns.py:238/:262; housing._tables; catalogue._collect", + "classification": "must-not-move", + "notes": "Fixed-width field." + }, + { + "constant": "acs_pums.py:48 DEFAULT_CHUNKSIZE", + "value": "100_000 rows (also acs_person_coverage_columns.py:236 `0 < chunksize <= 100_000` and acs_person_coverage_authentication.py:750 `chunksize=min(1000, len(keys))`)", + "enforced_at": [ + "acs_pums.py:229-230 `if chunksize <= 0: raise ValueError`; :446-452 pd.read_csv(chunksize=...)", + "acs_person_coverage_columns.py:236-237 ValueError(\"ACS coverage chunksize must be in 1..100000\"); :252-253 batch flush", + "acs_person_coverage_authentication.py:750" + ], + "refusal_codes": [ + "ValueError (acs_pums.py:230; columns.py:237)" + ], + "encodes": "Row batch sizes for the pandas reader and the literal reader; not byte bounds.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a", + "bytes_at_full_source": "n/a", + "measurement_basis": "code read", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> housing.prepare_acs_housing_population -> build_acs_pums_unit_frame (default chunksize); auth.py:747-751 -> read_acs_person_coverage_columns(chunksize=min(1000, len(keys)))", + "classification": "not-a-transport", + "notes": "Batch sizes; the selected tables still materialise whole (acs_pums.py:482 `pd.concat(pieces)`, columns.py:261)." + }, + { + "constant": "acs_sources.py:49 _DEFAULT_CHUNK_BYTES", + "value": "1024 * 1024 = 1,048,576", + "enforced_at": [ + "acs_sources.py:146 default chunk_bytes of fetch_acs_pums_sources; :151 `if chunk_bytes <= 0: raise ValueError`; :196 `_iter_chunks(response, chunk_bytes)`", + "acs_sources.py:283 `_iter_chunks(source, _DEFAULT_CHUNK_BYTES)` (_file_identity)" + ], + "refusal_codes": [ + "ValueError(\"chunk_bytes must be positive.\") (:152)" + ], + "encodes": "Download / hashing chunk; no document.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a", + "bytes_at_full_source": "n/a", + "measurement_basis": "code read", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "fetch_acs_pums_sources is called from no src module (grep); the runners reach acs_sources only through load_acs_source_manifest (housing.py:106 _pins), which reads the packaged manifest", + "classification": "not-a-transport", + "notes": "The fetch path's size checks (:198-202, :273-276, :300-304) are enforcement sites of the manifest size_bytes row, also unreachable from the runners." + }, + { + "constant": "acs_population_catalogue.py:25 _ARCHIVE_BYTES", + "value": "8 * 1024**3 = 8,589,934,592", + "enforced_at": [ + "acs_population_catalogue.py:484 `_require(sum(p[3] for p in pins) <= _ARCHIVE_BYTES, \"ARCHIVE_LIMIT\")`", + "survey_population_preparation.py:513-514 (_file_limits for acs/csv_hus.zip and acs/csv_pus.zip) and :546-553 `sum(lstat().st_size ...) <= acs_catalogue._ARCHIVE_BYTES` -> \"ARCHIVE_LIMIT\"", + "acs_population_catalogue.py:121-128 identity-checked against the literal 8*1024**3 (\"LIMITS\")" + ], + "refusal_codes": [ + "ARCHIVE_LIMIT (ACSSourceCatalogueError :484; SurveyPopulationPreparationError preparation.py:552)", + "LIMITS (:127)" + ], + "encodes": "Upstream: the two pinned archives' sizes (854,347,733).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "854,347,733, unchanged", + "bytes_at_full_source": "854,347,733, unchanged", + "measurement_basis": "manifest", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> prepare_authenticated_survey_population -> _source_files (:1929) -> :546-553; -> acs_catalogue.issue_acs_source_catalogue (:1939) -> :484", + "classification": "must-not-move", + "notes": "Twin of binding.MAX_ARCHIVE_BYTES; pinned by the LIMITS check (:121-128) so it cannot be raised without editing the literal tuple." + }, + { + "constant": "acs_population_catalogue.py:26 _EXPANDED_BYTES", + "value": "16 * 1024**3 = 17,179,869,184", + "enforced_at": [ + "acs_population_catalogue.py:151-152 `expanded += sum(member.file_size for member in members)`; `_require(expanded <= _EXPANDED_BYTES, \"EXPANDED_LIMIT\")` (combined)", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "EXPANDED_LIMIT (ACSSourceCatalogueError, :152)" + ], + "encodes": "Upstream: combined declared member bytes (3,383,159,949).", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "3,383,159,949, unchanged", + "bytes_at_full_source": "3,383,159,949, unchanged", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_catalogue.issue_acs_source_catalogue (:1939) -> _preflight (:489) -> :151-152", + "classification": "must-not-move", + "notes": "Real upstream size." + }, + { + "constant": "acs_population_catalogue.py:27 _MEMBERS", + "value": "64 (a count)", + "enforced_at": [ + "acs_population_catalogue.py:150 `_require(len(members) <= _MEMBERS, \"MEMBER_LIMIT\")`", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "MEMBER_LIMIT (ACSSourceCatalogueError, :150)" + ], + "encodes": "Count of ZIP members per archive (3), not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a (3)", + "bytes_at_full_source": "n/a (3)", + "measurement_basis": "cited row-count census", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> catalogue._preflight (:489) -> :150", + "classification": "not-a-transport", + "notes": "Count." + }, + { + "constant": "acs_population_catalogue.py:28 _SOURCE_ROWS", + "value": "6_000_000 (a row count)", + "enforced_at": [ + "acs_population_catalogue.py:157-160 `sum(member[\"rows\"] for member in inventory) <= _SOURCE_ROWS` -> \"SOURCE_ROW_LIMIT\"", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "SOURCE_ROW_LIMIT (ACSSourceCatalogueError, :159)" + ], + "encodes": "Upstream rows per role (1,631,969 / 3,422,888), not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a", + "bytes_at_full_source": "n/a", + "measurement_basis": "preparation.json /catalogues/acs/counts", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> catalogue._preflight (:489) -> :155-160 (coverage._inventory with frozenset())", + "classification": "not-a-transport", + "notes": "Row count; upstream-file-size rule." + }, + { + "constant": "acs_population_catalogue.py:29 _BATCH_PEOPLE", + "value": "100_000 (a count)", + "enforced_at": [ + "acs_population_catalogue.py:255 `_require(count <= _BATCH_PEOPLE, \"HOUSEHOLD_EXCEEDS_BATCH\")` (one household's NP)", + "acs_population_catalogue.py:315-318 `count <= min(_BATCH_PEOPLE, literal.MAX_SELECTED_ROWS)` -> \"BATCH_PERSON_LIMIT\"", + "acs_population_catalogue.py:332 batching threshold (no refusal)", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "HOUSEHOLD_EXCEEDS_BATCH (:255)", + "BATCH_PERSON_LIMIT (:318)" + ], + "encodes": "Persons per assembly batch (the pilot recorded literal_batches 35 for 3,422,888 people), not bytes.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "n/a", + "bytes_at_full_source": "n/a", + "measurement_basis": "preparation.json /catalogues/acs/counts/literal_batches", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> issue_acs_source_catalogue -> _collect (:510) -> :255/:316/:332", + "classification": "not-a-transport", + "notes": "Count." + }, + { + "constant": "acs_population_catalogue.py:30 _CATALOGUE_BYTES", + "value": "2 * 1024**3 = 2,147,483,648", + "enforced_at": [ + "acs_population_catalogue.py:204-209 `_charge`: `raw = native.coverage._json(record, max(0, _CATALOGUE_BYTES - size - 1)) + b\"\\n\"` (per-record cap shrinking with the running total -> auth.py:93/:147 \"CANONICAL_SIZE\") and `_require(size <= _CATALOGUE_BYTES, \"CATALOGUE_BYTE_LIMIT\")`", + "acs_population_catalogue.py:230-238 `reserve` (prospective charge of every household and person before retention; same two codes)", + "acs_population_catalogue.py:339 `_require(size == prospective_size, \"CATALOGUE_BYTE_ACCOUNTING\")`", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "CATALOGUE_BYTE_LIMIT (ACSSourceCatalogueError, :207/:238)", + "CANONICAL_SIZE (ACSCoverageAuthenticationError at the _json caps, surfaced as CATALOGUE_ISSUANCE_REFUSED :607)", + "CATALOGUE_BYTE_ACCOUNTING (:339)" + ], + "encodes": "The canonical record stream of the whole source: one household tuple per record (1,631,969, incl. vacancies) with its nested person tuples (3,422,888); identical at every fraction (catalogue.py:490 passes serialnos=None). Digested, never joined.", + "shape": "upstream", + "site_kind": "streams", + "bytes_at_tenth": "374,174,650, unchanged", + "bytes_at_full_source": "374,174,650 MEASURED (preparation.json /catalogues/acs/counts/canonical_record_bytes) = 0.174 x 2 GiB", + "measurement_basis": "cited from the pilot receipt; reproduced through auth._json on invented records at full-source widths (58 B household record without members, 81 B per nested person at 3-digit PWGTP / 7-digit ordinal -> 371,908,130, 0.6% under the measured total)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_catalogue.issue_acs_source_catalogue (survey_population_preparation.py:1939) -> _collect (:510) -> reserve (:256/:295) and consume -> _charge (:325)", + "classification": "stays", + "notes": "5.7x headroom; does not move with the fraction. The bytes are streamed, but the records themselves are retained whole as Python tuples in `_Owned.records`/`vacancies` (:574-577) with no byte bound, and `.households` (:408-409) rebuilds 1.63M AcsHousehold dataclasses on each access (preparation.py:1944 `acs.households`)." + }, + { + "constant": "acs_population_catalogue.py:31 _RECEIPT_BYTES", + "value": "1024**2 = 1,048,576", + "enforced_at": [ + "acs_population_catalogue.py:516-565 `receipt = native.coverage._json({...}, _RECEIPT_BYTES)` (catalogue receipt) -> auth.py:93/:147 \"CANONICAL_SIZE\"", + "acs_population_catalogue.py:443, :452, :486, :590 `native.coverage._json(_producer(), _RECEIPT_BYTES)` (producer document; compared for PRODUCER_CHANGED)", + "acs_population_catalogue.py:474-478 `len(candidate) <= _RECEIPT_BYTES` -> \"CANDIDATE_TYPE_SIZE\" (no candidate on the runner path)", + ":121-128 LIMITS identity" + ], + "refusal_codes": [ + "CANONICAL_SIZE (surfaced as CATALOGUE_ISSUANCE_REFUSED :607 / CATALOGUE_VERIFICATION_REFUSED :461)", + "CANDIDATE_TYPE_SIZE (:478)", + "PRODUCER_CHANGED (:444/:453/:591)" + ], + "encodes": "Fixed-shape: the catalogue receipt (producer closure 17,808 B, archives, six-member inventories without header_hex, counts as integers, field contract, record schema) and the producer document alone.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "receipt 21,769 (with the real six-member inventory; 20,992 invented); producer document 17,808", + "bytes_at_full_source": "same, unchanged (0.021x)", + "measurement_basis": "the real catalogue issuance on invented archives at this head (catalogue.payload) re-encoded with the pilot-shaped six-member inventories; `len(auth._json(catalogue._producer(), _RECEIPT_BYTES))` live (measure_acs_family.py part 2)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> acs_catalogue.issue_acs_source_catalogue (:1939) -> :485-486, :516-565; verify_acs_source_catalogue (preparation.py:1787) -> _checked (:440) -> :443/:452", + "classification": "stays", + "notes": "48x headroom. The 1/1000 artifact's /producer/acs is this producer document (17,886 B then; 17,808 B at this head)." + }, + { + "constant": "acs_population_catalogue.py:121 ceilings tuple (LIMITS)", + "value": "(8 * 1024**3, 16 * 1024**3, 64, 6_000_000, 100_000, 2 * 1024**3, 1024**2): each catalogue constant must satisfy `0 < v <= cap`", + "enforced_at": [ + "acs_population_catalogue.py:112-128 `_require(all(type(v) is int and 0 < v <= cap for v, cap in zip(limits, ceilings, strict=True)), \"LIMITS\")` inside _producer (runs at issuance :485 and every verify :443/:452/:590)" + ], + "refusal_codes": [ + "LIMITS (ACSSourceCatalogueError, :127)" + ], + "encodes": "A pin on the seven catalogue constants (their literal maxima); no document.", + "shape": "fixed", + "site_kind": "streams", + "bytes_at_tenth": "n/a", + "bytes_at_full_source": "n/a", + "measurement_basis": "code read", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "19", + "reach_chain": "... -> catalogue._producer (:105) -> :112-128", + "classification": "not-a-transport", + "notes": "Raising _CATALOGUE_BYTES or _RECEIPT_BYTES above these literals refuses LIMITS; a transport change to either must edit the tuple in the same commit. The producer document also carries `limits` (:140), so any move changes issued receipts." + }, + { + "constant": "acs_housing_universe_source.py:81-88 _parse_json (consumer-side re-encoding)", + "value": "`_require(type(payload) is bytes and len(payload) <= cap, \"JSON_SIZE\")`; `_require(type(value) is dict and _json(value) == payload, \"CANONICAL_JSON\")`; cap supplied per call: graph_acs_housing_universe.py:177 frame_context 64 MiB, :202/:218/:308/:310 receipts ACS_HU_RECEIPT_MAX_BYTES 1 MiB, :205/:309 projection ACS_HU_PAYLOAD_MAX_BYTES 64 MiB, :269 header 65,536", + "enforced_at": [ + "acs_housing_universe_source.py:82 JSON_SIZE, :85 CANONICAL_JSON, :86-87 JSON_DEPTH", + "graph_acs_housing_universe.py:177, :202, :204-206, :218, :269, :308, :309, :310 (the only callers; grep of packages/*/src)" + ], + "refusal_codes": [ + "JSON_SIZE, CANONICAL_JSON, JSON_DEPTH (ACSHousingSourceError; wrapped as GRAPH_ACS_TRANSPORT_CONTRACT at graph_acs_housing_universe.py:380)" + ], + "encodes": "Re-encodes every parsed document whole (canonical_json of the parsed value == payload). Roster-shaped inputs: the selected projection (64 MiB cap) and the prepared receipt (1 MiB cap); fixed inputs: source receipt, header, frame_context.", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "selected projection ~28.5 MB (0.43 x 64 MiB); prepared receipt 2,453,094 (2.34 x 1 MiB)", + "bytes_at_full_source": "selected projection ~285.3 MB (4.3 x 64 MiB); prepared receipt 24,508,342 (23.4 x 1 MiB)", + "measurement_basis": "the projection and prepared-receipt rows above", + "binds_at_tenth": true, + "binds_at_full_source": true, + "reachable_from": "none", + "reach_chain": "graph_acs_housing_universe.ACSHousingCreateKernel.run (:457) -> encode_acs_housing_evidence (:482 -> :196) and ACSHousingSelectKernel.run -> bind_acs_housing_evidence (:548 -> :249); the codec `us-acs-housing-universe-2024-v1` is registered at graph_sources.py:308 but the recovered 19-node graph declares only `us-survey-population-source-v1` and `raw-bytes-v1` sources, and no node uses `us.acs_housing.*` kernels", + "classification": "consumer", + "notes": "The prepared receipt would bind at 1/10 here (65,535 keys admitted) if the ACS housing graph were run; it is not on any runner path. Binds flags refer to those two documents at these caps." + }, + { + "constant": "graph_acs_housing_universe.py:52 ACS_HU_PAYLOAD_MAX_BYTES", + "value": "64 * 1024**2 = 67,108,864", + "enforced_at": [ + "graph_acs_housing_universe.py:245 `_require(len(payload) <= ACS_HU_PAYLOAD_MAX_BYTES, \"PAYLOAD_SIZE\")` (encode: MAGIC + encode_acs_housing_evidence (:482); ACSHousingSelectKernel.run (:509) -> bind_acs_housing_evidence (:548). Not a node of the 19/45 graphs (graph.json sources: us-survey-population-source-v1, raw-bytes-v1)", + "classification": "segment", + "notes": "Would bind at full source if this graph were run; unreachable from the census runners. `_verify_preparation` (:141) also re-encodes `canonical_json(chosen)` at :174 (uncapped, 24.5 MB) and the select kernel encodes `canonical_json(selected_projection)` at :607." + }, + { + "constant": "graph_acs_housing_universe.py:53 ACS_HU_HEADER_MAX_BYTES", + "value": "65536", + "enforced_at": [ + "graph_acs_housing_universe.py:242 `_require(len(encoded) <= ACS_HU_HEADER_MAX_BYTES, \"HEADER_SIZE\")`", + "graph_acs_housing_universe.py:264-267 (bind) HEADER_SIZE; :269 _parse_json cap" + ], + "refusal_codes": [ + "GRAPH_ACS_HEADER_SIZE (ACSHousingSourceError)" + ], + "encodes": "Fixed-shape header (:224-240): 15 keys of digests and integers.", + "shape": "fixed", + "site_kind": "materialises", + "bytes_at_tenth": "under 1 KB (15 short keys, seven 64-hex digests), unchanged", + "bytes_at_full_source": "same", + "measurement_basis": "code read of :224-240 (not encoded; unreachable module)", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "encode_acs_housing_evidence (:196) / bind_acs_housing_evidence (:249); not reached by the 19/45 runners", + "classification": "stays", + "notes": "Length prefix `struct.pack(\" `canonical_json(selection)` (cap: none)", + "value": "none", + "enforced_at": [ + "graph_acs_housing_universe.py:634-637 artifact `\"selection\": canonical_json(selection)`; :641 `\"selection_sha256\": source._sha(canonical_json(selection))` (encoded twice)" + ], + "refusal_codes": [ + "none" + ], + "encodes": "Per-row roster: positions (person and household int lists), native_keys (household SERIALNOs, person [SERIALNO, SPORDER] pairs), ordered_ids for all six entities (tax units, families, marital units, SPM units are SCALED ratios of the 1/1000 artifact).", + "shape": "roster", + "site_kind": "materialises", + "bytes_at_tenth": "~13.2 MB ESTIMATE (86 B per household with two persons, excluding the four group-entity id lists)", + "bytes_at_full_source": "~131.7 MB ESTIMATE, plus the group-entity ids (SCALED)", + "measurement_basis": "canonical_json on invented positions/native_keys/ordered_ids at 1,000 and 2,000 households (measure_acs_family.py part 1 graph_selection_artifact); small ints used, 7-digit ints at full source add bytes", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "ACSHousingSelectKernel.run (:509) -> :593-641; not a node of the census graphs", + "classification": "segment", + "notes": "Uncapped whole copy; recorded for completeness only." + }, + { + "constant": "asec_housing_universe.py:42 PAYLOAD_MAX_BYTES (enforced by graph_housing_universe.py; definition belongs to the ASEC family)", + "value": "len(MAGIC) 9 + 4 + HEADER_MAX_BYTES 65,536 + MAX_HOUSEHOLDS 400,000 x (8 x 7 RAW_COLUMNS + 7 DERIVED_COLUMNS = 63) + 32 = 25,265,581", + "enforced_at": [ + "graph_housing_universe.py:78-82 `hu._require(type(payload) is bytes and len(hu.MAGIC) + 4 + 32 < len(payload) <= hu.PAYLOAD_MAX_BYTES, \"BOUND_HU_SIZE\")`", + "graph_housing_universe.py:151-155 `expected_size = cursor + rows * (8 * len(hu.RAW_COLUMNS) + len(hu.DERIVED_COLUMNS)) + 32`; `_require(len(payload) == expected_size, \"BOUND_HU_LENGTH\")` (the 63-byte-per-household width)", + "graph_housing_universe.py:144-150 `0 < rows <= hu.MAX_HOUSEHOLDS` -> \"BOUND_HU_ROWS\"", + "graph_housing_universe.py:98-103 `struct.unpack_from(\" \"BOUND_HU_HEADER_SIZE\"", + "asec_housing_universe.py:382 (the ASEC owner's own decode; ASEC family)" + ], + "refusal_codes": [ + "BOUND_HU_SIZE, BOUND_HU_LENGTH, BOUND_HU_ROWS, BOUND_HU_HEADER_SIZE (HousingUniverseRefusalError via hu._require)" + ], + "encodes": "A numpy body: 7 int64 + 7 uint8 columns per ASEC household (63 B), over the ASEC source's own households (55,762), identical at every fraction; plus a fixed header.", + "shape": "upstream", + "site_kind": "materialises", + "bytes_at_tenth": "3,513,006 body (63 x 55,762) + header, unchanged", + "bytes_at_full_source": "3,513,006 + header, unchanged (0.139x)", + "measurement_basis": "constants read from asec_housing_universe.py:19-47 and multiplied (measure_acs_family.py part 1 asec_housing_universe); ASEC household count from brief section 3", + "binds_at_tenth": false, + "binds_at_full_source": false, + "reachable_from": "none", + "reach_chain": "bind_housing_universe is called from graph_composed_asec_binding.py:592 and graph_asec_prepared.py:440/:666 (and verify_graph_housing_rows from :686 / :443 / :680); none of those modules is in the brief's 19/45 node table", + "classification": "must-not-move", + "notes": "Fixed width x an upstream roster (ASEC file rows, not the stacked survey). Listed here only for its enforcement sites in graph_housing_universe.py; the row belongs to the ASEC family's census keyed at asec_housing_universe.py:42. MAX_HOUSEHOLDS 400,000 (:41) is a count wearing no byte name: 7.2x headroom over 55,762." + } + ], + "measurement_scripts": [ + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/acs/measure_acs_family.py (run with .venv/bin/python, sys.path = packages/*/src; part 1 encodes invented rows through auth._json/_json_size/_json_roster/_body_row_bound, housing._json, literal._json, canonical_json at 1,000 and 2,000 rows; part 2 drives the real issue_acs_source_catalogue and issue_acs_native_coverage on a two-household invented archive pair and measures every fixed-shape document at this head)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/acs/measurements.json (receipt; head 7e04682421041c986c4599d609d591672c22743c)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/acs/measure_acs_family.log (stdout)", + "/private/tmp/claude-501/-Users-maxghenis-PolicyEngine--worktrees-microcosm-native-byte-transports/ec2daf10-fa52-4bad-9397-4b0e60c15937/scratchpad/census/acs/invented/ (the invented archives and the private captures the issuance wrote)", + "experiments/native-byte-transports/byte-census-measurements.json (cited for the coverage body, cells JSON, key list, serialno list, old charges, raw sizes and state counts)" + ], + "open_questions": [ + "HEAD moved during the census from 77bafbb3acd8db70e13ddee58d44d2592a56742a to 7e04682421041c986c4599d609d591672c22743c (commit 'Stream the budget view's comparison; drop the segment-at-most-maximum guards'). `git diff --stat 77bafbb3a HEAD` over all twelve family files, asec_housing_universe.py and the sources manifest is empty, and sha256 of acs_person_coverage_authentication.py still equals binding._ACCEPTED (b8400706...), so every line cited holds at both heads.", + "docs/us-native-byte-transports.md is referenced by the code comments at acs_person_coverage_authentication.py:46 and acs_native_coverage_binding.py:38 but does not exist in the tree at this head (docs/ has us-native-row-ceilings.md, us-native-scale-transport.md, us-native-verification-once.md, us-native-spm-role-source-enrichment.md).", + "Row keying: where one constant fronts several roster documents (MAX_ROSTER_BYTES: charge, serialno pre-check, key-pair digest, issuance receipt; MAX_BODY_BYTES: body), I emitted one row per document with the constant's definition as the key prefix and a bracketed document tag, listing every enforcement site in each, so each verifier receives one document. Merge them if the orchestrator wants strictly one row per constant.", + "Four whole-roster encodings in this family have no bound at all at head (housing.py:913 prepared receipt; housing.py:605 selected-serialno digest; columns.py:292-294 native_roster_sha256; columns.py:295-297 projection_sha256, the last ~440 MB at full source). The brief's classification list has no 'uncapped' class; I recorded them as `segment` with value 'none'. The orchestrator should decide whether they are rows or notes.", + "Upstream-shaped derived documents that never move with the fraction (the 292,484,053-byte full lexical projection under 8 GiB; the 374,174,650-byte catalogue record stream under 2 GiB) are classified `stays` with shape `upstream`, reserving `must-not-move` for real file sizes, widths and scalars as the brief defines it.", + "WGTP/PWGTP token widths are not in the section-4 inputs. The 3-digit variant reproduces the row-count lane's measured full-projection.json to 0.03% (292,401,960 vs 292,484,053) and the catalogue-record estimate lands 0.6% under the pilot receipt, so the selected-projection numbers (~28.5 MB / ~285.3 MB) are calibrated ESTIMATES, flagged as such.", + "Full-source selection semantics: the runner always passes an explicit key tuple (`serialnos=acs_keys`, survey_population_preparation.py:1946-1963, ACS keys only), so the issuance receipt, prepared receipt and serialno pre-check carry 1,531,614 entries at full source; the `serialnos is None` route (fixed-shape receipts) is never taken. The row-count lane's verifier note that used 1,587,376 (stacked households) for ACS_HU_RECEIPT_MAX_BYTES conflicts with the brief's section-3 count; I used 1,531,614.", + "acs_inputs.py, acs_housing_universe.py and source_csv_builtin.py were read in full and carry no byte bound (zero rows). acs_sources.py's only byte constant is a chunk size; its fetch path is reachable from no src module.", + "The `age` runner (survey_age_calibration) was not traced into this family: it consumes an issued preparation, and any re-verification of the ACS native coverage would go through survey_population_preparation._validate (:1785-1795), which the 19-node chain already covers.", + "graph_housing_universe.py enforces only constants defined in asec_housing_universe.py (ASEC family); that row is included here for its enforcement sites and should be merged with the ASEC family's row keyed at asec_housing_universe.py:42.", + "Per-preparation cost not captured by any byte bound: four private captures each copy both archives (854 MB) and require 17.8 GiB free; the 292 MB full projection is built twice and read back whole once; the 24.5 MB issuance receipt and the ~285 MB selected projection are json.loads-ed whole on every verify_acs_native_coverage (preparation.py:1789, current_survey_predictors.py:99); the catalogue retains 1.63M household tuples with nested persons in memory." + ] + } + ] +} \ No newline at end of file diff --git a/experiments/native-byte-transports/dependent_test_battery.py b/experiments/native-byte-transports/dependent_test_battery.py new file mode 100644 index 000000000..eda3087d5 --- /dev/null +++ b/experiments/native-byte-transports/dependent_test_battery.py @@ -0,0 +1,125 @@ +"""Run every test file that names a module this lane touched, one per process. + +CI runs the build shard's groups in separate pytest processes so that one +file's ``monkeypatch.setattr`` on a module-level function cannot trip +``survey_population_preparation._producer``'s live-code seal for every later +file; the native-scale lane's report §9a is the write-up of what happens when +they share one. This runs them the way CI does, with a bounded number of +processes at a time, and records each file's own summary line. The pattern is +the retention lane's ``dependent_test_battery.py``; the markers are this lane's. + + python experiments/native-byte-transports/dependent_test_battery.py OUT.json [JOBS] +""" + +import concurrent.futures +import json +import pathlib +import subprocess +import sys +import time + +ROOT = pathlib.Path(__file__).resolve().parents[2] # the worktree root +PYTHON = ROOT / ".venv/bin/python" +MARKERS = ( + "acs_person_coverage_authentication", + "acs_native_coverage_binding", + "acs_population_catalogue", + "survey_population_preparation", + "graph_survey_population", + "survey_origin_budget", + "graph_survey_budget", + "graph_survey_calibration", + "graph_survey_age_artifact", + "current_survey_predictors", + "graph_child_property_income", + "current_child_property_income_source", + "graph_joint_empirical", +) +FILES = sorted( + str(path.relative_to(ROOT)) + for package in ("microcosm-build", "microcosm-fit") + for path in (ROOT / "packages" / package / "tests").glob("test_*.py") + if any(marker in path.read_text(encoding="utf-8") for marker in MARKERS) +) + + +def run(relative): + started = time.monotonic() + completed = subprocess.run( + [ + str(PYTHON), + "-m", + "pytest", + relative, + "-p", + "no:randomly", + "-p", + "no:cacheprovider", + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + tail = [line for line in completed.stdout.splitlines() if line.strip()] + summary = next( + ( + line + for line in reversed(tail) + if " passed" in line or " failed" in line or " error" in line + ), + "", + ) + return { + "file": relative, + "returncode": completed.returncode, + "summary": summary.strip(), + "wall_seconds": round(time.monotonic() - started, 1), + } + + +def main(): + out = pathlib.Path(sys.argv[1]) + jobs = int(sys.argv[2]) if len(sys.argv) > 2 else 4 + head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True + ).stdout.strip() + dirty = subprocess.run( + ["git", "status", "--porcelain", "--", "packages"], + cwd=ROOT, + capture_output=True, + text=True, + ).stdout.strip() + results = [] + print(f"head={head} dirty_packages={'yes' if dirty else 'no'} files={len(FILES)}") + with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool: + for result in pool.map(run, FILES): + results.append(result) + print( + f"{'ok ' if result['returncode'] == 0 else 'FAIL'} " + f"{result['file']} {result['summary']} ({result['wall_seconds']}s)", + flush=True, + ) + out.write_text( + json.dumps( + { + "scope": "diagnostic; not a build, certification or release artifact", + "release_eligible": False, + "invocation": "one pytest process per file, as CI runs them", + "head": head, + "packages_dirty_at_start": bool(dirty), + "markers": list(MARKERS), + "files": len(results), + "failing_files": [r["file"] for r in results if r["returncode"] != 0], + "results": sorted(results, key=lambda r: r["file"]), + }, + indent=2, + ) + + "\n" + ) + print( + f"\nfiles={len(results)} failing={len([r for r in results if r['returncode']])}" + ) + + +if __name__ == "__main__": + main() diff --git a/experiments/native-byte-transports/fifteenth/harness19_diag_fifteenth.py b/experiments/native-byte-transports/fifteenth/harness19_diag_fifteenth.py new file mode 100644 index 000000000..dc375f264 --- /dev/null +++ b/experiments/native-byte-transports/fifteenth/harness19_diag_fifteenth.py @@ -0,0 +1,747 @@ +"""The retention-seal lane's 1/15 run: above the transport ceiling that was. + +A copy of the native-scale lane's committed ``harness19_tenth.py`` with exactly +three changes, each named here and nowhere else in the file: + +* the sample is 1/15 -- 105,825 of the 1,587,376 source households, where the + single bounded encode refused at 96,860 -- rather than 1/10; +* ``RSS_CEILING`` is the retention brief's 48 GiB rather than 64 GiB; +* the label and the docstring say which lane and which fraction. + +It carries two further corrections the 1/15 measurement harness did not: its +``sample.fraction`` and its ``scope`` sentence report 1/15, where that harness +left both as the tenth's 1/10 while running ``Fraction(1, 15)``. Neither field +reaches any code path; both are receipt prose. + +Finally it carries the 1/10 refusal diagnostic's committed ``sys.monitoring`` +RAISE block verbatim, inserted at the same point in the file, so the cause the +three nested catch-alls discard is observed rather than inferred. + +Everything else, including the CPU and wall ceilings, the sampler, the flush +cadence, the receipt capture and every refusal, is byte-for-byte the tenth +harness's. Max chose 1/15 now and 1/10 when the machine is quiet (2026-09-17), +so this file exists beside the tenth harness rather than replacing it. + +Original docstring follows. + +The native-scale lane's 1/10 run: above the transport ceiling that was. + +Identical to the lane's baseline harness except for the sample, the ceilings and +one addition. The sample is 1/10 -- 158,737 of the 1,587,376 source households, +where the single bounded encode refused at 96,860 -- so it runs against its own +source tree, whose ``selection-request.json`` carries ``fraction: [1,10]`` +because the staged pilot request pins 1/1000 and ``_request`` requires the +argument to match it. Every other source file in that tree is an APFS clone of +the staged original and hashes identically; the recovered originals are not +written, linked or moved. + +The addition: every flush captures the roster transport's own on-disk receipt +(``snapshots/preparation/header.json``) and the allocation receipt from the +manifest if one exists yet. The ceiling question is answered by the preparation +and allocation nodes, which run in the first minutes, so a run the CPU ceiling +truncates still carries the evidence for it. + +Original docstring follows. + +Baseline for the native-scale lane: the 19-node graph, uncapped by node. + +This is ``harness19_verify_once.py`` from the verify-once lane with four +additions and no change to what is measured -- same call, same arguments, same +sample, same seed, same sampler, same interval, same refusals: + +* the ceilings are the native-scale brief's (9,000 CPU-s, 48 GiB, 12,000 wall-s) + rather than the verify-once lane's, so no node is cut short; +* a current-resident-size series is sampled every second through ``libproc``'s + ``proc_pidinfo``, because ``ru_maxrss`` is a high-water mark and a retention + slope needs the trace, not the peak; +* the ``run_atomic_survey_financial`` call is timed on its own, so the runner + time outside the node loop is a subtraction between measured quantities + rather than an inference; +* the returned manifest's per-node ``wall_time_s`` is recorded, which is what + the subtraction subtracts. + +Nothing is wrapped: the runtime still refuses instrumented producers, so +attribution is still the stack sampler's. + +Original docstring follows. + +Nineteen-node measurement for the verify-once lane, on this branch only. + +The instrumentation is ``probe_verify_once.py`` unchanged -- the same stack +sampler at the same interval, the same ceilings, the same refusals, the same +output shape. The only difference is the call: this runs the base financial +graph with the exact arguments the accepted v4 cold pilot ran +(``~/PolicyEngine/_recovered/pilot-runs/native19-v2/run/test_native_nineteen_node_financial.py``: +fraction 1/1000, seed 20260908, geography seed 20260908, +``demographic_conditioning=True``, ``n_estimators=2``, ``resume="auto"``), so +its wall, CPU and peak RSS are comparable with that receipt's +5,362.05 s / 5,278.61 s / 8.67 GiB. + +Two differences from that receipt are known and are not this change: it ran on +source snapshot ``2ca11c85a``, and its ``person-income-attachment.h5`` was the +pre-restoration ``5996dcdd...`` rather than the staged ``9ebc0ef2...``. Both +files are exactly 666,333,922 B, so the bytes the source tree hashes are the +same volume either way. + +Original docstring follows. + +Before/after measurement for the verify-once lane: the nine-node prefix. + +A parameterised copy of the v5 pilot probe +(``~/PolicyEngine/_recovered/pilot-runs/native45-v5/probe_repeated_verification.py``). +Everything that decides *what is measured* is verbatim: the same nine-node +population prefix, the same 1/1000 sample, the same seed, the same stack +sampler at the same interval, the same ceilings, the same output shape. The +only changes are the ones a before/after comparison needs: + +* the source tree, the staged run inputs and the output directory come from + the environment, so the identical file measures both trees; +* every imported ``microcosm`` module is asserted to live under that tree, + so a stray editable install cannot silently measure the wrong source; +* a resident-memory ceiling is enforced alongside the CPU ceiling; +* the load average and the interpreter's third-party versions are recorded + with each run, because this machine carries other lanes' work. + +Sources are read by path and never written. Nothing here is a build, a +certification or a release artifact. + + MEASURE_BASE= MEASURE_RUN= \ + MEASURE_PROBE= MEASURE_LABEL=