Consolidate the US survey graph build and enrichment pipeline

.gitattributes

@@ -0,0 +1,2 @@
+# Preserve exact unified-diff context markers in the archived measured experiment.
+experiments/us-acs-code-memo-not-adopted-20260910.patch -whitespace

.github/workflows/test.yml

@@ -168,6 +168,8 @@ jobs:
         # python3 (not bare `python`): guaranteed on the Ubuntu image, and this
         # step runs before any sync so it must hold with the stdlib alone.
         run: python3 tools/ci_test_groups.py --verify
+      - name: Check file-shard and workflow coverage without importing data tests
+        run: python3 -I -B -S packages/microcosm-build/tests/test_ci_test_groups.py
       - name: Verify the graph acceptance burndown is monotone
         # The charter's second process rule, executable: the number of red
         # properties in the microcosm-graph acceptance suite never rises
@@ -186,18 +188,29 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.13", "3.14"]
-        group: [trade, spine-uk, rest]
+        lane:
+          - {"group": "trade", "shard": "1/1"}
+          - {"group": "spine-uk", "shard": "1/1"}
+          - {"group": "rest", "shard": "1/6"}
+          - {"group": "rest", "shard": "2/6"}
+          - {"group": "rest", "shard": "3/6"}
+          - {"group": "rest", "shard": "4/6"}
+          - {"group": "rest", "shard": "5/6"}
+          - {"group": "rest", "shard": "6/6"}
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v6
         with:
           python-version: ${{ matrix.python-version }}
       - name: Sync workspace without engines
         run: uv sync --all-packages --locked
-      - name: Engine-free tests (${{ matrix.group }})
+      - name: Engine-free tests (${{ matrix.lane.group }} ${{ matrix.lane.shard }})
         run: |
-          mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.group }}")
-          /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider
+          set -euo pipefail
+          selected="$(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.lane.group }}" --shard "${{ matrix.lane.shard }}")"
+          test -n "$selected"
+          mapfile -t files <<< "$selected"
+          /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25
 
   engine-shared:
     # Deliberately does NOT depend on `changes`: it reads no classifier output,
@@ -207,7 +220,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.13", "3.14"]
+        python-version: ["3.13", "3.14.4"]
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v6
@@ -217,8 +230,75 @@ jobs:
         run: uv sync --all-packages --locked --extra us --extra uk
       - name: Shared/spec engine tests
         run: |
-          mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list shared-spec)
-          /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider
+          set -euo pipefail
+          selected="$(uv run --no-sync python tools/ci_test_groups.py --list shared-spec)"
+          test -n "$selected"
+          mapfile -t files <<< "$selected"
+          /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25
+
+  spec-seed-diagnostics:
+    # Produce bounded diagnosis immediately, independently of the full suite.
+    # This remains required by ci-ok and never updates assertion goldens.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.14.4"
+      - name: Sync workspace with US and UK engines
+        run: uv sync --all-packages --locked --extra us --extra uk
+      - name: Check diagnostic bootstrap and failure reporting
+        timeout-minutes: 2
+        env:
+          PYTHONDONTWRITEBYTECODE: "1"
+          PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
+        run: >-
+          uv run --no-sync pytest --noconftest -p no:cacheprovider -q
+          packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py
+          packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py
+          packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py
+          packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py
+          packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py
+      - name: Derive candidate spec and seed identities (diagnostic only)
+        id: spec_seed_diagnostics
+        timeout-minutes: 15
+        env:
+          DIAG_WORKFLOW_SHA: ${{ github.workflow_sha }}
+          DIAG_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          DIAG_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          DIAG_MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }}
+          PYTHONDONTWRITEBYTECODE: "1"
+          PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
+          OMP_NUM_THREADS: "1"
+          OPENBLAS_NUM_THREADS: "1"
+          MKL_NUM_THREADS: "1"
+          VECLIB_MAXIMUM_THREADS: "1"
+          NUMEXPR_NUM_THREADS: "1"
+          BLIS_NUM_THREADS: "1"
+          POPULACE_FIT_N_JOBS: "1"
+          POPULACE_FIT_PREDICT_WORKERS: "1"
+        run: |
+          DIAG_CHECKOUT_SHA="$(git rev-parse --verify HEAD)"
+          DIAG_UV_VERSION="$(uv --version)"
+          export DIAG_CHECKOUT_SHA DIAG_UV_VERSION
+          uv run --no-sync python tools/spec_seed_identity_diagnostics.py \
+            --output-dir "$RUNNER_TEMP/spec-seed-diagnostics"
+      - name: Retain bounded candidate identity evidence
+        if: ${{ always() && steps.spec_seed_diagnostics.outcome != 'skipped' }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: spec-seed-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}-py3.14.4
+          path: |
+            ${{ runner.temp }}/spec-seed-diagnostics/candidate-digests.json
+            ${{ runner.temp }}/spec-seed-diagnostics/seed-protocol.json
+            ${{ runner.temp }}/spec-seed-diagnostics/seed-map.json
+            ${{ runner.temp }}/spec-seed-diagnostics/seed-bindings.json
+            ${{ runner.temp }}/spec-seed-diagnostics/environment-and-source.json
+            ${{ runner.temp }}/spec-seed-diagnostics/diagnostic-status.json
+          if-no-files-found: error
+          retention-days: 3
+          compression-level: 0
+          include-hidden-files: false
 
   engine-us:
     needs: changes
@@ -228,20 +308,33 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.13", "3.14"]
-        group: [us-p, us-qs, us-not, us-am]
+        lane:
+          - {"group": "us-p", "shard": "1/1"}
+          - {"group": "us-qs", "shard": "1/1"}
+          - {"group": "us-not", "shard": "1/1"}
+          - {"group": "us-am", "shard": "1/4"}
+          - {"group": "us-am", "shard": "2/4"}
+          - {"group": "us-am", "shard": "3/4"}
+          - {"group": "us-am", "shard": "4/4"}
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v6
         with:
           python-version: ${{ matrix.python-version }}
       - name: Sync workspace with US and UK engines
         run: uv sync --all-packages --locked --extra us --extra uk
-      - name: US engine tests (${{ matrix.group }})
+      - name: US engine tests (${{ matrix.lane.group }} ${{ matrix.lane.shard }})
         run: |
-          while IFS= read -r proc; do
-            mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.group }}:${proc}")
-            /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider
-          done < <(uv run --no-sync python tools/ci_test_groups.py --procs "${{ matrix.group }}")
+          set -euo pipefail
+          selected_procs="$(uv run --no-sync python tools/ci_test_groups.py --procs "${{ matrix.lane.group }}")"
+          test -n "$selected_procs"
+          mapfile -t procs <<< "$selected_procs"
+          for proc in "${procs[@]}"; do
+            selected="$(uv run --no-sync python tools/ci_test_groups.py --list "${{ matrix.lane.group }}:${proc}" --shard "${{ matrix.lane.shard }}")"
+            test -n "$selected"
+            mapfile -t files <<< "$selected"
+            /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25
+          done
 
   engine-uk:
     needs: changes
@@ -261,10 +354,16 @@ jobs:
         run: uv sync --all-packages --locked --extra us --extra uk
       - name: UK engine tests
         run: |
-          while IFS= read -r proc; do
-            mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list "uk:${proc}")
-            /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider
-          done < <(uv run --no-sync python tools/ci_test_groups.py --procs uk)
+          set -euo pipefail
+          selected_procs="$(uv run --no-sync python tools/ci_test_groups.py --procs uk)"
+          test -n "$selected_procs"
+          mapfile -t procs <<< "$selected_procs"
+          for proc in "${procs[@]}"; do
+            selected="$(uv run --no-sync python tools/ci_test_groups.py --list "uk:${proc}")"
+            test -n "$selected"
+            mapfile -t files <<< "$selected"
+            /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider --durations=25
+          done
 
   wheels:
     # The charter-mandated packaging gate: editable installs hide packaging
@@ -277,6 +376,15 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.13", "3.14"]
+        lane:
+          - {"group": "wheels", "shard": "1/8"}
+          - {"group": "wheels", "shard": "2/8"}
+          - {"group": "wheels", "shard": "3/8"}
+          - {"group": "wheels", "shard": "4/8"}
+          - {"group": "wheels", "shard": "5/8"}
+          - {"group": "wheels", "shard": "6/8"}
+          - {"group": "wheels", "shard": "7/8"}
+          - {"group": "wheels", "shard": "8/8"}
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v6
@@ -336,10 +444,56 @@ jobs:
           PY
       - name: Spec identity section digests (cross-environment diffing aid)
         run: env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/spec_envelope_digests.py be uk
-      - name: Run the suite against the installed wheels
+      - name: Add engine-free source preparation extras
+        # Real source fixtures require microunit for tax-unit construction and
+        # PyTables for pandas HDF input, alongside h5py. Request the extras from
+        # built wheels so their metadata, not editable installs, supplies these
+        # dependencies. The preceding step still checks the base wheel boundary;
+        # neither source extra installs a country rules engine.
+        run: |
+          frame_wheel="$(ls dist/microcosm_frame-*.whl)"
+          build_wheel="$(ls dist/microcosm_build-*.whl)"
+          uv pip install --python /tmp/wheels-venv/bin/python \
+            --constraint /tmp/wheel-constraints.txt \
+            "microcosm-frame[us] @ file://$(realpath "$frame_wheel")" \
+            "microcosm-build[source-io] @ file://$(realpath "$build_wheel")"
+          env -u PYTHONPATH /tmp/wheels-venv/bin/python -I - <<'PY'
+          from importlib import metadata, util
+          from pathlib import Path
+          from tempfile import TemporaryDirectory
+
+          import h5py
+          import pandas as pd
+          import tables
+
+          assert util.find_spec("microunit") is not None, (
+              "microcosm-frame[us] must install microunit into the wheel venv"
+          )
+          # Exercise the same HDF reader used by source preparation. Imports
+          # alone do not establish that the compiled PyTables backend works.
+          with TemporaryDirectory() as directory:
+              path = Path(directory) / "invented-source.h5"
+              expected = pd.DataFrame({"person_id": [1, 2], "amount": [0.0, 12.5]})
+              expected.to_hdf(path, key="person")
+              pd.testing.assert_frame_equal(pd.read_hdf(path, key="person"), expected)
+          assert util.find_spec("policyengine_us") is None
+          try:
+              metadata.distribution("policyengine-us")
+          except metadata.PackageNotFoundError:
+              pass
+          else:
+              raise AssertionError("policyengine-us must be absent from the base wheel gate")
+          print("microunit", metadata.version("microunit"))
+          print("h5py", h5py.__version__, "tables", tables.__version__)
+          PY
+      - name: Run the suite against the installed wheels (${{ matrix.lane.shard }})
         run: |
+          set -euo pipefail
+          selected="$(env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/ci_test_groups.py --list wheels --shard "${{ matrix.lane.shard }}")"
+          test -n "$selected"
+          mapfile -t files <<< "$selected"
           /usr/bin/time -v env -u PYTHONPATH /tmp/wheels-venv/bin/python -I -m pytest \
-            packages/*/tests -p no:cacheprovider
+            "${files[@]}" -p no:cacheprovider --durations=25
 
   ci-ok:
     # The single aggregate context for branch protection. Per-leg matrix names
@@ -350,7 +504,7 @@ jobs:
     # dependency-skip lets a failed `changes` take the engine lanes down without
     # any required check going red: a lane skip is accepted only when
     # classification itself succeeded.
-    needs: [changes, lint, fast, engine-shared, engine-us, engine-uk, wheels]
+    needs: [changes, lint, fast, engine-shared, spec-seed-diagnostics, engine-us, engine-uk, wheels]
     if: always()
     runs-on: ubuntu-latest
     steps:
@@ -365,6 +519,7 @@ jobs:
           require_success lint           "${{ needs.lint.result }}"
           require_success fast           "${{ needs.fast.result }}"
           require_success engine-shared  "${{ needs['engine-shared'].result }}"
+          require_success spec-seed-diagnostics "${{ needs['spec-seed-diagnostics'].result }}"
           require_success wheels         "${{ needs.wheels.result }}"
           for lane in "engine-us:${{ needs['engine-us'].result }}" "engine-uk:${{ needs['engine-uk'].result }}"; do
             name="${lane%%:*}"; result="${lane#*:}"

CLAUDE.md

@@ -26,7 +26,8 @@ PR CI (`.github/workflows/test.yml`) has four lanes — `lint`, `fast`,
 the diff into `shared`/`us`/`uk`. `lint` verifies
 `tools/ci_test_groups.py --verify`, syncs with `--locked`, and runs ruff.
 `fast` runs the full tracked test-file inventory without engine extras in
-three groups (`trade`, `spine-uk`, `rest`); engine-gated tests skip there
+three groups (`trade`, `spine-uk`, `rest`), with `rest` split across six
+parallel matrix jobs per Python version; engine-gated tests skip there
 through whichever guard they carry — the `requires_*` markers, or the
 `importorskip` calls that remain the norm on the US side. `engine-shared` always syncs
 `--extra us --extra uk` and runs the shared/spec group. `engine-us` and
@@ -39,6 +40,31 @@ shard's real wheel, install into a clean uv-export-constrained venv, assert
 the wheel/import boundary and spec digests, and run the suite against installed
 wheels.
 
+`us-am` likewise has four matrix subdivisions. Within each, build and other-shard
+tests retain separate pytest processes to isolate their import state. Every wheel
+job retains the complete build/install/smoke boundary and runs one of eight
+disjoint file subdivisions, including engine-only files whose markers skip when
+the engine is absent. Other groups and both Python versions are unchanged.
+`tools/ci_test_groups.py --list GROUP[:PROCESS] --shard INDEX/COUNT` selects
+sorted whole files round-robin; indices are 1-based and counts are capped at 64.
+An invalid or empty selection fails before pytest. `--verify` proves the
+configured shard unions and process partitions, while the stdlib-only
+`python3 -I -B -S packages/microcosm-build/tests/test_ci_test_groups.py` checks
+the actual matrix and selection contract. See [CI file sharding](docs/ci-file-sharding.md).
+File counts are not measured runtime weights; a single expensive module can
+still dominate a job, and module-scoped fixtures must remain intact.
+
+After checking the base wheel boundary, the wheel lane installs the built
+`microcosm-frame[us]` and `microcosm-build[source-io]` extras for engine-free
+source tests. An invented pandas HDF round trip checks the compiled reader;
+the country rules engine must still be absent.
+
+The shared engine lane also runs a bounded source/seed identity diagnostic on
+Python 3.14.4 after its normal tests, including when those tests fail. Its six
+JSON artifacts retain candidate digests and their complete canonical seed
+records for review. They never replace the test assertions or certify coverage;
+review the captured source and dependency identities before updating any pins.
+
 New commits to a PR cancel older unfinished CI runs for that same PR.
 Each main-push run has a unique concurrency group, so all main-push runs
 remain independent and can finish validating their merged changes.
@@ -53,10 +79,9 @@ are already editing that test for another reason.
 
 **Adding a test file.** It must sit directly in `packages/<shard>/tests/` — flat,
 no subdirectories; `fixtures/` and `golden/` hold data only — and be named
-`test_*.py`. The lanes run explicit file lists built from a flat pathspec, while
-local `uv run pytest` and the wheels lane discover recursively, so a test parked
-next to its fixtures would run locally and stay green in CI without ever
-executing against an engine. `--verify` fails on such a file rather than letting
+`test_*.py`. All lanes, including wheels, run explicit flat file lists, while
+local `uv run pytest` discovers recursively. A test parked next to its fixtures
+could run locally without being selected in CI. `--verify` fails on such a file rather than letting
 it hide. Build tests that exercise a country engine must be named `test_us_*` or
 `test_uk_*` so they land in that country's lane; an engine-dependent file named
 anything else falls into the always-on `shared-spec` group and runs on every PR.