Reduce PR CI to 26 jobs and avoid duplicate Python tests

.github/actions/bundle-metadata/action.yaml

@@ -0,0 +1,30 @@
+name: Validate bundle metadata
+description: Check the runtime metadata contract in a separate checkout and environment.
+runs:
+  using: composite
+  steps:
+    - name: Checkout tested merge commit
+      uses: actions/checkout@v6
+      with:
+        path: bundle-metadata
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.14"
+    - name: Resolve latest compatible PolicyEngine Core
+      shell: bash
+      working-directory: bundle-metadata
+      run: uv lock --upgrade-package policyengine-core
+    - name: Install package
+      shell: bash
+      working-directory: bundle-metadata
+      run: uv sync --locked --extra dev
+    - name: Install bundle validation tooling
+      # Preserve the independently pinned contract dependency.
+      shell: bash
+      working-directory: bundle-metadata
+      run: uv pip install --python .venv/bin/python "policyengine-bundles @ git+https://github.com/PolicyEngine/policyengine-bundles@8ae9f56fefcf89f69b8a7e3bc49928509c6207be"
+    - name: Validate runtime metadata contract
+      shell: bash
+      working-directory: bundle-metadata
+      run: uv run --no-sync python -m pytest policyengine_us/tests/test_build_metadata.py

.github/actions/candidate-wheel/action.yaml

@@ -0,0 +1,66 @@
+name: Build an unpublished release wheel
+description: Build one release candidate from a clean reviewed-head checkout and retain its receipt.
+inputs:
+  kind:
+    description: Candidate version kind (rc1 or final).
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Checkout reviewed PR head
+      uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        path: candidate-${{ inputs.kind }}
+    - name: Detect the release tooling on the PR head
+      id: tooling
+      # Older PR heads may predate the tooling present on the merge commit.
+      shell: bash
+      working-directory: candidate-${{ inputs.kind }}
+      run: |
+        if [ -f .github/release_lock.py ] && [ -f .github/release_build.py ]; then
+          echo "present=true" >> "$GITHUB_OUTPUT"
+        else
+          echo "present=false" >> "$GITHUB_OUTPUT"
+          echo "::notice title=Candidate wheel skipped::The PR head predates the release tooling (.github/release_lock.py and .github/release_build.py). Merge main into the branch to build its candidate wheel."
+        fi
+    - name: Check committed registry lock
+      if: steps.tooling.outputs.present == 'true'
+      shell: bash
+      working-directory: candidate-${{ inputs.kind }}
+      run: python .github/release_lock.py --committed
+    - name: Predict release version with the release helper
+      if: steps.tooling.outputs.present == 'true'
+      shell: bash
+      working-directory: candidate-${{ inputs.kind }}
+      run: python .github/release_build.py --prepare "${{ inputs.kind }}"
+    - name: Refresh only the release version in the registry lock
+      if: steps.tooling.outputs.present == 'true'
+      shell: bash
+      working-directory: candidate-${{ inputs.kind }}
+      run: python .github/release_lock.py --refresh
+    - name: Install package
+      if: steps.tooling.outputs.present == 'true'
+      shell: bash
+      working-directory: candidate-${{ inputs.kind }}
+      run: uv sync --locked --extra dev
+    - name: Build package from unchanged policy source
+      if: steps.tooling.outputs.present == 'true'
+      shell: bash
+      working-directory: candidate-${{ inputs.kind }}
+      run: uv run --no-sync make
+    - name: Record and verify release build
+      if: steps.tooling.outputs.present == 'true'
+      shell: bash
+      working-directory: candidate-${{ inputs.kind }}
+      run: uv run --no-sync python .github/release_build.py
+    - name: Retain unpublished wheel and build receipt
+      if: steps.tooling.outputs.present == 'true'
+      uses: actions/upload-artifact@v4
+      with:
+        name: release-wheel-${{ inputs.kind }}-${{ github.event.pull_request.head.sha }}
+        path: |
+          candidate-${{ inputs.kind }}/dist/*.whl
+          candidate-${{ inputs.kind }}/release-build/receipt.json
+        if-no-files-found: error
+        retention-days: 14

.github/actions/python-compat/action.yaml

@@ -0,0 +1,35 @@
+name: Python compatibility checks
+description: Install, import, and check dataset semantics in an isolated checkout and environment.
+inputs:
+  python-version:
+    description: Supported Python version to check.
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Checkout for Python ${{ inputs.python-version }}
+      uses: actions/checkout@v6
+      with:
+        path: python-${{ inputs.python-version }}
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ inputs.python-version }}
+    - name: Resolve latest compatible PolicyEngine Core
+      shell: bash
+      working-directory: python-${{ inputs.python-version }}
+      run: uv lock --upgrade-package policyengine-core
+    - name: Install package
+      shell: bash
+      working-directory: python-${{ inputs.python-version }}
+      run: uv sync --locked --python "${{ inputs.python-version }}" --extra dev
+    - name: Smoke-import
+      shell: bash
+      working-directory: python-${{ inputs.python-version }}
+      run: |
+        uv run --no-sync --python "${{ inputs.python-version }}" python -c "from policyengine_us import CountryTaxBenefitSystem, Simulation, Microsimulation; print('import OK')"
+    - name: Dataset copy/extension tests (real pandas semantics per Python version)
+      # Fixture-based and requires no dataset downloads.
+      shell: bash
+      working-directory: python-${{ inputs.python-version }}
+      run: uv run --no-sync --python "${{ inputs.python-version }}" python -m pytest policyengine_us/tests/microsimulation/data/ -q

.github/release_build.py

@@ -1,6 +1,6 @@
 """Verify clean policy source and record a locally built release wheel.
 
-The PR matrix can prepare either candidate using the actual release bump helper.
+The PR workflow prepares both candidates using the actual release bump helper.
 Both candidates and Publish record a receipt after the same make command.
 This script never resolves dependencies or publishes.
 """

.github/tests/test_release_build.py

@@ -31,11 +31,15 @@ def test_candidate_build_matches_publish_without_publication_or_policy_mutations
         self,
     ):
         pr = (ROOT / ".github/workflows/pr.yaml").read_text()
-        candidate = job(pr, "CandidateWheel")
+        candidate_job = job(pr, "PackageCompatibility")
+        candidate_action = (
+            ROOT / ".github/actions/candidate-wheel/action.yaml"
+        ).read_text()
+        candidate = candidate_job + candidate_action
         publish = job((ROOT / ".github/workflows/push.yaml").read_text(), "Publish")
         commands = (
             "python .github/release_lock.py --committed",
-            'python .github/release_build.py --prepare "${{ matrix.kind }}"',
+            'python .github/release_build.py --prepare "${{ inputs.kind }}"',
             "python .github/release_lock.py --refresh",
             "uv sync --locked --extra dev",
             "uv run --no-sync make",
@@ -47,8 +51,12 @@ def test_candidate_build_matches_publish_without_publication_or_policy_mutations
             sorted(candidate.index(value) for value in commands),
         )
         self.assertIn("github.event.pull_request.head.sha", candidate)
-        self.assertIn("kind: [rc1, final]", candidate)
-        self.assertIn("release-wheel-${{ matrix.kind }}-", candidate)
+        self.assertEqual(
+            candidate_job.count("uses: ./.github/actions/candidate-wheel"), 2
+        )
+        self.assertIn("kind: rc1", candidate_job)
+        self.assertIn("kind: final", candidate_job)
+        self.assertIn("release-wheel-${{ inputs.kind }}-", candidate_action)
         for expected in (
             "runs-on: ubuntu-24.04",
             'python-version: "3.14.7"',

.github/workflows/baseline-tests.yaml

@@ -0,0 +1,61 @@
+name: Baseline tests
+on:
+  workflow_call:
+# Group the existing matrix in the Actions view without adding runners.
+# The guarded PR workflow is the only caller.
+jobs:
+  Baseline:
+    name: Full Suite - Baseline (${{ matrix.group }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      PYTHONUNBUFFERED: 1
+      PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # One state per subprocess; retain four runners and one worker each.
+          # The former NY/OH/OK process peaked at 15.1 GB (run 36930199045).
+          - group: states-shard-1
+            cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --mode per-subdir --workers 1 --shard 1/4
+          - group: states-shard-2
+            cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --mode per-subdir --workers 1 --shard 2/4
+          - group: states-shard-3
+            cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --mode per-subdir --workers 1 --shard 3/4
+          - group: states-shard-4
+            cmd: python policyengine_us/tests/test_batched.py policyengine_us/tests/policy/baseline/gov/states --mode per-subdir --workers 1 --shard 4/4
+          - group: irs
+            cmd: make test-yaml-no-structural-other-irs
+          - group: household
+            cmd: make test-yaml-no-structural-other-household
+          - group: ssa-usda
+            cmd: make test-yaml-no-structural-other-ssa-usda
+          - group: rest-a
+            cmd: make test-yaml-no-structural-other-rest-a
+          - group: rest-b
+            cmd: make test-yaml-no-structural-other-rest-b
+          - group: contrib-hhs
+            cmd: make test-yaml-contrib-hhs
+          - group: reform
+            cmd: make test-yaml-reform
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.14
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.1.0
+        with:
+          version: "0.12.13"
+      - name: Resolve latest compatible PolicyEngine Core
+        run: uv lock --upgrade-package policyengine-core
+      - name: Install dependencies
+        run: uv sync --locked --extra dev
+      - name: Turn off default branching
+        shell: bash
+        run: bash ./update_itemization.sh
+      - name: Run Baseline YAML tests (${{ matrix.group }})
+        run: uv run --no-sync ${{ matrix.cmd }}

.github/workflows/contrib-tests.yaml

@@ -0,0 +1,58 @@
+name: Contrib tests
+on:
+  workflow_call:
+# Group the existing matrix in the Actions view without adding runners.
+# The guarded PR workflow is the only caller.
+jobs:
+  Contrib:
+    name: Full Suite - Contrib (${{ matrix.group }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      PYTHONUNBUFFERED: 1
+      PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - group: states-shard-1
+            target: test-yaml-structural-heavy-shard-1
+          - group: states-shard-2
+            target: test-yaml-structural-heavy-shard-2
+          - group: states-shard-3
+            target: test-yaml-structural-heavy-shard-3
+          - group: states-shard-4
+            target: test-yaml-structural-heavy-shard-4
+          - group: other-shard-1
+            target: test-yaml-structural-other
+          - group: other-shard-2a
+            target: test-yaml-structural-other-shard-2a
+          - group: other-shard-2b
+            target: test-yaml-structural-other-shard-2b
+          - group: other-shard-3
+            target: test-yaml-structural-other-shard-3
+          - group: congress
+            target: test-yaml-structural-congress
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.14
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.1.0
+        with:
+          version: "0.12.13"
+      - name: Resolve latest compatible PolicyEngine Core
+        run: uv lock --upgrade-package policyengine-core
+      - name: Install dependencies
+        run: uv sync --locked --extra dev
+      - name: Turn off default branching
+        shell: bash
+        run: bash ./update_itemization.sh
+      - name: Run Contrib YAML tests (${{ matrix.group }})
+        run: uv run --no-sync make ${{ matrix.target }}
+      - name: Run Contrib Python tests
+        if: ${{ matrix.group == 'other-shard-1' }}
+        run: uv run --no-sync make test-policy-contrib-python