hua7450 · Oct 13, 2026 · Oct 13, 2026 · Oct 13, 2026 · Oct 13, 2026
diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml
@@ -310,20 +310,23 @@ jobs:
       - name: Run Contrib Python tests
         if: ${{ matrix.group == 'other-shard-1' }}
         run: uv run --no-sync make test-policy-contrib-python
-  Rest:
+  RestPython:
     needs: ReleaseLock
-    name: Full Suite - Rest (Python + variables)
+    name: Full Suite - Rest (Python, shard-1 + variables)
     runs-on: ubuntu-latest
-    # Matches the PR Rest job (python-tests.yaml, raised to 90 in #9785).
-    # Oct 6-8 on push: p50 51 min, max success 58.8 min, and one run hit the
-    # old 60-minute limit (37719726248), which skipped that version's Publish.
+    # Historical group times, including variables: 33-46 minutes plus setup.
     timeout-minutes: 90
     if: |
       (github.repository == 'PolicyEngine/policyengine-us')
       && (github.event.head_commit.message == 'Update PolicyEngine US')
     env:
       PYTHONUNBUFFERED: 1
-      PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception"
+      OMP_NUM_THREADS: 4
+      OPENBLAS_NUM_THREADS: 4
+      MKL_NUM_THREADS: 4
+      VECLIB_MAXIMUM_THREADS: 4
+      NUMEXPR_NUM_THREADS: 4
+      OMP_WAIT_POLICY: PASSIVE
     steps:
       - name: Checkout repo
         uses: actions/checkout@v6
@@ -342,30 +345,98 @@ jobs:
       - name: Turn off default branching
         shell: bash
         run: bash ./update_itemization.sh
-      # Each Rest group runs in a separate process to release its heap.
-      # SPM simulation/API contracts run in the Microsimulation job instead.
+      # Each group remains a separate, sequential process on its runner.
+      # SPM simulation/API contracts stay in the Microsimulation job.
       - name: Run remaining Python tests
-        # Continue after failures, but stop when the workflow is cancelled.
         if: ${{ !cancelled() }}
         env:
           PYTEST_ADDOPTS: "--durations=0 -v -p no:unraisableexception -p no:threadexception"
-        run: uv run --no-sync make test-other-python-rest REST_REPORT_DIR=.
+        run: uv run --no-sync make test-other-python-rest REST_PYTHON_SHARD=1 REST_REPORT_DIR=.
+      - name: Upload Rest Python timing and memory reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: rest-python-test-timings-shard-1
+          path: |
+            rest-python-*.xml
+            rest-python-*-resources.txt
       - name: Run tests/variables YAML tests
-        # Continue after failures, but stop when the workflow is cancelled.
         if: ${{ !cancelled() }}
         env:
           PYTEST_ADDOPTS: "--durations=0 --junitxml=rest-variables.xml -p no:unraisableexception -p no:threadexception"
         run: /usr/bin/time -v -o rest-variables-resources.txt uv run --no-sync make test-yaml-variables
-      - name: Upload Rest timing and memory reports
+      - name: Upload Rest variables timing and memory reports
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: rest-test-timings
+          name: rest-variables-test-timings
           path: |
-            rest-python-*.xml
             rest-variables.xml
-            rest-python-*-resources.txt
             rest-variables-resources.txt
+  Rest:
+    needs: [RestPython]
+    name: Full Suite - Rest (Python + variables)
+    runs-on: ubuntu-latest
+    # Historical group times: 32-46 minutes plus setup.
+    # This runner also retains the legacy summary check, keeping PRs at 27 runners.
+    timeout-minutes: 90
+    if: |
+      always()
+      && (github.repository == 'PolicyEngine/policyengine-us')
+      && (github.event.head_commit.message == 'Update PolicyEngine US')
+    env:
+      PYTHONUNBUFFERED: 1
+      OMP_NUM_THREADS: 4
+      OPENBLAS_NUM_THREADS: 4
+      MKL_NUM_THREADS: 4
+      VECLIB_MAXIMUM_THREADS: 4
+      NUMEXPR_NUM_THREADS: 4
+      OMP_WAIT_POLICY: PASSIVE
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.14
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.1.0
+        with:
+          version: "0.12.13"
+      - name: Resolve latest compatible PolicyEngine Core
+        run: uv lock --upgrade-package policyengine-core
+      - name: Install dependencies
+        run: uv sync --locked --extra dev
+      - name: Turn off default branching
+        shell: bash
+        run: bash ./update_itemization.sh
+      # Each group remains a separate, sequential process on its runner.
+      # SPM simulation/API contracts stay in the Microsimulation job.
+      - name: Run remaining Python tests
+        id: python_tests
+        if: ${{ !cancelled() }}
+        env:
+          PYTEST_ADDOPTS: "--durations=0 -v -p no:unraisableexception -p no:threadexception"
+        run: uv run --no-sync make test-other-python-rest REST_PYTHON_SHARD=2 REST_REPORT_DIR=.
+      - name: Upload Rest Python timing and memory reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: rest-python-test-timings-shard-2
+          path: |
+            rest-python-*.xml
+            rest-python-*-resources.txt
+      - name: Check all Rest jobs succeeded
+        if: always()
+        shell: bash
+        env:
+          PYTHON_RESULT: ${{ needs.RestPython.result }}
+          SHARD_RESULT: ${{ steps.python_tests.outcome }}
+        run: |
+          if [[ "$PYTHON_RESULT" != "success" || "$SHARD_RESULT" != "success" ]]; then
+            echo "::error::Rest shards did not all succeed: shard-1=$PYTHON_RESULT, shard-2=$SHARD_RESULT"
+            exit 1
+          fi
   Microsimulation:
     needs: ReleaseLock
     name: Full Suite - Microsimulation
@@ -423,7 +494,7 @@ jobs:
     if: |
       (github.repository == 'PolicyEngine/policyengine-us')
       && (github.event.head_commit.message == 'Update PolicyEngine US')
-    needs: [ReleaseLock, ReleaseLockTests, Baseline, HouseholdAPIPartners, Contrib, Rest, Microsimulation]
+    needs: [ReleaseLock, ReleaseLockTests, Baseline, HouseholdAPIPartners, Contrib, RestPython, Rest, Microsimulation]
     # PyPI trusted publishing: PyPI accepts uploads only from this workflow in
     # the pypi environment, which is limited to main. No stored PyPI token.
     environment: pypi

diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml
@@ -1,19 +1,23 @@
 name: Python tests
 on:
   workflow_call:
-# Group these jobs in the Actions view while retaining two independent runners.
+# Two sequential Rest runners keep each shard below its timeout.
+# Shard 1 includes variables; shard 2 also supplies the legacy summary check.
 # Only the guarded PR workflow calls this file; it has no automatic triggers.
 jobs:
-  Rest:
-    name: Full Suite - Rest (Python + variables)
+  RestPython:
+    name: Full Suite - Rest (Python, shard-1 + variables)
     runs-on: ubuntu-latest
-    # Passing runs on 2026-10-04 took 42-55 minutes (#9653, #9766, #9769,
-    # #9777, #9784), so a 60-minute limit cancels PRs that add a few
-    # minutes of tests (#9695 was cancelled at 60 minutes twice).
+    # Historical group times, including variables: 33-46 minutes plus setup.
     timeout-minutes: 90
     env:
       PYTHONUNBUFFERED: 1
-      PYTEST_ADDOPTS: "--durations=25 -p no:unraisableexception -p no:threadexception"
+      OMP_NUM_THREADS: 4
+      OPENBLAS_NUM_THREADS: 4
+      MKL_NUM_THREADS: 4
+      VECLIB_MAXIMUM_THREADS: 4
+      NUMEXPR_NUM_THREADS: 4
+      OMP_WAIT_POLICY: PASSIVE
     steps:
       - name: Checkout repo
         uses: actions/checkout@v6
@@ -32,30 +36,95 @@ jobs:
       - name: Turn off default branching
         shell: bash
         run: bash ./update_itemization.sh
-      # Each Rest group runs in a separate process to release its heap.
-      # SPM simulation/API contracts run in the Microsimulation job instead.
+      # Each group remains a separate, sequential process on its runner.
+      # SPM simulation/API contracts stay in the Microsimulation job.
       - name: Run remaining Python tests
-        # Continue after failures, but stop when the workflow is cancelled.
         if: ${{ !cancelled() }}
         env:
           PYTEST_ADDOPTS: "--durations=0 -v -p no:unraisableexception -p no:threadexception"
-        run: uv run --no-sync make test-other-python-rest REST_REPORT_DIR=.
+        run: uv run --no-sync make test-other-python-rest REST_PYTHON_SHARD=1 REST_REPORT_DIR=.
+      - name: Upload Rest Python timing and memory reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: rest-python-test-timings-shard-1
+          path: |
+            rest-python-*.xml
+            rest-python-*-resources.txt
       - name: Run tests/variables YAML tests
-        # Continue after failures, but stop when the workflow is cancelled.
         if: ${{ !cancelled() }}
         env:
           PYTEST_ADDOPTS: "--durations=0 --junitxml=rest-variables.xml -p no:unraisableexception -p no:threadexception"
         run: /usr/bin/time -v -o rest-variables-resources.txt uv run --no-sync make test-yaml-variables
-      - name: Upload Rest timing and memory reports
+      - name: Upload Rest variables timing and memory reports
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: rest-test-timings
+          name: rest-variables-test-timings
           path: |
-            rest-python-*.xml
             rest-variables.xml
-            rest-python-*-resources.txt
             rest-variables-resources.txt
+  Rest:
+    needs: [RestPython]
+    if: ${{ always() }}
+    name: Full Suite - Rest (Python + variables)
+    runs-on: ubuntu-latest
+    # Historical group times: 32-46 minutes plus setup.
+    # This runner also retains the legacy summary check, keeping PRs at 27 runners.
+    timeout-minutes: 90
+    env:
+      PYTHONUNBUFFERED: 1
+      OMP_NUM_THREADS: 4
+      OPENBLAS_NUM_THREADS: 4
+      MKL_NUM_THREADS: 4
+      VECLIB_MAXIMUM_THREADS: 4
+      NUMEXPR_NUM_THREADS: 4
+      OMP_WAIT_POLICY: PASSIVE
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.14
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.1.0
+        with:
+          version: "0.12.13"
+      - name: Resolve latest compatible PolicyEngine Core
+        run: uv lock --upgrade-package policyengine-core
+      - name: Install dependencies
+        run: uv sync --locked --extra dev
+      - name: Turn off default branching
+        shell: bash
+        run: bash ./update_itemization.sh
+      # Each group remains a separate, sequential process on its runner.
+      # SPM simulation/API contracts stay in the Microsimulation job.
+      - name: Run remaining Python tests
+        id: python_tests
+        if: ${{ !cancelled() }}
+        env:
+          PYTEST_ADDOPTS: "--durations=0 -v -p no:unraisableexception -p no:threadexception"
+        run: uv run --no-sync make test-other-python-rest REST_PYTHON_SHARD=2 REST_REPORT_DIR=.
+      - name: Upload Rest Python timing and memory reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: rest-python-test-timings-shard-2
+          path: |
+            rest-python-*.xml
+            rest-python-*-resources.txt
+      - name: Check all Rest jobs succeeded
+        if: always()
+        shell: bash
+        env:
+          PYTHON_RESULT: ${{ needs.RestPython.result }}
+          SHARD_RESULT: ${{ steps.python_tests.outcome }}
+        run: |
+          if [[ "$PYTHON_RESULT" != "success" || "$SHARD_RESULT" != "success" ]]; then
+            echo "::error::Rest shards did not all succeed: shard-1=$PYTHON_RESULT, shard-2=$SHARD_RESULT"
+            exit 1
+          fi
   Microsimulation:
     name: Full Suite - Microsimulation
     runs-on: ubuntu-latest

diff --git a/Makefile b/Makefile
@@ -43,8 +43,25 @@ REST_SPM_TESTS := $(TESTS)/core/test_spm_policy_family.py \
 # in its grid fixture. Core reached 13.3 GB in CI run 38141803955 after this
 # and three smaller modules arrived; isolate SSI rather than filling heavy-a
 # (8.1 GB including the pending capital-gains MTR change). These groups remain
-# sequential subprocesses on the existing Rest runner.
+# sequential subprocesses within each Rest Python shard.
 REST_PYTHON_GROUPS := cached-arrays heavy-a heavy-b core-heavy core policy-parameters policy remaining-heavy remaining
+# CI runs two shards of these existing isolated processes. Across seven
+# recent runs (including 38326922385 and 38289790999), their summed group
+# times were 32-45 and 32-46 minutes; variables adds about a minute to shard 1.
+# Both stay near 45 minutes, while the old serial job hit its 90-minute limit.
+# No shard runs more than one Python group at a time. Locally, leaving
+# REST_PYTHON_SHARD unset still runs every group in the original order.
+REST_PYTHON_SHARD_1_GROUPS := cached-arrays core-heavy policy-parameters remaining-heavy remaining
+REST_PYTHON_SHARD_2_GROUPS := heavy-a heavy-b core policy
+REST_PYTHON_SHARD ?=
+ifeq ($(REST_PYTHON_SHARD),)
+REST_PYTHON_SELECTED_GROUPS := $(REST_PYTHON_GROUPS)
+else
+REST_PYTHON_SELECTED_GROUPS := $(REST_PYTHON_SHARD_$(REST_PYTHON_SHARD)_GROUPS)
+ifeq ($(strip $(REST_PYTHON_SELECTED_GROUPS)),)
+$(error REST_PYTHON_SHARD must be 1 or 2)
+endif
+endif
 REST_CACHED_ARRAY_TESTS := $(TESTS)/test_formulas_do_not_write_into_cached_arrays.py
 REST_HEAVY_A_TESTS := $(TESTS)/test_md_poverty_line_credit_invariants.py \
 	$(TESTS)/core/test_ald_determinism.py \
@@ -281,7 +298,7 @@ test-other-python-spm:
 	pytest $(REST_SPM_TESTS) --maxfail=0
 # Every group runs even after one fails; the target fails if any group did.
 test-other-python-rest:
-	@failed=""; for group in $(REST_PYTHON_GROUPS); do \
+	@failed=""; for group in $(REST_PYTHON_SELECTED_GROUPS); do \
 		echo "=== Rest Python group: $$group ==="; \
 		$(MAKE) --no-print-directory test-other-python-rest-$$group || failed="$$failed $$group"; \
 	done; \

diff --git a/changelog.d/ci-rest-parallel-headroom.fixed.md b/changelog.d/ci-rest-parallel-headroom.fixed.md
@@ -0,0 +1 @@
+Split the Rest CI suite into two sequential Python shards, folding variables YAML and the legacy aggregate check into those runners to avoid the 90-minute timeout while increasing the PR runner budget from 26 to 27. Retain isolated test processes and timing reports.
diff --git a/policyengine_us/tests/code_health/test_pr_runner_budget.py b/policyengine_us/tests/code_health/test_pr_runner_budget.py
@@ -43,13 +43,17 @@ def test_pr_runner_budget_keeps_the_heavy_suites_separate():
             counts[name] = len(matrix["include"])
         else:
             counts[name] = math.prod(len(values) for values in matrix.values())
-    assert sum(counts.values()) == 26
+    # Rest hit the 90-minute timeout on #10024, #9913, #10139 and #10029
+    # in October 2026. One extra runner buys about 45 minutes of headroom:
+    # two tested shards share variables and the legacy summary check.
+    assert sum(counts.values()) == 27
     assert {
         name: counts[name]
-        for name in ("Baseline", "Contrib", "Rest", "Microsimulation")
+        for name in ("Baseline", "Contrib", "RestPython", "Rest", "Microsimulation")
     } == {
         "Baseline": 11,
         "Contrib": 9,
+        "RestPython": 1,
         "Rest": 1,
         "Microsimulation": 1,
     }
@@ -71,16 +75,21 @@ def test_suite_groups_have_no_extra_runners_or_triggers():
         # PyYAML parses the bare `on` key as True. No push/PR trigger here:
         # the draft guard must control the only route to these runners.
         assert GROUP_WORKFLOWS[name][True] == {"workflow_call": None}
-    assert set(PYTHON_JOBS) == {"Rest", "Microsimulation"}
+    assert set(PYTHON_JOBS) == {"RestPython", "Rest", "Microsimulation"}
     assert {name: job["timeout-minutes"] for name, job in PYTHON_JOBS.items()} == {
+        "RestPython": 90,
         "Rest": 90,
         "Microsimulation": 60,
     }
-    for job in PYTHON_JOBS.values():
+    for name, job in PYTHON_JOBS.items():
         assert job["runs-on"] == "ubuntu-latest"
-        assert not job.get("needs")
+        assert job.get("needs", []) == (["RestPython"] if name == "Rest" else [])
         assert not job.get("strategy")
         assert not job.get("continue-on-error")
+    # The second tested runner also checks the first runner's result. There
+    # is no third runner just for the summary, or a variables-only runner.
+    assert PYTHON_JOBS["Rest"]["if"] == "${{ always() }}"
+    assert PYTHON_JOBS["Rest"]["name"] == "Full Suite - Rest (Python + variables)"
 
 
 def test_sequential_variants_preserve_failures_and_allow_later_results():
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Split the Rest CI suite into two sequential Python shards, folding variables YAML and the legacy aggregate check into those runners to avoid the 90-minute timeout while increasing the PR runner budget from 26 to 27. Retain isolated test processes and timing reports.