MaxGhenis · Oct 4, 2026 · Oct 4, 2026 · Oct 4, 2026 · Oct 4, 2026
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -483,16 +483,18 @@ jobs:
         shell: bash
         run: bash ./update_itemization.sh
       # Run sequentially on this runner; process exit releases the SPM heap
-      # before collecting the remaining Python files.
+      # before collecting the remaining Python files. Those run as one pytest
+      # process per group (REST_PYTHON_GROUPS in the Makefile), and each group
+      # writes rest-python-<group>.xml and rest-python-<group>-resources.txt.
       - name: Run SPM Python tests
         env:
           PYTEST_ADDOPTS: "--durations=0 -v --junitxml=rest-python-spm.xml -p no:unraisableexception -p no:threadexception"
         run: /usr/bin/time -v -o rest-python-spm-resources.txt uv run --no-sync make test-other-python-spm
       - name: Run remaining Python tests
         if: always()
         env:
-          PYTEST_ADDOPTS: "--durations=0 -v --junitxml=rest-python-other.xml -p no:unraisableexception -p no:threadexception"
-        run: /usr/bin/time -v -o rest-python-other-resources.txt uv run --no-sync make test-other-python-rest
+          PYTEST_ADDOPTS: "--durations=0 -v -p no:unraisableexception -p no:threadexception"
+        run: uv run --no-sync make test-other-python-rest REST_REPORT_DIR=.
       - name: Run tests/variables YAML tests
         if: always()
         env:
@@ -504,11 +506,9 @@ jobs:
         with:
           name: rest-test-timings
           path: |
-            rest-python-spm.xml
-            rest-python-other.xml
+            rest-python-*.xml
             rest-variables.xml
-            rest-python-spm-resources.txt
-            rest-python-other-resources.txt
+            rest-python-*-resources.txt
             rest-variables-resources.txt
   Microsimulation:
     needs: ReleaseLock

diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml
@@ -299,16 +299,18 @@ jobs:
         shell: bash
         run: bash ./update_itemization.sh
       # Run sequentially on this runner; process exit releases the SPM heap
-      # before collecting the remaining Python files.
+      # before collecting the remaining Python files. Those run as one pytest
+      # process per group (REST_PYTHON_GROUPS in the Makefile), and each group
+      # writes rest-python-<group>.xml and rest-python-<group>-resources.txt.
       - name: Run SPM Python tests
         env:
           PYTEST_ADDOPTS: "--durations=0 -v --junitxml=rest-python-spm.xml -p no:unraisableexception -p no:threadexception"
         run: /usr/bin/time -v -o rest-python-spm-resources.txt uv run --no-sync make test-other-python-spm
       - name: Run remaining Python tests
         if: always()
         env:
-          PYTEST_ADDOPTS: "--durations=0 -v --junitxml=rest-python-other.xml -p no:unraisableexception -p no:threadexception"
-        run: /usr/bin/time -v -o rest-python-other-resources.txt uv run --no-sync make test-other-python-rest
+          PYTEST_ADDOPTS: "--durations=0 -v -p no:unraisableexception -p no:threadexception"
+        run: uv run --no-sync make test-other-python-rest REST_REPORT_DIR=.
       - name: Run tests/variables YAML tests
         if: always()
         env:
@@ -320,11 +322,9 @@ jobs:
         with:
           name: rest-test-timings
           path: |
-            rest-python-spm.xml
-            rest-python-other.xml
+            rest-python-*.xml
             rest-variables.xml
-            rest-python-spm-resources.txt
-            rest-python-other-resources.txt
+            rest-python-*-resources.txt
             rest-variables-resources.txt
   Microsimulation:
     needs: ReleaseLock

diff --git a/Makefile b/Makefile
@@ -16,10 +16,53 @@ BATCH := python policyengine_us/tests/test_batched.py
 TESTS := policyengine_us/tests
 # Run the expensive SPM construction/isolation tests in a separate process
 # before the remaining files on the same CI runner, releasing their heap.
-# The remaining group discovers every other Python test, including future files.
+# The remaining Python tests then run in REST_PYTHON_GROUPS below.
 REST_SPM_TESTS := $(TESTS)/core/test_spm_policy_family.py \
 	$(TESTS)/core/test_spm_simulation_isolation.py \
 	$(TESTS)/core/test_spm_system.py
+# The remaining Python tests run as one pytest process per group, one after
+# another, so each exit releases its heap before the next group starts. As a
+# single process they peaked at 15.7 GB RSS on the 16 GB runner, with 1.5M
+# major page faults (CI run 37198184683). Four runs on 2026-10-04, such as
+# 37202679009, then lost the runner 1,424-1,454 tests into the step, inside
+# test_formulas_do_not_write_into_cached_arrays or at the start of
+# test_md_poverty_line_credit_invariants: "The runner has received a shutdown
+# signal", as test-yaml-reform records for a process past 16 GB.
+# Each file runs in the first group whose paths hold it: the listed heavy
+# modules, then core/, then policy/, and remaining takes every other file.
+# code_health/test_rest_python_groups.py checks that the groups together
+# collect each file of the old single process exactly once.
+# Test time per group, from run 37198184683 (1,453 s in all) plus the three
+# core modules added since, timed in runs 37171744596, 37196338557 and
+# 37198179476 and scaled to the speed of 37198184683:
+#   cached-arrays (403 s): test_formulas_do_not_write_into_cached_arrays.
+#   heavy-a (350 s): md_poverty_line_credit_invariants (134 s),
+#     ald_determinism (94 s), dependent_net_investment_income_invariants
+#     (64 s), ssi_state_supplement_medicaid_dependency (59 s).
+#   heavy-b (345 s): marginal_tax_rate_coverage (124 s),
+#     multi_year_simulation (88 s), behavioral_response_measurements (84 s),
+#     dependent_losses_agi_invariants (49 s).
+#   core (142 s), policy (185 s), remaining (239 s).
+REST_PYTHON_GROUPS := cached-arrays heavy-a heavy-b core policy remaining
+REST_CACHED_ARRAY_TESTS := $(TESTS)/test_formulas_do_not_write_into_cached_arrays.py
+REST_HEAVY_A_TESTS := $(TESTS)/test_md_poverty_line_credit_invariants.py \
+	$(TESTS)/core/test_ald_determinism.py \
+	$(TESTS)/core/test_dependent_net_investment_income_invariants.py \
+	$(TESTS)/core/test_ssi_state_supplement_medicaid_dependency.py
+REST_HEAVY_B_TESTS := $(TESTS)/core/test_marginal_tax_rate_coverage.py \
+	$(TESTS)/core/test_multi_year_simulation.py \
+	$(TESTS)/core/test_behavioral_response_measurements.py \
+	$(TESTS)/core/test_dependent_losses_agi_invariants.py
+REST_LISTED_TESTS := $(REST_CACHED_ARRAY_TESTS) $(REST_HEAVY_A_TESTS) $(REST_HEAVY_B_TESTS)
+REST_IGNORES := $(addprefix --ignore=,$(TESTS)/policy/contrib $(TESTS)/microsimulation $(REST_SPM_TESTS))
+# $(call rest_pytest,<group>,<paths>,<more paths to ignore>) is the pytest
+# command for one group. CI sets REST_REPORT_DIR, and each group then writes
+# its own JUnit XML and GNU time report there: rest-python-<group>.xml and
+# rest-python-<group>-resources.txt. Unset, it is plain pytest, since macOS
+# /usr/bin/time has no -v.
+rest_pytest = $(strip $(if $(REST_REPORT_DIR),/usr/bin/time -v -o $(REST_REPORT_DIR)/rest-python-$(1)-resources.txt) \
+	pytest $(2) --maxfail=0 $(REST_IGNORES) $(addprefix --ignore=,$(3)) \
+	$(if $(REST_REPORT_DIR),--junitxml=$(REST_REPORT_DIR)/rest-python-$(1).xml))
 
 all: build
 format:
@@ -185,8 +228,25 @@ test-other-python:
 	pytest policyengine_us/tests/ --maxfail=0 --ignore=$(TESTS)/policy/contrib --ignore=$(TESTS)/microsimulation
 test-other-python-spm:
 	pytest $(REST_SPM_TESTS) --maxfail=0
+# Every group runs even after one fails; the target fails if any group did.
 test-other-python-rest:
-	pytest $(TESTS)/ --maxfail=0 --ignore=$(TESTS)/policy/contrib --ignore=$(TESTS)/microsimulation $(addprefix --ignore=,$(REST_SPM_TESTS))
+	@failed=""; for group in $(REST_PYTHON_GROUPS); do \
+		echo "=== Rest Python group: $$group ==="; \
+		$(MAKE) --no-print-directory test-other-python-rest-$$group || failed="$$failed $$group"; \
+	done; \
+	if [ -n "$$failed" ]; then echo "Failed Rest Python groups:$$failed"; exit 1; fi
+test-other-python-rest-cached-arrays:
+	$(call rest_pytest,cached-arrays,$(REST_CACHED_ARRAY_TESTS))
+test-other-python-rest-heavy-a:
+	$(call rest_pytest,heavy-a,$(REST_HEAVY_A_TESTS))
+test-other-python-rest-heavy-b:
+	$(call rest_pytest,heavy-b,$(REST_HEAVY_B_TESTS))
+test-other-python-rest-core:
+	$(call rest_pytest,core,$(TESTS)/core,$(REST_LISTED_TESTS))
+test-other-python-rest-policy:
+	$(call rest_pytest,policy,$(TESTS)/policy,$(REST_LISTED_TESTS))
+test-other-python-rest-remaining:
+	$(call rest_pytest,remaining,$(TESTS)/,$(REST_LISTED_TESTS) $(TESTS)/core $(TESTS)/policy)
 test-microsimulation:
 	pytest $(TESTS)/microsimulation --maxfail=0
 # Local convenience: run both halves back to back.

diff --git a/changelog.d/ci-split-rest-job.changed.md b/changelog.d/ci-split-rest-job.changed.md
@@ -0,0 +1 @@
+Run the Full Suite - Rest job's remaining Python tests as six sequential pytest processes instead of one, each with its own JUnit and memory report, to keep each process well below the 16 GB runner's memory; a code-health test checks that the groups run every test file of the old process exactly once.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Run the Full Suite - Rest job's remaining Python tests as six sequential pytest processes instead of one, each with its own JUnit and memory report, to keep each process well below the 16 GB runner's memory; a code-health test checks that the groups run every test file of the old process exactly once.