Run make test as CI's bounded batches instead of one process over the whole tree

CLAUDE.md

@@ -28,7 +28,9 @@ pip install -e .[dev]
 # Format code
 make format  # Runs ruff format
 
-# Run all tests
+# Run all tests: CI's suites, one bounded subprocess at a time. Never point
+# one `policyengine-core test` process at a whole directory tree or hundreds
+# of files (see CONTRIBUTING.md, "Memory: running suites locally").
 make test
 
 # Run specific test file or directory
@@ -37,8 +39,9 @@ pytest policyengine_us/tests/path/to/test_file.py
 # Run specific test function
 pytest policyengine_us/tests/path/to/test_file.py::test_function_name
 
-# Run specific YAML tests
-policyengine-core test path/to/tests -c policyengine_us [-v]
+# Run specific YAML tests (a few files; for a folder use test_batched.py)
+policyengine-core test path/to/test.yaml -c policyengine_us [-v]
+uv run python policyengine_us/tests/test_batched.py path/to/folder --mode per-subdir --workers 1
 
 # Run microsimulation test
 pytest policyengine_us/tests/microsimulation/test_microsim.py

CONTRIBUTING.md

@@ -17,6 +17,27 @@ uv run policyengine-core test policyengine_us/tests/path/to/test.yaml -c policye
 uv run pytest policyengine_us/tests/path/to/test_file.py::test_name -v
 ```
 
+### Memory: running suites locally
+
+`make test` runs the same suites as CI, one batched subprocess at a time
+(`policyengine_us/tests/test_batched.py`). CI sizes those batches to stay
+under about 8 GB each on 16 GB runners, so this is the safe way to run
+everything on a laptop.
+
+- One area: its `make test-yaml-*` target, or
+  `uv run python policyengine_us/tests/test_batched.py <dir> --mode per-subdir --workers 1`.
+- A few files: `uv run policyengine-core test <files> -c policyengine_us`.
+- Never give one `policyengine-core test` process a whole directory tree or
+  hundreds of files. The YAML runner keeps every case's simulation, a full
+  copy of the tax-benefit system for each distinct `reforms` /
+  dotted-parameter combination (about 1.2 GB each), and a copy of the
+  parameter tree for every date a case asked about, all for the life of the
+  process. A 1,500-file baseline run reached 118 GB on 2026-10-02.
+  PolicyEngine/policyengine-core#569 and #570 (open drafts) would bound
+  this; until they are released, keep each process to a bounded batch.
+- Run one large suite at a time: don't start several `make test-yaml-*`
+  targets, or several `test_batched.py` runs, in parallel.
+
 Python 3.9–3.14 (`requires-python = ">=3.9,<3.15"`; CI smoke-imports on every minor). Default branch: `main`.
 
 ## Writing variables and programs

Makefile

@@ -70,10 +70,40 @@ format:
 	uv run ruff check .
 install:
 	pip install -e .[dev]
-test:
-	pytest $(TESTS)/ --maxfail=0
-	coverage run -a --branch -m policyengine_core.scripts.policyengine_command test $(TESTS)/policy/ -c policyengine_us
-	coverage xml -i
+# Run every target serially, whatever -j says: several YAML suites resident
+# at once is the memory blow-up `make test` exists to prevent. GNU make 3.81
+# (macOS) supports only this whole-file form. CI calls single targets.
+.NOTPARALLEL:
+
+# Full local suite: the same suites CI runs, one bounded subprocess at a time.
+# Never hand the whole policy tree, or a long file list, to one
+# `policyengine-core test` process: a 1,500-file baseline run reached 118 GB
+# on a 128 GB Mac on 2026-10-02 (see CONTRIBUTING.md, "Memory").
+test: test-other-python-spm test-other-python-rest test-microsimulation test-policy-contrib-python test-yaml
+# Every YAML suite in .github/workflows/pr.yaml, in sequence. .NOTPARALLEL
+# below keeps Make to one prerequisite at a time even under `make -j` or a
+# -j in MAKEFLAGS, and each target already runs --workers 1 (partners runs
+# two small batches at once), so at most one large batch is resident.
+# test_make_test_matches_ci.py keeps this list in step with CI.
+test-yaml: test-yaml-no-structural-states \
+	test-yaml-no-structural-other-irs \
+	test-yaml-no-structural-other-household \
+	test-yaml-no-structural-other-ssa-usda \
+	test-yaml-no-structural-other-rest-a \
+	test-yaml-no-structural-other-rest-b \
+	test-yaml-contrib-hhs \
+	test-yaml-reform \
+	test-yaml-no-structural-other-partners \
+	test-yaml-structural-heavy-shard-1 \
+	test-yaml-structural-heavy-shard-2 \
+	test-yaml-structural-heavy-shard-3 \
+	test-yaml-structural-heavy-shard-4 \
+	test-yaml-structural-other \
+	test-yaml-structural-other-shard-2a \
+	test-yaml-structural-other-shard-2b \
+	test-yaml-structural-other-shard-3 \
+	test-yaml-structural-congress \
+	test-yaml-variables
 test-yaml-structural:
 	$(BATCH) $(TESTS)/policy/contrib --exclude states
 test-yaml-structural-heavy:

changelog.d/bounded-local-test-runs.changed.md

@@ -0,0 +1 @@
+Make `make test` run the same batched suites as CI, one bounded subprocess at a time, instead of the whole policy tree in one `policyengine-core test` process, and document the safe local invocation.

policyengine_us/tests/code_health/test_make_test_matches_ci.py

@@ -0,0 +1,96 @@
+"""`make test` runs what CI runs, and only in bounded subprocesses.
+
+CI splits the suites into batched subprocesses so none outgrows a 16 GB
+runner. `make test` used to run the whole policy tree in one
+`policyengine-core test` process instead, and a 1,500-file baseline run of
+that shape reached 118 GB on a 128 GB Mac (2026-10-02). These checks keep the
+local full run equal to CI's suites and free of single-process tree runs.
+"""
+
+import re
+import shutil
+import subprocess
+from functools import lru_cache
+from pathlib import Path
+
+import pytest
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+WORKFLOW = REPO_ROOT / ".github" / "workflows" / "pr.yaml"
+SHARD = re.compile(r"\s+--shard\s+\d+/\d+")
+MAKE_TARGET = re.compile(r"\bmake\s+([\w-]+)")
+
+pytestmark = pytest.mark.skipif(
+    shutil.which("make") is None or not WORKFLOW.exists(),
+    reason="needs make and the repository's CI workflow",
+)
+
+
+@lru_cache(maxsize=None)
+def make_commands(target: str) -> frozenset:
+    """The shell commands ``make <target>`` would run, comments dropped."""
+    output = subprocess.run(
+        ["make", "-n", target],
+        cwd=REPO_ROOT,
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout
+    return frozenset(
+        line.strip()
+        for line in output.splitlines()
+        if line.strip() and not line.strip().startswith("#")
+    )
+
+
+def ci_suites():
+    """Every Make target and batch-runner command the PR workflow runs."""
+    targets, commands = set(), set()
+    workflow = yaml.safe_load(WORKFLOW.read_text())
+    for job in workflow["jobs"].values():
+        matrix = (job.get("strategy") or {}).get("matrix") or {}
+        for entry in matrix.get("include") or []:
+            if "target" in entry:
+                targets.add(entry["target"])
+            if "cmd" in entry:
+                cmd = entry["cmd"]
+                targets.update(MAKE_TARGET.findall(cmd))
+                if "test_batched.py" in cmd:
+                    commands.add(cmd.strip())
+        for step in job.get("steps") or []:
+            for target in MAKE_TARGET.findall(step.get("run") or ""):
+                if not target.startswith("$"):
+                    targets.add(target)
+    return {t for t in targets if t.startswith("test")}, commands
+
+
+def test_make_test_runs_every_ci_suite():
+    local = make_commands("test")
+    targets, commands = ci_suites()
+    assert targets, "found no Make targets in the CI workflow"
+    missing = {
+        target: sorted(make_commands(target) - local)
+        for target in sorted(targets)
+        if make_commands(target) - local
+    }
+    # A sharded CI command runs one shard; locally the unsharded command
+    # runs every shard of the same batches.
+    missing.update(
+        {cmd: [] for cmd in sorted(commands) if SHARD.sub("", cmd) not in local}
+    )
+    assert not missing, f"make test skips CI suites: {missing}"
+
+
+def test_make_test_never_runs_a_whole_tree_in_one_process():
+    single_process = [
+        command
+        for command in make_commands("test")
+        if re.search(
+            r"policyengine(-core|_core\.scripts\.policyengine_command) test", command
+        )
+    ]
+    assert not single_process, (
+        "run YAML suites through policyengine_us/tests/test_batched.py, "
+        f"not one policyengine-core process: {single_process}"
+    )