diff --git a/CLAUDE.md b/CLAUDE.md index 20f8b488e81..f9f5a725b4e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,7 +28,9 @@ pip install -e .[dev] # Format code make format # Runs ruff format -# Run all tests +# Run all tests: CI's suites, one bounded subprocess at a time. Never point +# one `policyengine-core test` process at a whole directory tree or hundreds +# of files (see CONTRIBUTING.md, "Memory: running suites locally"). make test # Run specific test file or directory @@ -37,8 +39,9 @@ pytest policyengine_us/tests/path/to/test_file.py # Run specific test function pytest policyengine_us/tests/path/to/test_file.py::test_function_name -# Run specific YAML tests -policyengine-core test path/to/tests -c policyengine_us [-v] +# Run specific YAML tests (a few files; for a folder use test_batched.py) +policyengine-core test path/to/test.yaml -c policyengine_us [-v] +uv run python policyengine_us/tests/test_batched.py path/to/folder --mode per-subdir --workers 1 # Run microsimulation test pytest policyengine_us/tests/microsimulation/test_microsim.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 84f0488c726..b929048a687 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,6 +17,27 @@ uv run policyengine-core test policyengine_us/tests/path/to/test.yaml -c policye uv run pytest policyengine_us/tests/path/to/test_file.py::test_name -v ``` +### Memory: running suites locally + +`make test` runs the same suites as CI, one batched subprocess at a time +(`policyengine_us/tests/test_batched.py`). CI sizes those batches to stay +under about 8 GB each on 16 GB runners, so this is the safe way to run +everything on a laptop. + +- One area: its `make test-yaml-*` target, or + `uv run python policyengine_us/tests/test_batched.py --mode per-subdir --workers 1`. +- A few files: `uv run policyengine-core test -c policyengine_us`. +- Never give one `policyengine-core test` process a whole directory tree or + hundreds of files. The YAML runner keeps every case's simulation, a full + copy of the tax-benefit system for each distinct `reforms` / + dotted-parameter combination (about 1.2 GB each), and a copy of the + parameter tree for every date a case asked about, all for the life of the + process. A 1,500-file baseline run reached 118 GB on 2026-10-02. + PolicyEngine/policyengine-core#569 and #570 (open drafts) would bound + this; until they are released, keep each process to a bounded batch. +- Run one large suite at a time: don't start several `make test-yaml-*` + targets, or several `test_batched.py` runs, in parallel. + Python 3.9–3.14 (`requires-python = ">=3.9,<3.15"`; CI smoke-imports on every minor). Default branch: `main`. ## Writing variables and programs diff --git a/Makefile b/Makefile index d44c7c1b754..a7af971e0a0 100644 --- a/Makefile +++ b/Makefile @@ -70,10 +70,40 @@ format: uv run ruff check . install: pip install -e .[dev] -test: - pytest $(TESTS)/ --maxfail=0 - coverage run -a --branch -m policyengine_core.scripts.policyengine_command test $(TESTS)/policy/ -c policyengine_us - coverage xml -i +# Run every target serially, whatever -j says: several YAML suites resident +# at once is the memory blow-up `make test` exists to prevent. GNU make 3.81 +# (macOS) supports only this whole-file form. CI calls single targets. +.NOTPARALLEL: + +# Full local suite: the same suites CI runs, one bounded subprocess at a time. +# Never hand the whole policy tree, or a long file list, to one +# `policyengine-core test` process: a 1,500-file baseline run reached 118 GB +# on a 128 GB Mac on 2026-10-02 (see CONTRIBUTING.md, "Memory"). +test: test-other-python-spm test-other-python-rest test-microsimulation test-policy-contrib-python test-yaml +# Every YAML suite in .github/workflows/pr.yaml, in sequence. .NOTPARALLEL +# below keeps Make to one prerequisite at a time even under `make -j` or a +# -j in MAKEFLAGS, and each target already runs --workers 1 (partners runs +# two small batches at once), so at most one large batch is resident. +# test_make_test_matches_ci.py keeps this list in step with CI. +test-yaml: test-yaml-no-structural-states \ + test-yaml-no-structural-other-irs \ + test-yaml-no-structural-other-household \ + test-yaml-no-structural-other-ssa-usda \ + test-yaml-no-structural-other-rest-a \ + test-yaml-no-structural-other-rest-b \ + test-yaml-contrib-hhs \ + test-yaml-reform \ + test-yaml-no-structural-other-partners \ + test-yaml-structural-heavy-shard-1 \ + test-yaml-structural-heavy-shard-2 \ + test-yaml-structural-heavy-shard-3 \ + test-yaml-structural-heavy-shard-4 \ + test-yaml-structural-other \ + test-yaml-structural-other-shard-2a \ + test-yaml-structural-other-shard-2b \ + test-yaml-structural-other-shard-3 \ + test-yaml-structural-congress \ + test-yaml-variables test-yaml-structural: $(BATCH) $(TESTS)/policy/contrib --exclude states test-yaml-structural-heavy: diff --git a/changelog.d/bounded-local-test-runs.changed.md b/changelog.d/bounded-local-test-runs.changed.md new file mode 100644 index 00000000000..53e5e4f12a4 --- /dev/null +++ b/changelog.d/bounded-local-test-runs.changed.md @@ -0,0 +1 @@ +Make `make test` run the same batched suites as CI, one bounded subprocess at a time, instead of the whole policy tree in one `policyengine-core test` process, and document the safe local invocation. diff --git a/policyengine_us/tests/code_health/test_make_test_matches_ci.py b/policyengine_us/tests/code_health/test_make_test_matches_ci.py new file mode 100644 index 00000000000..cefa797b67a --- /dev/null +++ b/policyengine_us/tests/code_health/test_make_test_matches_ci.py @@ -0,0 +1,96 @@ +"""`make test` runs what CI runs, and only in bounded subprocesses. + +CI splits the suites into batched subprocesses so none outgrows a 16 GB +runner. `make test` used to run the whole policy tree in one +`policyengine-core test` process instead, and a 1,500-file baseline run of +that shape reached 118 GB on a 128 GB Mac (2026-10-02). These checks keep the +local full run equal to CI's suites and free of single-process tree runs. +""" + +import re +import shutil +import subprocess +from functools import lru_cache +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[3] +WORKFLOW = REPO_ROOT / ".github" / "workflows" / "pr.yaml" +SHARD = re.compile(r"\s+--shard\s+\d+/\d+") +MAKE_TARGET = re.compile(r"\bmake\s+([\w-]+)") + +pytestmark = pytest.mark.skipif( + shutil.which("make") is None or not WORKFLOW.exists(), + reason="needs make and the repository's CI workflow", +) + + +@lru_cache(maxsize=None) +def make_commands(target: str) -> frozenset: + """The shell commands ``make `` would run, comments dropped.""" + output = subprocess.run( + ["make", "-n", target], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout + return frozenset( + line.strip() + for line in output.splitlines() + if line.strip() and not line.strip().startswith("#") + ) + + +def ci_suites(): + """Every Make target and batch-runner command the PR workflow runs.""" + targets, commands = set(), set() + workflow = yaml.safe_load(WORKFLOW.read_text()) + for job in workflow["jobs"].values(): + matrix = (job.get("strategy") or {}).get("matrix") or {} + for entry in matrix.get("include") or []: + if "target" in entry: + targets.add(entry["target"]) + if "cmd" in entry: + cmd = entry["cmd"] + targets.update(MAKE_TARGET.findall(cmd)) + if "test_batched.py" in cmd: + commands.add(cmd.strip()) + for step in job.get("steps") or []: + for target in MAKE_TARGET.findall(step.get("run") or ""): + if not target.startswith("$"): + targets.add(target) + return {t for t in targets if t.startswith("test")}, commands + + +def test_make_test_runs_every_ci_suite(): + local = make_commands("test") + targets, commands = ci_suites() + assert targets, "found no Make targets in the CI workflow" + missing = { + target: sorted(make_commands(target) - local) + for target in sorted(targets) + if make_commands(target) - local + } + # A sharded CI command runs one shard; locally the unsharded command + # runs every shard of the same batches. + missing.update( + {cmd: [] for cmd in sorted(commands) if SHARD.sub("", cmd) not in local} + ) + assert not missing, f"make test skips CI suites: {missing}" + + +def test_make_test_never_runs_a_whole_tree_in_one_process(): + single_process = [ + command + for command in make_commands("test") + if re.search( + r"policyengine(-core|_core\.scripts\.policyengine_command) test", command + ) + ] + assert not single_process, ( + "run YAML suites through policyengine_us/tests/test_batched.py, " + f"not one policyengine-core process: {single_process}" + )