diff --git a/Makefile b/Makefile index a7af971e0a0..481a6cdb1ff 100644 --- a/Makefile +++ b/Makefile @@ -32,18 +32,19 @@ REST_SPM_TESTS := $(TESTS)/core/test_spm_policy_family.py \ # modules, then core/, then policy/, and remaining takes every other file. # code_health/test_rest_python_groups.py checks that the groups together # collect each file of the old single process exactly once. -# Test time per group, from run 37198184683 (1,453 s in all) plus the three -# core modules added since, timed in runs 37171744596, 37196338557 and -# 37198179476 and scaled to the speed of 37198184683: -# cached-arrays (403 s): test_formulas_do_not_write_into_cached_arrays. -# heavy-a (350 s): md_poverty_line_credit_invariants (134 s), -# ald_determinism (94 s), dependent_net_investment_income_invariants -# (64 s), ssi_state_supplement_medicaid_dependency (59 s). -# heavy-b (345 s): marginal_tax_rate_coverage (124 s), -# multi_year_simulation (88 s), behavioral_response_measurements (84 s), -# dependent_losses_agi_invariants (49 s). +# CI run 38141803955 measured RSS/wall: cached-arrays 3.3 GB/5:26, +# heavy-a 8.1 GB/9:55 (including pending #10058), heavy-b 8.5 GB/7:31, +# core 13.3 GB/11:51, policy 13.9 GB/8:46, remaining 14.4 GB/22:18. +# JUnit measures per-file elapsed time, not memory. The extra boundaries +# below isolate retained full models and target <= ~10 GB per process; +# verify those estimates against each group's GNU time report in CI. # core (142 s), policy (185 s), remaining (239 s). -REST_PYTHON_GROUPS := cached-arrays heavy-a heavy-b core policy remaining +# SSI resource deeming retains a reformed system at import and builds another +# in its grid fixture. Core reached 13.3 GB in CI run 38141803955 after this +# and three smaller modules arrived; isolate SSI rather than filling heavy-a +# (8.1 GB including the pending capital-gains MTR change). These groups remain +# sequential subprocesses on the existing Rest runner. +REST_PYTHON_GROUPS := cached-arrays heavy-a heavy-b core-heavy core policy-parameters policy remaining-heavy remaining REST_CACHED_ARRAY_TESTS := $(TESTS)/test_formulas_do_not_write_into_cached_arrays.py REST_HEAVY_A_TESTS := $(TESTS)/test_md_poverty_line_credit_invariants.py \ $(TESTS)/core/test_ald_determinism.py \ @@ -53,7 +54,27 @@ REST_HEAVY_B_TESTS := $(TESTS)/core/test_marginal_tax_rate_coverage.py \ $(TESTS)/core/test_multi_year_simulation.py \ $(TESTS)/core/test_behavioral_response_measurements.py \ $(TESTS)/core/test_dependent_losses_agi_invariants.py -REST_LISTED_TESTS := $(REST_CACHED_ARRAY_TESTS) $(REST_HEAVY_A_TESTS) $(REST_HEAVY_B_TESTS) +# SSI runs before the federal-credit fixture and NY/AL pinned-system cache +# are built. Pins have three bounded entries; keep them apart from the NIIT, +# CTC and Medicaid systems below, which stay resident after their files end. +REST_CORE_HEAVY_TESTS := $(TESTS)/core/test_ssi_resource_deeming_invariants.py \ + $(TESTS)/test_federal_credit_limit_order.py \ + $(TESTS)/test_pinned_branch_credit_dependents.py +# These three parameter modules construct full reform trees; one also loads +# a default Microsimulation only to read parameters. Release their heap before +# the remaining policy tests, rather than stacking both in the 13.9 GB group. +REST_POLICY_PARAMETER_TESTS := $(TESTS)/policy/baseline/parameters +# The first four files retain three independent full systems (the two CTC +# files share one). The last two use transient TX simulations and balance +# elapsed time without warming the NY/AL pinned-system cache. +REST_REMAINING_HEAVY_TESTS := $(TESTS)/test_niit_section_911_magi.py \ + $(TESTS)/test_ctc_foreign_earned_income_exclusion.py \ + $(TESTS)/test_ctc_foreign_earned_income_exclusion_property.py \ + $(TESTS)/unit/test_medicaid_abd_ssi_rate_standard_properties.py \ + $(TESTS)/test_capital_gain_distributions_loss_limit.py \ + $(TESTS)/test_schedule_d_filer_net_capital_gain.py +REST_LISTED_TESTS := $(REST_CACHED_ARRAY_TESTS) $(REST_HEAVY_A_TESTS) $(REST_HEAVY_B_TESTS) \ + $(REST_CORE_HEAVY_TESTS) $(REST_POLICY_PARAMETER_TESTS) $(REST_REMAINING_HEAVY_TESTS) REST_IGNORES := $(addprefix --ignore=,$(TESTS)/policy/contrib $(TESTS)/microsimulation $(REST_SPM_TESTS)) # $(call rest_pytest,,,) is the pytest # command for one group. CI sets REST_REPORT_DIR, and each group then writes @@ -271,10 +292,16 @@ test-other-python-rest-heavy-a: $(call rest_pytest,heavy-a,$(REST_HEAVY_A_TESTS)) test-other-python-rest-heavy-b: $(call rest_pytest,heavy-b,$(REST_HEAVY_B_TESTS)) +test-other-python-rest-core-heavy: + $(call rest_pytest,core-heavy,$(REST_CORE_HEAVY_TESTS)) test-other-python-rest-core: $(call rest_pytest,core,$(TESTS)/core,$(REST_LISTED_TESTS)) +test-other-python-rest-policy-parameters: + $(call rest_pytest,policy-parameters,$(REST_POLICY_PARAMETER_TESTS)) test-other-python-rest-policy: $(call rest_pytest,policy,$(TESTS)/policy,$(REST_LISTED_TESTS)) +test-other-python-rest-remaining-heavy: + $(call rest_pytest,remaining-heavy,$(REST_REMAINING_HEAVY_TESTS)) test-other-python-rest-remaining: $(call rest_pytest,remaining,$(TESTS)/,$(REST_LISTED_TESTS) $(TESTS)/core $(TESTS)/policy) test-microsimulation: diff --git a/changelog.d/rebalance-rest-core-group.changed.md b/changelog.d/rebalance-rest-core-group.changed.md new file mode 100644 index 00000000000..9e16ccb4214 --- /dev/null +++ b/changelog.d/rebalance-rest-core-group.changed.md @@ -0,0 +1 @@ +Isolate memory-heavy Rest Python tests in sequential groups on the existing CI runner. diff --git a/policyengine_us/tests/code_health/test_rest_python_groups.py b/policyengine_us/tests/code_health/test_rest_python_groups.py index cb35d7a6294..da2a2045a57 100644 --- a/policyengine_us/tests/code_health/test_rest_python_groups.py +++ b/policyengine_us/tests/code_health/test_rest_python_groups.py @@ -312,17 +312,15 @@ def test_test_files_keep_distinct_import_names(reference): """ -@pytest.mark.parametrize( - "outcomes", - [ - ["0", "1", "0", "killed", "0", "0"], - ["killed", "1", "1", "1", "1", "1"], - ["0"] * 6, - ], - ids=["one-fails-one-killed", "all-fail", "all-pass"], -) -def test_every_group_runs_and_any_failure_fails_the_target(tmp_path, groups, outcomes): - assert len(groups) == len(outcomes), "give each group one outcome" +@pytest.mark.parametrize("scenario", ["one-fails-one-killed", "all-fail", "all-pass"]) +def test_every_group_runs_and_any_failure_fails_the_target(tmp_path, groups, scenario): + outcomes = ["0"] * len(groups) + if scenario == "one-fails-one-killed": + assert len(groups) >= 4, "need groups before and after the failures" + outcomes[1] = "1" + outcomes[3] = "killed" + elif scenario == "all-fail": + outcomes = ["killed"] + ["1"] * (len(groups) - 1) stub = tmp_path / "bin" / "pytest" stub.parent.mkdir() stub.write_text(STUB_PYTEST)