diff --git a/.github/workflows/pr_code_changes.yaml b/.github/workflows/pr_code_changes.yaml index 38fddb6ee7..e959391554 100644 --- a/.github/workflows/pr_code_changes.yaml +++ b/.github/workflows/pr_code_changes.yaml @@ -162,7 +162,7 @@ jobs: run: uv pip install policyengine --system - name: Test documentation builds run: make documentation - # The YAML tests, the Python tests on four runners, and the check that + # The YAML tests, the Python tests on five runners, and the check that # every test ran once and passed, in the locked environment. As one job # they took 59 minutes on 10 October 2026 (run 38075384588), and runs were # cancelled at a 60-minute limit (issues #2099 and #2257). The release diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index e0ea3c5f84..dddd56c9d2 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -4,10 +4,10 @@ # publish. All three run the same jobs in the same locked environment. # # Jobs: the YAML policy tests (which also list every Python test), the Python -# tests split over four runners, and Result, which fails unless every listed +# tests split over five runners, and Result, which fails unless every listed # Python test ran in exactly one shard and passed, apart from the outcomes # accepted in .github/test_outcome_allowlist.toml. A caller that names its job -# Test gets the checks "Test / YAML", "Test / Python K/4" and "Test / Result"; +# Test gets the checks "Test / YAML", "Test / Python K/5" and "Test / Result"; # "Test / Result" is the one to require. name: Test suite @@ -122,17 +122,23 @@ jobs: # the 16 GB runners peaked at 15.6 GB, 283 MB short of running out # (run 38156967136; two shards of an earlier run lost their runners). # With four, each worker runs about a quarter fewer tests. + # + # Five, not four, for time: on 20 and 21 October shard 1/4 took 50 to + # 52 minutes on unrelated pull requests, close to the timeout. A pull + # request adding about a fifth more parameters (#1855) hit the timeout + # on shard 1/4 twice, and its other shards took 48 to 59 minutes (run + # 39568525017). name: Python ${{ matrix.shard }}/${{ strategy.job-total }} if: ${{ !inputs.skip }} runs-on: ubuntu-24.04-arm # The shards took 15, 20 and 15 minutes on 10 October (run - # 38085920460). 60 minutes leaves room for a slow runner and still - # stops a hung shard. + # 38085920460), and up to 52 minutes with four shards on 21 October. + # 60 minutes leaves room for a slow runner and still stops a hung shard. timeout-minutes: 60 strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4] + shard: [1, 2, 3, 4, 5] steps: - name: Checkout repo uses: actions/checkout@v6 diff --git a/Makefile b/Makefile index 44b7ac8e29..eee5b42189 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-yaml: # Every Python test, serially. To spread them over four cores add # PYTEST_ARGS="-n 4 --dist worksteal" (pytest-xdist); to run what one CI -# shard runs, add "--shard K/4" (policyengine_uk/tests/conftest.py). CI runs +# shard runs, add "--shard K/5" (policyengine_uk/tests/conftest.py). CI runs # these tests through .github/workflows/test-suite.yaml, in the locked # environment (uv sync --frozen --extra dev), and also checks each test's # outcome against .github/test_outcome_allowlist.toml. diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index b506b44125..9e4df81565 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -86,10 +86,10 @@ the same suite, tree; a release tests the wheel it is about to publish, installed on its own. The suite runs the YAML policy tests in one job and splits the Python tests -over four runners. Each runner runs one shard: +over five runners. Each runner runs one shard: ```bash -pytest policyengine_uk/tests -n 4 --dist worksteal --shard K/4 -o xfail_strict=true +pytest policyengine_uk/tests -n 4 --dist worksteal --shard K/5 -o xfail_strict=true ``` `--shard K/N` is defined in `policyengine_uk/tests/conftest.py`. It keeps the