diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 3d924078e44..0f86cc1374d 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -10,6 +10,8 @@ jobs: (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') runs-on: ubuntu-latest + # Oct 6-8: 29 runs, max 0.4 min. + timeout-minutes: 10 steps: - uses: actions/checkout@v6 - name: Set up Python @@ -25,6 +27,8 @@ jobs: ReleaseLockTests: name: Release lock guard tests runs-on: ubuntu-latest + # Oct 6-8: 104 successful runs, max 1.4 min; one hung run failed at 45 min. + timeout-minutes: 10 steps: - uses: actions/checkout@v6 - name: Set up Python @@ -42,6 +46,8 @@ jobs: Lint: needs: ReleaseLock runs-on: ubuntu-latest + # Oct 6-8: 28 runs, max 0.7 min. + timeout-minutes: 10 if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') @@ -62,30 +68,63 @@ jobs: (github.repository == 'PolicyEngine/policyengine-us') && !(github.event.head_commit.message == 'Update PolicyEngine US') runs-on: ubuntu-latest + # Oct 6-8: successful runs max 4.5 min, any run max 12 min. + timeout-minutes: 20 steps: + # When several PRs merge before this job gets a runner, only the newest + # merge can push a version bump: the others would commit on a stale + # parent and fail on push. The newest merge's job consumes every pending + # changelog fragment (towncrier builds all of changelog.d), so an older + # job has nothing left to do. It skips only when GitHub's compare API + # says main is strictly ahead of this commit (status "ahead": this + # commit is an ancestor of main and main has more). Any other answer + # (identical, behind, diverged) or a failed lookup runs versioning as + # before, so a stale read can never suppress the newest merge's bump. + - name: Skip if main already contains this commit and more + id: head + env: + GH_TOKEN: ${{ github.token }} + run: | + if ! status=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${GITHUB_SHA}...main" --jq .status); then + echo "::warning::Could not compare this commit with main; running versioning anyway." + status=unknown + fi + if [ "$status" = "ahead" ]; then + echo "current=false" >> "$GITHUB_OUTPUT" + echo "::notice title=Versioning superseded::main already contains ${GITHUB_SHA} and newer commits. The versioning job for main's newest merge bumps the version once with every pending changelog fragment, including this commit's. If that job fails, re-run it." + else + echo "current=true" >> "$GITHUB_OUTPUT" + fi - name: Generate GitHub App token + if: steps.head.outputs.current == 'true' id: app-token uses: actions/create-github-app-token@v3 with: app-id: ${{ secrets.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} - uses: actions/checkout@v6 + if: steps.head.outputs.current == 'true' with: fetch-depth: 0 token: ${{ steps.app-token.outputs.token }} - name: Install uv + if: steps.head.outputs.current == 'true' uses: astral-sh/setup-uv@v8.1.0 with: version: "0.12.13" - name: Set up Python + if: steps.head.outputs.current == 'true' uses: actions/setup-python@v6 with: python-version: 3.14 - name: Check committed registry lock before bump + if: steps.head.outputs.current == 'true' run: python .github/release_lock.py --committed - name: Bump version once + if: steps.head.outputs.current == 'true' run: python .github/bump_version.py - name: Build changelog with isolated registry tooling + if: steps.head.outputs.current == 'true' run: | python - <<'PYTHON' import os @@ -113,8 +152,10 @@ jobs: ) PYTHON - name: Refresh and check release version in registry lock + if: steps.head.outputs.current == 'true' run: python .github/release_lock.py --refresh - name: Update changelog + if: steps.head.outputs.current == 'true' uses: EndBug/add-and-commit@v10 with: add: "pyproject.toml uv.lock CHANGELOG.md changelog.d" @@ -273,7 +314,10 @@ jobs: needs: ReleaseLock name: Full Suite - Rest (Python + variables) runs-on: ubuntu-latest - timeout-minutes: 60 + # Matches the PR Rest job (python-tests.yaml, raised to 90 in #9785). + # Oct 6-8 on push: p50 51 min, max success 58.8 min, and one run hit the + # old 60-minute limit (37719726248), which skipped that version's Publish. + timeout-minutes: 90 if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') @@ -374,6 +418,8 @@ jobs: simulation-*-resources.txt Publish: runs-on: ubuntu-24.04 + # Oct 6-8: 25 runs, max 1.3 min. + timeout-minutes: 20 if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') diff --git a/changelog.d/ci-push-versioning-timeouts.fixed.md b/changelog.d/ci-push-versioning-timeouts.fixed.md new file mode 100644 index 00000000000..80e9bf66857 --- /dev/null +++ b/changelog.d/ci-push-versioning-timeouts.fixed.md @@ -0,0 +1 @@ +Skip, rather than fail, the Push workflow's versioning job when a newer push to main has superseded its commit, and give every Push job a timeout; the push Rest job's limit rises from 60 to 90 minutes to match the PR job. diff --git a/policyengine_us/tests/code_health/test_push_versioning_superseded.py b/policyengine_us/tests/code_health/test_push_versioning_superseded.py new file mode 100644 index 00000000000..11f2e1907b7 --- /dev/null +++ b/policyengine_us/tests/code_health/test_push_versioning_superseded.py @@ -0,0 +1,95 @@ +"""The Push versioning job skips only when main strictly contains its commit. + +The skip step's shell script is run against a stub `gh` for every status +GitHub's compare API can return, plus a failed lookup. Versioning must run +unless the status is exactly "ahead", so a stale or failed read can never +suppress the newest merge's version bump. +""" + +import os +import stat +import subprocess +from pathlib import Path + +import pytest +import yaml + +WORKFLOW = Path(__file__).resolve().parents[3] / ".github" / "workflows" / "push.yaml" +SHA = "a" * 40 + + +def _versioning(): + return yaml.safe_load(WORKFLOW.read_text())["jobs"]["versioning"] + + +def _skip_step(): + steps = [s for s in _versioning()["steps"] if s.get("id") == "head"] + assert len(steps) == 1 + return steps[0] + + +def _run(tmp_path, gh_body): + gh = tmp_path / "bin" / "gh" + gh.parent.mkdir() + gh.write_text("#!/bin/sh\n" + gh_body + "\n") + gh.chmod(gh.stat().st_mode | stat.S_IEXEC) + output = tmp_path / "output" + output.write_text("") + env = { + **os.environ, + "PATH": f"{gh.parent}{os.pathsep}{os.environ['PATH']}", + "GITHUB_REPOSITORY": "PolicyEngine/policyengine-us", + "GITHUB_SHA": SHA, + "GITHUB_OUTPUT": str(output), + } + result = subprocess.run( + ["bash", "-e", "-c", _skip_step()["run"]], + env=env, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + lines = [l for l in output.read_text().splitlines() if l.startswith("current=")] + assert len(lines) == 1, lines + return lines[0].split("=", 1)[1], result.stdout + + +@pytest.mark.parametrize( + "status, current", + [ + ("ahead", "false"), + ("identical", "true"), + ("behind", "true"), + ("diverged", "true"), + ("", "true"), + ("null", "true"), + ], +) +def test_skips_only_when_main_is_strictly_ahead(tmp_path, status, current): + got, _ = _run(tmp_path, f'echo "{status}"') + assert got == current + + +def test_failed_lookup_runs_versioning(tmp_path): + got, stdout = _run(tmp_path, "echo boom >&2; exit 1") + assert got == "true" + assert "::warning::" in stdout + + +def test_compare_is_from_this_commit_to_main(tmp_path): + got, _ = _run( + tmp_path, + 'case "$*" in *"compare/' + + SHA + + '...main"*) echo ahead ;; *) echo identical ;; esac', + ) + assert got == "false" + + +def test_every_later_step_is_gated_on_the_skip(): + steps = _versioning()["steps"] + index = next(i for i, s in enumerate(steps) if s.get("id") == "head") + for step in steps[index + 1 :]: + assert step.get("if") == "steps.head.outputs.current == 'true'", step.get( + "name" + )