From ec60a59ad49597720518cc4476e3302276510822 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 13:07:07 -0400 Subject: [PATCH 1/4] Skip, rather than fail, versioning for superseded merges When several PRs merge before the versioning job gets a runner, every job but the newest commits a version bump on a stale parent and fails on push. Oct 6 00:00Z to Oct 8 16:28Z, 47 of 51 failed Push runs were exactly this: in all 47, main had already moved past the job's commit before the job started, and in every successful versioning run it had not. The newest merge's bump consumes all pending changelog fragments (for example, 7c857cf439 removed the three fragments from the 13:02Z burst on Oct 8), so the older jobs had nothing to do. The job now reads main's head first. If main has moved past github.sha it records a notice and skips the remaining steps, so the run succeeds and "Push" failures on main again mean something. If the lookup fails, the job proceeds as before. The job is not cancelled and keeps no concurrency group. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/push.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 3d924078e44..50b9a4a2d88 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -63,29 +63,57 @@ jobs: && !(github.event.head_commit.message == 'Update PolicyEngine US') runs-on: ubuntu-latest steps: + # When several PRs merge before this job gets a runner, only the newest + # merge can push a version bump: the others would commit on a stale + # parent and fail on push. The newest merge's job consumes every pending + # changelog fragment (towncrier builds all of changelog.d), so an older + # job has nothing left to do. It exits successfully with a notice instead + # of failing. If the lookup itself fails, the job proceeds as before. + - name: Skip if a newer push to main superseded this commit + id: head + env: + GH_TOKEN: ${{ github.token }} + run: | + if ! head=$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq .object.sha); then + echo "::warning::Could not read the main branch head; running versioning anyway." + head="$GITHUB_SHA" + fi + if [ "$head" = "$GITHUB_SHA" ]; then + echo "current=true" >> "$GITHUB_OUTPUT" + else + echo "current=false" >> "$GITHUB_OUTPUT" + echo "::notice title=Versioning superseded::main is at ${head}, not ${GITHUB_SHA}. That newer push's versioning job bumps the version once and includes this commit's changelog fragments." + fi - name: Generate GitHub App token + if: steps.head.outputs.current == 'true' id: app-token uses: actions/create-github-app-token@v3 with: app-id: ${{ secrets.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} - uses: actions/checkout@v6 + if: steps.head.outputs.current == 'true' with: fetch-depth: 0 token: ${{ steps.app-token.outputs.token }} - name: Install uv + if: steps.head.outputs.current == 'true' uses: astral-sh/setup-uv@v8.1.0 with: version: "0.12.13" - name: Set up Python + if: steps.head.outputs.current == 'true' uses: actions/setup-python@v6 with: python-version: 3.14 - name: Check committed registry lock before bump + if: steps.head.outputs.current == 'true' run: python .github/release_lock.py --committed - name: Bump version once + if: steps.head.outputs.current == 'true' run: python .github/bump_version.py - name: Build changelog with isolated registry tooling + if: steps.head.outputs.current == 'true' run: | python - <<'PYTHON' import os @@ -113,8 +141,10 @@ jobs: ) PYTHON - name: Refresh and check release version in registry lock + if: steps.head.outputs.current == 'true' run: python .github/release_lock.py --refresh - name: Update changelog + if: steps.head.outputs.current == 'true' uses: EndBug/add-and-commit@v10 with: add: "pyproject.toml uv.lock CHANGELOG.md changelog.d" From 77cef0be0905aa839c0d675b3911e1dedde2ab80 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 13:07:33 -0400 Subject: [PATCH 2/4] Bound every Push job with a timeout from observed durations Five Push jobs had no timeout-minutes and so inherited GitHub's 360-minute default. Each now has a limit well above its Oct 6-8 maximum: ReleaseLock, ReleaseLockTests and Lint 10 min (max 0.4, 1.4 and 0.7), versioning 20 (max 12) and Publish 20 (max 1.3). A ReleaseLockTests run that hung for 45 min (37650158537) would have stopped at 10. The push Rest job's limit rises from 60 to 90 minutes, matching the PR Rest job after #9785. On push its p50 was 51 min and its successful max 58.8, and run 37719726248 hit the 60-minute limit, which skipped that version's Publish. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/push.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 50b9a4a2d88..9b67da41675 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -10,6 +10,8 @@ jobs: (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') runs-on: ubuntu-latest + # Oct 6-8: 29 runs, max 0.4 min. + timeout-minutes: 10 steps: - uses: actions/checkout@v6 - name: Set up Python @@ -25,6 +27,8 @@ jobs: ReleaseLockTests: name: Release lock guard tests runs-on: ubuntu-latest + # Oct 6-8: 104 successful runs, max 1.4 min; one hung run failed at 45 min. + timeout-minutes: 10 steps: - uses: actions/checkout@v6 - name: Set up Python @@ -42,6 +46,8 @@ jobs: Lint: needs: ReleaseLock runs-on: ubuntu-latest + # Oct 6-8: 28 runs, max 0.7 min. + timeout-minutes: 10 if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') @@ -62,6 +68,8 @@ jobs: (github.repository == 'PolicyEngine/policyengine-us') && !(github.event.head_commit.message == 'Update PolicyEngine US') runs-on: ubuntu-latest + # Oct 6-8: successful runs max 4.5 min, any run max 12 min. + timeout-minutes: 20 steps: # When several PRs merge before this job gets a runner, only the newest # merge can push a version bump: the others would commit on a stale @@ -303,7 +311,10 @@ jobs: needs: ReleaseLock name: Full Suite - Rest (Python + variables) runs-on: ubuntu-latest - timeout-minutes: 60 + # Matches the PR Rest job (python-tests.yaml, raised to 90 in #9785). + # Oct 6-8 on push: p50 51 min, max success 58.8 min, and one run hit the + # old 60-minute limit (37719726248), which skipped that version's Publish. + timeout-minutes: 90 if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') @@ -404,6 +415,8 @@ jobs: simulation-*-resources.txt Publish: runs-on: ubuntu-24.04 + # Oct 6-8: 25 runs, max 1.3 min. + timeout-minutes: 20 if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') From 32905c36d5c04ebe8cb0afb2d4d86667c6337ab1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 15:20:12 -0400 Subject: [PATCH 3/4] Add changelog fragment for the Push versioning and timeout fixes Co-Authored-By: Claude Opus 5.5 --- changelog.d/ci-push-versioning-timeouts.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/ci-push-versioning-timeouts.fixed.md diff --git a/changelog.d/ci-push-versioning-timeouts.fixed.md b/changelog.d/ci-push-versioning-timeouts.fixed.md new file mode 100644 index 00000000000..80e9bf66857 --- /dev/null +++ b/changelog.d/ci-push-versioning-timeouts.fixed.md @@ -0,0 +1 @@ +Skip, rather than fail, the Push workflow's versioning job when a newer push to main has superseded its commit, and give every Push job a timeout; the push Rest job's limit rises from 60 to 90 minutes to match the PR job. From 59d4fbd54d8fbb448fd0bc3289983eaec5fe2cec Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 20:16:07 -0400 Subject: [PATCH 4/4] Skip superseded versioning only when main strictly contains the commit The skip compared main's head SHA with this commit for equality, so any mismatch skipped, including a stale read that returned an older head. If the newest merge's job got such a read, every job would skip and no version bump would land. The step now asks GitHub's compare API for the status of this commit against main and skips only on "ahead" (this commit is an ancestor of main and main has more). Identical, behind, diverged, an empty answer or a failed lookup all run versioning as before. A new code-health test runs the step's script against a stub gh for every compare status and for a failed lookup, and checks that every later step in the job is gated on the result. Replacing the condition with a looser one fails five of its nine cases. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/push.yaml | 23 +++-- .../test_push_versioning_superseded.py | 95 +++++++++++++++++++ 2 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 policyengine_us/tests/code_health/test_push_versioning_superseded.py diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 9b67da41675..0f86cc1374d 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -75,22 +75,25 @@ jobs: # merge can push a version bump: the others would commit on a stale # parent and fail on push. The newest merge's job consumes every pending # changelog fragment (towncrier builds all of changelog.d), so an older - # job has nothing left to do. It exits successfully with a notice instead - # of failing. If the lookup itself fails, the job proceeds as before. - - name: Skip if a newer push to main superseded this commit + # job has nothing left to do. It skips only when GitHub's compare API + # says main is strictly ahead of this commit (status "ahead": this + # commit is an ancestor of main and main has more). Any other answer + # (identical, behind, diverged) or a failed lookup runs versioning as + # before, so a stale read can never suppress the newest merge's bump. + - name: Skip if main already contains this commit and more id: head env: GH_TOKEN: ${{ github.token }} run: | - if ! head=$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq .object.sha); then - echo "::warning::Could not read the main branch head; running versioning anyway." - head="$GITHUB_SHA" + if ! status=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${GITHUB_SHA}...main" --jq .status); then + echo "::warning::Could not compare this commit with main; running versioning anyway." + status=unknown fi - if [ "$head" = "$GITHUB_SHA" ]; then - echo "current=true" >> "$GITHUB_OUTPUT" - else + if [ "$status" = "ahead" ]; then echo "current=false" >> "$GITHUB_OUTPUT" - echo "::notice title=Versioning superseded::main is at ${head}, not ${GITHUB_SHA}. That newer push's versioning job bumps the version once and includes this commit's changelog fragments." + echo "::notice title=Versioning superseded::main already contains ${GITHUB_SHA} and newer commits. The versioning job for main's newest merge bumps the version once with every pending changelog fragment, including this commit's. If that job fails, re-run it." + else + echo "current=true" >> "$GITHUB_OUTPUT" fi - name: Generate GitHub App token if: steps.head.outputs.current == 'true' diff --git a/policyengine_us/tests/code_health/test_push_versioning_superseded.py b/policyengine_us/tests/code_health/test_push_versioning_superseded.py new file mode 100644 index 00000000000..11f2e1907b7 --- /dev/null +++ b/policyengine_us/tests/code_health/test_push_versioning_superseded.py @@ -0,0 +1,95 @@ +"""The Push versioning job skips only when main strictly contains its commit. + +The skip step's shell script is run against a stub `gh` for every status +GitHub's compare API can return, plus a failed lookup. Versioning must run +unless the status is exactly "ahead", so a stale or failed read can never +suppress the newest merge's version bump. +""" + +import os +import stat +import subprocess +from pathlib import Path + +import pytest +import yaml + +WORKFLOW = Path(__file__).resolve().parents[3] / ".github" / "workflows" / "push.yaml" +SHA = "a" * 40 + + +def _versioning(): + return yaml.safe_load(WORKFLOW.read_text())["jobs"]["versioning"] + + +def _skip_step(): + steps = [s for s in _versioning()["steps"] if s.get("id") == "head"] + assert len(steps) == 1 + return steps[0] + + +def _run(tmp_path, gh_body): + gh = tmp_path / "bin" / "gh" + gh.parent.mkdir() + gh.write_text("#!/bin/sh\n" + gh_body + "\n") + gh.chmod(gh.stat().st_mode | stat.S_IEXEC) + output = tmp_path / "output" + output.write_text("") + env = { + **os.environ, + "PATH": f"{gh.parent}{os.pathsep}{os.environ['PATH']}", + "GITHUB_REPOSITORY": "PolicyEngine/policyengine-us", + "GITHUB_SHA": SHA, + "GITHUB_OUTPUT": str(output), + } + result = subprocess.run( + ["bash", "-e", "-c", _skip_step()["run"]], + env=env, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + lines = [l for l in output.read_text().splitlines() if l.startswith("current=")] + assert len(lines) == 1, lines + return lines[0].split("=", 1)[1], result.stdout + + +@pytest.mark.parametrize( + "status, current", + [ + ("ahead", "false"), + ("identical", "true"), + ("behind", "true"), + ("diverged", "true"), + ("", "true"), + ("null", "true"), + ], +) +def test_skips_only_when_main_is_strictly_ahead(tmp_path, status, current): + got, _ = _run(tmp_path, f'echo "{status}"') + assert got == current + + +def test_failed_lookup_runs_versioning(tmp_path): + got, stdout = _run(tmp_path, "echo boom >&2; exit 1") + assert got == "true" + assert "::warning::" in stdout + + +def test_compare_is_from_this_commit_to_main(tmp_path): + got, _ = _run( + tmp_path, + 'case "$*" in *"compare/' + + SHA + + '...main"*) echo ahead ;; *) echo identical ;; esac', + ) + assert got == "false" + + +def test_every_later_step_is_gated_on_the_skip(): + steps = _versioning()["steps"] + index = next(i for i, s in enumerate(steps) if s.get("id") == "head") + for step in steps[index + 1 :]: + assert step.get("if") == "steps.head.outputs.current == 'true'", step.get( + "name" + )