From ec60a59ad49597720518cc4476e3302276510822 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 13:07:07 -0400 Subject: [PATCH 1/5] Skip, rather than fail, versioning for superseded merges When several PRs merge before the versioning job gets a runner, every job but the newest commits a version bump on a stale parent and fails on push. Oct 6 00:00Z to Oct 8 16:28Z, 47 of 51 failed Push runs were exactly this: in all 47, main had already moved past the job's commit before the job started, and in every successful versioning run it had not. The newest merge's bump consumes all pending changelog fragments (for example, 7c857cf439 removed the three fragments from the 13:02Z burst on Oct 8), so the older jobs had nothing to do. The job now reads main's head first. If main has moved past github.sha it records a notice and skips the remaining steps, so the run succeeds and "Push" failures on main again mean something. If the lookup fails, the job proceeds as before. The job is not cancelled and keeps no concurrency group. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/push.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 3d924078e44..50b9a4a2d88 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -63,29 +63,57 @@ jobs: && !(github.event.head_commit.message == 'Update PolicyEngine US') runs-on: ubuntu-latest steps: + # When several PRs merge before this job gets a runner, only the newest + # merge can push a version bump: the others would commit on a stale + # parent and fail on push. The newest merge's job consumes every pending + # changelog fragment (towncrier builds all of changelog.d), so an older + # job has nothing left to do. It exits successfully with a notice instead + # of failing. If the lookup itself fails, the job proceeds as before. + - name: Skip if a newer push to main superseded this commit + id: head + env: + GH_TOKEN: ${{ github.token }} + run: | + if ! head=$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq .object.sha); then + echo "::warning::Could not read the main branch head; running versioning anyway." + head="$GITHUB_SHA" + fi + if [ "$head" = "$GITHUB_SHA" ]; then + echo "current=true" >> "$GITHUB_OUTPUT" + else + echo "current=false" >> "$GITHUB_OUTPUT" + echo "::notice title=Versioning superseded::main is at ${head}, not ${GITHUB_SHA}. That newer push's versioning job bumps the version once and includes this commit's changelog fragments." + fi - name: Generate GitHub App token + if: steps.head.outputs.current == 'true' id: app-token uses: actions/create-github-app-token@v3 with: app-id: ${{ secrets.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} - uses: actions/checkout@v6 + if: steps.head.outputs.current == 'true' with: fetch-depth: 0 token: ${{ steps.app-token.outputs.token }} - name: Install uv + if: steps.head.outputs.current == 'true' uses: astral-sh/setup-uv@v8.1.0 with: version: "0.12.13" - name: Set up Python + if: steps.head.outputs.current == 'true' uses: actions/setup-python@v6 with: python-version: 3.14 - name: Check committed registry lock before bump + if: steps.head.outputs.current == 'true' run: python .github/release_lock.py --committed - name: Bump version once + if: steps.head.outputs.current == 'true' run: python .github/bump_version.py - name: Build changelog with isolated registry tooling + if: steps.head.outputs.current == 'true' run: | python - <<'PYTHON' import os @@ -113,8 +141,10 @@ jobs: ) PYTHON - name: Refresh and check release version in registry lock + if: steps.head.outputs.current == 'true' run: python .github/release_lock.py --refresh - name: Update changelog + if: steps.head.outputs.current == 'true' uses: EndBug/add-and-commit@v10 with: add: "pyproject.toml uv.lock CHANGELOG.md changelog.d" From 77cef0be0905aa839c0d675b3911e1dedde2ab80 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 13:07:33 -0400 Subject: [PATCH 2/5] Bound every Push job with a timeout from observed durations Five Push jobs had no timeout-minutes and so inherited GitHub's 360-minute default. Each now has a limit well above its Oct 6-8 maximum: ReleaseLock, ReleaseLockTests and Lint 10 min (max 0.4, 1.4 and 0.7), versioning 20 (max 12) and Publish 20 (max 1.3). A ReleaseLockTests run that hung for 45 min (37650158537) would have stopped at 10. The push Rest job's limit rises from 60 to 90 minutes, matching the PR Rest job after #9785. On push its p50 was 51 min and its successful max 58.8, and run 37719726248 hit the 60-minute limit, which skipped that version's Publish. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/push.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 50b9a4a2d88..9b67da41675 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -10,6 +10,8 @@ jobs: (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') runs-on: ubuntu-latest + # Oct 6-8: 29 runs, max 0.4 min. + timeout-minutes: 10 steps: - uses: actions/checkout@v6 - name: Set up Python @@ -25,6 +27,8 @@ jobs: ReleaseLockTests: name: Release lock guard tests runs-on: ubuntu-latest + # Oct 6-8: 104 successful runs, max 1.4 min; one hung run failed at 45 min. + timeout-minutes: 10 steps: - uses: actions/checkout@v6 - name: Set up Python @@ -42,6 +46,8 @@ jobs: Lint: needs: ReleaseLock runs-on: ubuntu-latest + # Oct 6-8: 28 runs, max 0.7 min. + timeout-minutes: 10 if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') @@ -62,6 +68,8 @@ jobs: (github.repository == 'PolicyEngine/policyengine-us') && !(github.event.head_commit.message == 'Update PolicyEngine US') runs-on: ubuntu-latest + # Oct 6-8: successful runs max 4.5 min, any run max 12 min. + timeout-minutes: 20 steps: # When several PRs merge before this job gets a runner, only the newest # merge can push a version bump: the others would commit on a stale @@ -303,7 +311,10 @@ jobs: needs: ReleaseLock name: Full Suite - Rest (Python + variables) runs-on: ubuntu-latest - timeout-minutes: 60 + # Matches the PR Rest job (python-tests.yaml, raised to 90 in #9785). + # Oct 6-8 on push: p50 51 min, max success 58.8 min, and one run hit the + # old 60-minute limit (37719726248), which skipped that version's Publish. + timeout-minutes: 90 if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') @@ -404,6 +415,8 @@ jobs: simulation-*-resources.txt Publish: runs-on: ubuntu-24.04 + # Oct 6-8: 25 runs, max 1.3 min. + timeout-minutes: 20 if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') From 32905c36d5c04ebe8cb0afb2d4d86667c6337ab1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 15:20:12 -0400 Subject: [PATCH 3/5] Add changelog fragment for the Push versioning and timeout fixes Co-Authored-By: Claude Opus 5.5 --- changelog.d/ci-push-versioning-timeouts.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/ci-push-versioning-timeouts.fixed.md diff --git a/changelog.d/ci-push-versioning-timeouts.fixed.md b/changelog.d/ci-push-versioning-timeouts.fixed.md new file mode 100644 index 00000000000..80e9bf66857 --- /dev/null +++ b/changelog.d/ci-push-versioning-timeouts.fixed.md @@ -0,0 +1 @@ +Skip, rather than fail, the Push workflow's versioning job when a newer push to main has superseded its commit, and give every Push job a timeout; the push Rest job's limit rises from 60 to 90 minutes to match the PR job. From 88596fac1061ab6c31016c5dc17e322fae2ec0e9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 13:06:03 -0400 Subject: [PATCH 4/5] Cancel superseded full-suite shards on main Each full-suite test job on the bot's version-bump commit now takes a job-level concurrency group per shard (Baseline and Contrib groups include matrix.group), so a newer `Update PolicyEngine US` push cancels only the older run's matching shard. ReleaseLock, ReleaseLockTests, Lint, versioning and Publish have no group and are never cancelled; an older run whose shards are cancelled skips Publish because a needed job did not succeed, and the newer version (a superset of its changes) is published instead. Re-runs (run_attempt > 1) set cancel-in-progress to false, so re-running an old run waits for the newer shard instead of cancelling the newest release. Oct 6 00:00Z to Oct 8 16:28Z: 13 of 28 full-suite push runs had a newer version-bump push created before their last test job finished; per-shard cancellation would have freed about 1,690 runner-minutes. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/push.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 9b67da41675..ae6aad058b5 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -3,6 +3,17 @@ on: push: branches: [main] +# Full-suite test jobs (Baseline, HouseholdAPIPartners, Contrib, Rest, +# Microsimulation) run only on the bot's `Update PolicyEngine US` commit, and +# each takes a job-level concurrency group of its own, one per matrix shard. +# When a newer version-bump commit reaches the same shard, the older commit's +# shard is cancelled: the newer commit contains every change in the older one, +# so only the newest version is tested and published. The group never covers +# ReleaseLock, ReleaseLockTests, Lint, versioning or Publish, so those are never +# cancelled; an older run whose shards were cancelled skips its Publish because +# a needed job did not succeed. A re-run (run_attempt > 1) waits for the newer +# shard instead of cancelling it, so re-running an old run cannot cancel the +# newest release's tests. jobs: ReleaseLock: name: Check committed registry lock @@ -166,6 +177,9 @@ jobs: name: Full Suite - Baseline (${{ matrix.group }}) runs-on: ubuntu-latest timeout-minutes: 60 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-baseline-${{ matrix.group }} + cancel-in-progress: ${{ github.run_attempt == '1' }} if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') @@ -225,6 +239,9 @@ jobs: name: Household API Partners runs-on: ubuntu-latest timeout-minutes: 60 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-household-api-partners + cancel-in-progress: ${{ github.run_attempt == '1' }} if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') @@ -256,6 +273,9 @@ jobs: name: Full Suite - Contrib (${{ matrix.group }}) runs-on: ubuntu-latest timeout-minutes: 60 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-contrib-${{ matrix.group }} + cancel-in-progress: ${{ github.run_attempt == '1' }} if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') @@ -315,6 +335,9 @@ jobs: # Oct 6-8 on push: p50 51 min, max success 58.8 min, and one run hit the # old 60-minute limit (37719726248), which skipped that version's Publish. timeout-minutes: 90 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-rest + cancel-in-progress: ${{ github.run_attempt == '1' }} if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') @@ -368,6 +391,9 @@ jobs: name: Full Suite - Microsimulation runs-on: ubuntu-latest timeout-minutes: 60 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-microsimulation + cancel-in-progress: ${{ github.run_attempt == '1' }} if: | (github.repository == 'PolicyEngine/policyengine-us') && (github.event.head_commit.message == 'Update PolicyEngine US') From 0075b3b65962ac82546db963a282c1e9b0b72431 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 15:21:26 -0400 Subject: [PATCH 5/5] Add changelog fragment for per-shard Push concurrency Co-Authored-By: Claude Opus 5.5 --- changelog.d/ci-push-shard-concurrency.changed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/ci-push-shard-concurrency.changed.md diff --git a/changelog.d/ci-push-shard-concurrency.changed.md b/changelog.d/ci-push-shard-concurrency.changed.md new file mode 100644 index 00000000000..7f7c4259a11 --- /dev/null +++ b/changelog.d/ci-push-shard-concurrency.changed.md @@ -0,0 +1 @@ +Cancel a superseded full-suite shard in the Push workflow when a newer version-bump commit reaches the same shard; only the newest version is tested and published to PyPI, so superseded versions are skipped.