Cancel superseded full-suite shards on main (decision needed: skips superseded PyPI versions)

.github/workflows/push.yaml

@@ -3,13 +3,26 @@ on:
   push:
     branches: [main]
 
+# Full-suite test jobs (Baseline, HouseholdAPIPartners, Contrib, Rest,
+# Microsimulation) run only on the bot's `Update PolicyEngine US` commit, and
+# each takes a job-level concurrency group of its own, one per matrix shard.
+# When a newer version-bump commit reaches the same shard, the older commit's
+# shard is cancelled: the newer commit contains every change in the older one,
+# so only the newest version is tested and published. The group never covers
+# ReleaseLock, ReleaseLockTests, Lint, versioning or Publish, so those are never
+# cancelled; an older run whose shards were cancelled skips its Publish because
+# a needed job did not succeed. A re-run (run_attempt > 1) waits for the newer
+# shard instead of cancelling it, so re-running an old run cannot cancel the
+# newest release's tests.
 jobs:
   ReleaseLock:
     name: Check committed registry lock
     if: |
       (github.repository == 'PolicyEngine/policyengine-us')
       && (github.event.head_commit.message == 'Update PolicyEngine US')
     runs-on: ubuntu-latest
+    # Oct 6-8: 29 runs, max 0.4 min.
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v6
       - name: Set up Python
@@ -25,6 +38,8 @@ jobs:
   ReleaseLockTests:
     name: Release lock guard tests
     runs-on: ubuntu-latest
+    # Oct 6-8: 104 successful runs, max 1.4 min; one hung run failed at 45 min.
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v6
       - name: Set up Python
@@ -42,6 +57,8 @@ jobs:
   Lint:
     needs: ReleaseLock
     runs-on: ubuntu-latest
+    # Oct 6-8: 28 runs, max 0.7 min.
+    timeout-minutes: 10
     if: |
       (github.repository == 'PolicyEngine/policyengine-us')
       && (github.event.head_commit.message == 'Update PolicyEngine US')
@@ -62,30 +79,60 @@ jobs:
       (github.repository == 'PolicyEngine/policyengine-us')
       && !(github.event.head_commit.message == 'Update PolicyEngine US')
     runs-on: ubuntu-latest
+    # Oct 6-8: successful runs max 4.5 min, any run max 12 min.
+    timeout-minutes: 20
     steps:
+      # When several PRs merge before this job gets a runner, only the newest
+      # merge can push a version bump: the others would commit on a stale
+      # parent and fail on push. The newest merge's job consumes every pending
+      # changelog fragment (towncrier builds all of changelog.d), so an older
+      # job has nothing left to do. It exits successfully with a notice instead
+      # of failing. If the lookup itself fails, the job proceeds as before.
+      - name: Skip if a newer push to main superseded this commit
+        id: head
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if ! head=$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq .object.sha); then
+            echo "::warning::Could not read the main branch head; running versioning anyway."
+            head="$GITHUB_SHA"
+          fi
+          if [ "$head" = "$GITHUB_SHA" ]; then
+            echo "current=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "current=false" >> "$GITHUB_OUTPUT"
+            echo "::notice title=Versioning superseded::main is at ${head}, not ${GITHUB_SHA}. That newer push's versioning job bumps the version once and includes this commit's changelog fragments."
+          fi
       - name: Generate GitHub App token
+        if: steps.head.outputs.current == 'true'
         id: app-token
         uses: actions/create-github-app-token@v3
         with:
           app-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
       - uses: actions/checkout@v6
+        if: steps.head.outputs.current == 'true'
         with:
           fetch-depth: 0
           token: ${{ steps.app-token.outputs.token }}
       - name: Install uv
+        if: steps.head.outputs.current == 'true'
         uses: astral-sh/setup-uv@v8.1.0
         with:
           version: "0.12.13"
       - name: Set up Python
+        if: steps.head.outputs.current == 'true'
         uses: actions/setup-python@v6
         with:
           python-version: 3.14
       - name: Check committed registry lock before bump
+        if: steps.head.outputs.current == 'true'
         run: python .github/release_lock.py --committed
       - name: Bump version once
+        if: steps.head.outputs.current == 'true'
         run: python .github/bump_version.py
       - name: Build changelog with isolated registry tooling
+        if: steps.head.outputs.current == 'true'
         run: |
           python - <<'PYTHON'
           import os
@@ -113,8 +160,10 @@ jobs:
           )
           PYTHON
       - name: Refresh and check release version in registry lock
+        if: steps.head.outputs.current == 'true'
         run: python .github/release_lock.py --refresh
       - name: Update changelog
+        if: steps.head.outputs.current == 'true'
         uses: EndBug/add-and-commit@v10
         with:
           add: "pyproject.toml uv.lock CHANGELOG.md changelog.d"
@@ -128,6 +177,9 @@ jobs:
     name: Full Suite - Baseline (${{ matrix.group }})
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-baseline-${{ matrix.group }}
+      cancel-in-progress: ${{ github.run_attempt == '1' }}
     if: |
       (github.repository == 'PolicyEngine/policyengine-us')
       && (github.event.head_commit.message == 'Update PolicyEngine US')
@@ -187,6 +239,9 @@ jobs:
     name: Household API Partners
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-household-api-partners
+      cancel-in-progress: ${{ github.run_attempt == '1' }}
     if: |
       (github.repository == 'PolicyEngine/policyengine-us')
       && (github.event.head_commit.message == 'Update PolicyEngine US')
@@ -218,6 +273,9 @@ jobs:
     name: Full Suite - Contrib (${{ matrix.group }})
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-contrib-${{ matrix.group }}
+      cancel-in-progress: ${{ github.run_attempt == '1' }}
     if: |
       (github.repository == 'PolicyEngine/policyengine-us')
       && (github.event.head_commit.message == 'Update PolicyEngine US')
@@ -273,7 +331,13 @@ jobs:
     needs: ReleaseLock
     name: Full Suite - Rest (Python + variables)
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    # Matches the PR Rest job (python-tests.yaml, raised to 90 in #9785).
+    # Oct 6-8 on push: p50 51 min, max success 58.8 min, and one run hit the
+    # old 60-minute limit (37719726248), which skipped that version's Publish.
+    timeout-minutes: 90
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-rest
+      cancel-in-progress: ${{ github.run_attempt == '1' }}
     if: |
       (github.repository == 'PolicyEngine/policyengine-us')
       && (github.event.head_commit.message == 'Update PolicyEngine US')
@@ -327,6 +391,9 @@ jobs:
     name: Full Suite - Microsimulation
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-microsimulation
+      cancel-in-progress: ${{ github.run_attempt == '1' }}
     if: |
       (github.repository == 'PolicyEngine/policyengine-us')
       && (github.event.head_commit.message == 'Update PolicyEngine US')
@@ -374,6 +441,8 @@ jobs:
             simulation-*-resources.txt
   Publish:
     runs-on: ubuntu-24.04
+    # Oct 6-8: 25 runs, max 1.3 min.
+    timeout-minutes: 20
     if: |
       (github.repository == 'PolicyEngine/policyengine-us')
       && (github.event.head_commit.message == 'Update PolicyEngine US')

changelog.d/ci-push-shard-concurrency.changed.md

@@ -0,0 +1 @@
+Cancel a superseded full-suite shard in the Push workflow when a newer version-bump commit reaches the same shard; only the newest version is tested and published to PyPI, so superseded versions are skipped.

changelog.d/ci-push-versioning-timeouts.fixed.md

@@ -0,0 +1 @@
+Skip, rather than fail, the Push workflow's versioning job when a newer push to main has superseded its commit, and give every Push job a timeout; the push Rest job's limit rises from 60 to 90 minutes to match the PR job.