Add an opt-in integration branch so small fixes run the full suite once per batch

.github/actions/candidate-wheel/action.yaml

@@ -22,7 +22,7 @@ runs:
           echo "present=true" >> "$GITHUB_OUTPUT"
         else
           echo "present=false" >> "$GITHUB_OUTPUT"
-          echo "::notice title=Candidate wheel skipped::The PR head predates the release tooling (.github/release_lock.py and .github/release_build.py). Merge main into the branch to build its candidate wheel."
+          echo "::notice title=Candidate wheel skipped::The PR head predates the release tooling (.github/release_lock.py and .github/release_build.py). Merge the PR base branch into the branch to build its candidate wheel."
         fi
     - name: Check committed registry lock
       if: steps.tooling.outputs.present == 'true'

.github/promote_integration.py

@@ -0,0 +1,224 @@
+"""Synchronize integration and open one immutable promotion at a time.
+
+Uses only git, gh and the Python standard library; never imports the model.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import json
+import os
+import subprocess
+
+
+def git(*args: str) -> str:
+    return subprocess.run(
+        ["git", *args], check=True, capture_output=True, text=True
+    ).stdout.strip()
+
+
+def api(repository: str, resource: str, *, fields=None, paginate=False):
+    command = ["gh", "api", f"repos/{repository}/{resource}"]
+    if fields is not None:
+        command.extend(["--method", "POST", "--input", "-"])
+    if paginate:
+        command.extend(["--paginate", "--slurp"])
+    result = subprocess.run(
+        command,
+        input=json.dumps(fields) if fields is not None else None,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    data = json.loads(result.stdout)
+    return [item for page in data for item in page] if paginate else data
+
+
+def ancestor(older: str, newer: str) -> bool:
+    result = subprocess.run(
+        ["git", "merge-base", "--is-ancestor", older, newer],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode not in (0, 1):
+        result.check_returncode()
+    return result.returncode == 0
+
+
+def is_promotion(repository: str, pull: dict) -> bool:
+    head = pull["head"]
+    return (
+        head["ref"].startswith("promote/")
+        and (head.get("repo") or {}).get("full_name", "").lower() == repository.lower()
+    )
+
+
+def recovered_promotions() -> set[int]:
+    path = ".github/promotion-recovered.txt"
+    try:
+        contents = git("show", f"origin/main:{path}")
+    except subprocess.CalledProcessError:
+        # Only an absent file is empty; other Git failures must stop the tick.
+        if git("ls-tree", "origin/main", "--", path):
+            raise
+        return set()
+    numbers = (line.partition("#")[0].strip() for line in contents.splitlines())
+    return {int(number) for number in numbers if number}
+
+
+def latest_merged_promotion(repository: str, recovered: set[int]) -> dict | None:
+    latest = None
+    page = 1
+    while True:
+        pulls = api(
+            repository,
+            "pulls?state=closed&base=main&sort=updated&direction=desc"
+            f"&per_page=100&page={page}",
+        )
+        for pull in pulls:
+            if (
+                pull.get("merged_at")
+                and pull["number"] not in recovered
+                and is_promotion(repository, pull)
+                and (latest is None or pull["merged_at"] > latest["merged_at"])
+            ):
+                latest = pull
+        if len(pulls) < 100:
+            return latest
+        # merged_at <= updated_at for every PR. In updated-descending order,
+        # once this page's oldest update precedes the best merge, no later
+        # page can contain a newer merge (including unacknowledged promotions).
+        if latest and pulls[-1]["updated_at"] < latest["merged_at"]:
+            return latest
+        page += 1
+
+
+def run(repository: str, now: datetime | None = None) -> None:
+    if not git("ls-remote", "--heads", "origin", "refs/heads/integration"):
+        print("::notice::Integration branch is absent; the pilot is inactive.")
+        return
+    git(
+        "fetch",
+        "--no-tags",
+        "origin",
+        "+refs/heads/main:refs/remotes/origin/main",
+        "+refs/heads/integration:refs/remotes/origin/integration",
+    )
+    # Checking for an open promotion and creating one are separate API calls.
+    # The workflow's constant concurrency group serializes scheduled and
+    # dispatched runs so they cannot create competing snapshots.
+    opened = api(repository, "pulls?state=open&base=main&per_page=100", paginate=True)
+    for pull in opened:
+        if is_promotion(repository, pull):
+            print(f"::notice::Promotion #{pull['number']} is open; leaving it frozen.")
+            return
+
+    latest = latest_merged_promotion(repository, recovered_promotions())
+    if latest:
+        try:
+            # GitHub retains the PR head ref after deleting the source branch.
+            git(
+                "fetch",
+                "--no-tags",
+                "origin",
+                f"+refs/pull/{latest['number']}/head:refs/remotes/origin/promotion-head",
+            )
+            preserved = ancestor("origin/promotion-head", "origin/main")
+        except (subprocess.CalledProcessError, OSError) as exc:
+            detail = (
+                (exc.stderr or exc.stdout or str(exc))
+                if isinstance(exc, subprocess.CalledProcessError)
+                else str(exc)
+            )
+            reason = " ".join(detail.splitlines())
+            print(f"::error::Cannot verify promotion #{latest['number']}: {reason}")
+            raise SystemExit(1) from None
+        if not preserved:
+            print(
+                f"::error::Promotion #{latest['number']} was not merged with a merge "
+                "commit and needs manual recovery before integration can be synchronized."
+            )
+            raise SystemExit(1)
+
+    git("checkout", "-B", "integration", "origin/integration")
+    if not ancestor("origin/main", "HEAD"):
+        try:
+            git("merge", "--no-edit", "origin/main")
+        except subprocess.CalledProcessError as exc:
+            git("merge", "--abort")
+            print(exc.stdout or "")
+            print(exc.stderr or "")
+            print(
+                "::warning::Main could not merge into integration. Resolve the "
+                "conflict on integration before the next promotion."
+            )
+            return
+        # A concurrent member merge rejects this ordinary push. Fail the tick
+        # rather than overwrite it or create a snapshot from a stale head.
+        git("push", "origin", "HEAD:refs/heads/integration")
+
+    commits = git("rev-list", "--reverse", "origin/main..HEAD").splitlines()
+    if not commits:
+        print("::notice::Integration has no unpromoted commits; no promotion needed.")
+        return
+    closed = api(
+        repository, "pulls?state=closed&base=integration&per_page=100", paginate=True
+    )
+    members = [
+        pull
+        for pull in closed
+        if pull.get("merged_at") and pull.get("merge_commit_sha") in commits
+    ]
+    entries = git("log", "--reverse", "--format=%H%x09%s", "origin/main..HEAD")
+    snapshot = git("rev-parse", "HEAD")
+    good = git("rev-parse", "origin/main")
+    stamp = (now or datetime.now(timezone.utc)).astimezone(timezone.utc)
+    branch = f"promote/{stamp:%Y%m%dT%H%M%S%fZ}"
+    body = [
+        f"Frozen integration snapshot: `{snapshot}`. Pre-batch main: `{good}`.",
+        "",
+        "Merge this PR with a **merge commit**, not squash or rebase, so each "
+        "fix retains its commits and changelog fragments.",
+        "",
+        "Member PRs:",
+    ]
+    body.extend(f"- #{pull['number']}: {pull['title']}" for pull in members)
+    if not members:
+        body.append("- No associated merged integration PRs; see the commits below.")
+    body.extend(["", "Member commits:"])
+    for entry in entries.splitlines():
+        sha, subject = entry.split("\t", 1)
+        body.append(
+            f"- [{sha[:12]}](https://github.com/{repository}/commit/{sha}) {subject}"
+        )
+    body.extend(
+        [
+            "",
+            "If CI fails, follow CONTRIBUTING.md's red-promotion procedure. "
+            "Close this PR after correcting integration, then dispatch Promote "
+            "integration to cut a new snapshot.",
+        ]
+    )
+    # Creating a ref rejects a name collision instead of moving a snapshot.
+    api(repository, "git/refs", fields={"ref": f"refs/heads/{branch}", "sha": snapshot})
+    pull = api(
+        repository,
+        "pulls",
+        fields={
+            "head": branch,
+            "base": "main",
+            "title": f"Promote integration ({stamp:%Y-%m-%d %H:%M:%S UTC})",
+            "body": "\n".join(body),
+            "draft": False,
+        },
+    )
+    print(f"::notice::Created promotion {pull['html_url']}")
+
+
+if __name__ == "__main__":
+    try:
+        run(os.environ["GITHUB_REPOSITORY"])
+    except subprocess.CalledProcessError as exc:
+        print(exc.stdout or "")
+        print(exc.stderr or "")
+        raise

.github/promotion-recovered.txt

@@ -0,0 +1 @@
+# Reconciled rewritten promotion PRs: one PR number per line, with # comments allowed.

.github/workflows/pr.yaml

@@ -1,8 +1,8 @@
 name: Pull request
 on:
   pull_request:
-    branches: [main]
-    # Draft PRs skip CI; marking a PR ready runs the full suite. Converting a
+    branches: [main, integration]
+    # Draft PRs skip CI; marking a PR ready runs the tier for its base. Converting a
     # PR back to draft starts a run whose jobs all skip, which cancels any
     # in-progress run through the concurrency group below.
     types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
@@ -245,9 +245,11 @@ jobs:
   Baseline:
     name: Baseline tests
     needs: ReleaseLock
+    if: github.base_ref == 'main'
     uses: ./.github/workflows/baseline-tests.yaml
   HouseholdAPIPartners:
     needs: ReleaseLock
+    if: github.base_ref == 'main'
     name: Household API Partners
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -277,8 +279,10 @@ jobs:
   Contrib:
     name: Contrib tests
     needs: ReleaseLock
+    if: github.base_ref == 'main'
     uses: ./.github/workflows/contrib-tests.yaml
   PythonTests:
     name: Python tests
     needs: ReleaseLock
+    if: github.base_ref == 'main'
     uses: ./.github/workflows/python-tests.yaml

.github/workflows/promote-integration.yaml

@@ -0,0 +1,56 @@
+name: Promote integration
+
+on:
+  schedule:
+    - cron: "17 */4 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: promote-integration
+  cancel-in-progress: false
+
+jobs:
+  promote:
+    if: github.repository == 'PolicyEngine/policyengine-us'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v6
+      - name: Check whether the pilot is enabled
+        id: pilot
+        shell: bash
+        run: |
+          set -euo pipefail
+          branch=$(git ls-remote --heads origin refs/heads/integration)
+          if [ -z "$branch" ]; then
+            echo "::notice::Integration branch is absent; the pilot is inactive."
+            echo "enabled=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "enabled=true" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Generate GitHub App token
+        if: steps.pilot.outputs.enabled == 'true'
+        id: app-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          permission-contents: write
+          permission-pull-requests: write
+          permission-workflows: write
+      - uses: actions/checkout@v6
+        if: steps.pilot.outputs.enabled == 'true'
+        with:
+          fetch-depth: 0
+          token: ${{ steps.app-token.outputs.token }}
+      - name: Synchronize and promote integration
+        if: steps.pilot.outputs.enabled == 'true'
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          python .github/promote_integration.py