Fix four failing workflows: traffic, message center, catalog, PR sweeper

.github/scripts/pr-sweeper/report.mjs

@@ -267,13 +267,31 @@ async function resolvePrNumberFromSha(sha) {
   // Bind the PR number to the TRUSTED head SHA from the workflow_run event.
   // Works for fork PRs (where workflow_run.pull_requests is empty) because the
   // head commit is reachable via refs/pull/N/head in the base repo.
-  const res = await gh("GET", `/repos/${REPO}/commits/${encodeURIComponent(sha)}/pulls?per_page=100`);
-  if (!res.ok) return null;
-  const pulls = await res.json();
-  if (!Array.isArray(pulls) || pulls.length === 0) return null;
-  const match =
-    pulls.find((p) => p.state === "open" && p.base?.repo?.full_name === REPO) || pulls[0];
-  return match?.number ?? null;
+  //
+  // Retried: GitHub's commit -> PR association index lags PR creation by tens of
+  // seconds, and this job routinely starts within that window on fork PRs.
+  const delays = [0, 3000, 8000, 15000];
+  for (const wait of delays) {
+    if (wait) await sleep(wait);
+    const res = await gh("GET", `/repos/${REPO}/commits/${encodeURIComponent(sha)}/pulls?per_page=100`);
+    if (!res.ok) {
+      console.warn(`Commit -> PR lookup returned ${res.status}; retrying.`);
+      continue;
+    }
+    const pulls = await res.json();
+    if (!Array.isArray(pulls) || pulls.length === 0) {
+      console.warn("Commit -> PR lookup returned no pull requests yet; retrying.");
+      continue;
+    }
+    const match =
+      pulls.find((p) => p.state === "open" && p.base?.repo?.full_name === REPO) || pulls[0];
+    if (match?.number) return match.number;
+  }
+  return null;
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
 async function main() {
@@ -308,8 +326,14 @@ async function main() {
   if (!number && headSha) number = await resolvePrNumberFromSha(headSha);
 
   if (!number) {
-    console.error("Could not resolve a trusted PR number from the workflow_run event; aborting.");
-    process.exit(1);
+    // The security gate is the part that actually matters, and it only needs the
+    // trusted head SHA. Set it, then stop — a missing sticky comment is cosmetic
+    // and must not turn into a red workflow on every fork PR.
+    console.warn(
+      "Could not resolve a trusted PR number from the workflow_run event; posting the commit status only."
+    );
+    await setStatus(headSha, guardrails);
+    return;
   }
 
   const body = buildComment(config, guardrails, ai, meta);

.github/workflows/build-catalog.yml

@@ -52,6 +52,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write # Only the publish job may push the rebuilt catalog.
+      pull-requests: write # Needed when branch protection forces a PR instead.
     steps:
       - uses: actions/checkout@v4
         with:
@@ -71,7 +72,14 @@ jobs:
           CLARITY_API_TOKEN: ${{ secrets.CLARITY_API_TOKEN }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Commit generated catalog
+        env:
+          # Prefer a PAT when configured; the built-in token can be blocked from
+          # opening PRs by the "Allow GitHub Actions to create and approve pull
+          # requests" org/repo policy.
+          GH_TOKEN: ${{ secrets.AUTOMATION_PAT || secrets.GITHUB_TOKEN }}
         run: |
+          set -euo pipefail
+
           git add catalog.json resource-stats.json traffic-data/clarity-views.json resource-discussions.json
           if [ -f design-concepts/resource-stats.json ]; then
             git add design-concepts/resource-stats.json
@@ -83,4 +91,73 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git commit -m "chore: rebuild catalog [skip ci]"
-          git push
+
+          # Fast path: push straight to the default branch. Another push landing
+          # first is normal on a busy branch, so rebase once and retry.
+          if git push || { git pull --rebase origin "${GITHUB_REF_NAME}" && git push; }; then
+            echo "✅ Catalog pushed to ${GITHUB_REF_NAME}."
+            exit 0
+          fi
+
+          # Branch protection ("Changes must be made through a pull request") rejects
+          # the direct push. Route the rebuild through a PR instead of losing it.
+          echo "::notice::Direct push rejected — falling back to a pull request."
+
+          BRANCH="catalog-refresh/$(date -u +%Y-%m-%d)-${GITHUB_SHA:0:7}"
+          git checkout -b "$BRANCH"
+          # Seed the remote-tracking ref so --force-with-lease has a basis on re-runs.
+          if git fetch origin "$BRANCH"; then
+            git update-ref "refs/remotes/origin/$BRANCH" FETCH_HEAD
+          fi
+          git push --force-with-lease --set-upstream origin "$BRANCH"
+
+          COMPARE_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/compare/${GITHUB_REF_NAME}...${BRANCH}?expand=1"
+
+          existing_pr=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number // empty')
+          if [ -z "$existing_pr" ]; then
+            if create_output=$(gh pr create \
+              --title "chore: rebuild catalog" \
+              --body "Automated catalog + resource stats rebuild for ${GITHUB_SHA}." \
+              --head "$BRANCH" \
+              --base "${GITHUB_REF_NAME}" 2>&1); then
+              echo "✅ PR created: $create_output"
+              # Read the number straight out of the returned URL. Re-listing here
+              # would race GitHub's PR index and could report a phantom failure.
+              existing_pr=$(printf '%s' "$create_output" | grep -oE 'pull/[0-9]+' | tail -n1 | cut -d/ -f2)
+            else
+              echo "$create_output"
+              if printf '%s' "$create_output" | grep -qi "not permitted to create or approve pull requests"; then
+                echo "::warning::Catalog rebuild is on branch $BRANCH, but this repo blocks GitHub Actions from opening pull requests. Open it manually: $COMPARE_URL"
+                {
+                  echo "### ⚠️ Catalog rebuild needs a manual PR"
+                  echo ""
+                  echo "The rebuilt catalog was pushed to \`$BRANCH\`."
+                  echo ""
+                  echo "**Open the PR:** $COMPARE_URL"
+                } >> "$GITHUB_STEP_SUMMARY"
+                exit 0
+              fi
+              echo "::error::Failed to open the catalog rebuild PR for $BRANCH."
+              exit 1
+            fi
+          fi
+
+          # Reaching here without a PR means the rebuild is stranded on a branch and
+          # will never land — that is a failure, not a silent success.
+          if [ -z "$existing_pr" ]; then
+            echo "::error::Pushed $BRANCH but could not confirm an open pull request. Open it manually: $COMPARE_URL"
+            exit 1
+          fi
+
+          # Auto-merge needs the repo-level "Allow auto-merge" setting. The PR itself
+          # already exists and is reviewable, so this stays a warning — but a loud one.
+          if ! gh pr merge "$existing_pr" --auto --squash; then
+            echo "::warning::Could not enable auto-merge on PR #$existing_pr — merge it manually: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/pull/$existing_pr"
+            {
+              echo "### ⚠️ Catalog rebuild PR #$existing_pr needs a manual merge"
+              echo ""
+              echo "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/pull/$existing_pr"
+            } >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "✅ Auto-merge enabled for PR #$existing_pr"
+          fi

.github/workflows/traffic-stats.yml

@@ -49,21 +49,33 @@ jobs:
           DATA_DIR="traffic-data"
           DATE=$(date -u +"%Y-%m-%d")
 
+          # Fail fast with a precise message instead of three opaque retries.
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "::error::TRAFFIC_TOKEN secret is empty or unset. The traffic API needs a PAT with administration:read on $REPO — the built-in GITHUB_TOKEN cannot read traffic."
+            exit 1
+          fi
+
           gh_api_retry() {
             local max_attempts=3
             local delay=5
             local attempt=1
             local output
+            local err_file
+            err_file=$(mktemp)
             while [ "$attempt" -le "$max_attempts" ]; do
-              if output=$(gh api "$@" 2>/dev/null); then
+              if output=$(gh api "$@" 2>"$err_file"); then
+                rm -f "$err_file"
                 printf '%s' "$output"
                 return 0
               fi
-              echo "::warning::gh api attempt $attempt/$max_attempts failed, retrying in ${delay}s..." >&2
-              sleep "$delay"
+              # Keep gh's stderr: without it every failure looks identical and the
+              # real cause (401 / 403 / 404 / rate limit) is invisible in the log.
+              echo "::warning::gh api attempt $attempt/$max_attempts failed: $(tr '\n' ' ' <"$err_file" | cut -c1-400)" >&2
+              [ "$attempt" -lt "$max_attempts" ] && sleep "$delay"
               delay=$((delay * 2))
               attempt=$((attempt + 1))
             done
+            rm -f "$err_file"
             return 1
           }
 
@@ -72,14 +84,17 @@ jobs:
             local endpoint="$2"
             local response
 
+            # These messages MUST go to stderr: this function runs inside a command
+            # substitution, so anything on stdout is captured into the caller's
+            # variable instead of being printed to the workflow log.
             if ! response=$(gh_api_retry "$endpoint"); then
-              echo "::error::Failed to fetch $label after retries"
-              exit 1
+              echo "::error::Failed to fetch $label from $endpoint after retries. Most likely the TRAFFIC_TOKEN secret has expired or lost administration:read on $REPO." >&2
+              return 1
             fi
 
             if ! printf '%s' "$response" | jq -e . >/dev/null 2>&1; then
-              echo "::error::Invalid JSON returned for $label"
-              exit 1
+              echo "::error::Invalid JSON returned for $label" >&2
+              return 1
             fi
 
             printf '%s' "$response"
@@ -89,11 +104,12 @@ jobs:
 
           echo "📊 Fetching traffic data for $REPO on $DATE..."
 
-          # Fetch all four traffic endpoints
-          views=$(fetch_required_json "views" "repos/$REPO/traffic/views")
-          clones=$(fetch_required_json "clones" "repos/$REPO/traffic/clones")
-          referrers=$(fetch_required_json "referrers" "repos/$REPO/traffic/popular/referrers")
-          paths=$(fetch_required_json "paths" "repos/$REPO/traffic/popular/paths")
+          # Fetch all four traffic endpoints ("|| exit 1" is explicit: set -e already
+          # aborts on a failed assignment, but this makes the intent unmissable).
+          views=$(fetch_required_json "views" "repos/$REPO/traffic/views") || exit 1
+          clones=$(fetch_required_json "clones" "repos/$REPO/traffic/clones") || exit 1
+          referrers=$(fetch_required_json "referrers" "repos/$REPO/traffic/popular/referrers") || exit 1
+          paths=$(fetch_required_json "paths" "repos/$REPO/traffic/popular/paths") || exit 1
 
           # Build a combined JSON snapshot for today
 

.github/workflows/update-message-center.yml

@@ -42,11 +42,13 @@ jobs:
       - name: Commit and open PR
         if: steps.changes.outputs.changed == 'true'
         env:
-          # Use the built-in GITHUB_TOKEN. The job already grants
-          # contents:write + pull-requests:write, which is sufficient to push
-          # the branch and open the PR without a separate long-lived PAT.
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Prefer a PAT when one is configured. Fall back to the built-in token,
+          # which can push the branch but may be blocked from opening the PR by the
+          # org/repo policy "Allow GitHub Actions to create and approve pull requests".
+          GH_TOKEN: ${{ secrets.AUTOMATION_PAT || secrets.GITHUB_TOKEN }}
         run: |
+          set -euo pipefail
+
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git add copilot-agent-strategy/copilot-agents-guide/index.html
@@ -56,17 +58,53 @@ jobs:
 
           git checkout -b "$BRANCH"
           git commit -m "chore: weekly message center posts refresh $DATE"
-          git push --set-upstream origin "$BRANCH"
+          # A re-run on the same day must be able to update its own branch. Seed the
+          # remote-tracking ref first, otherwise --force-with-lease aborts on "stale
+          # info" because actions/checkout only fetched the default branch.
+          if git fetch origin "$BRANCH"; then
+            git update-ref "refs/remotes/origin/$BRANCH" FETCH_HEAD
+          fi
+          git push --force-with-lease --set-upstream origin "$BRANCH"
+
+          COMPARE_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/compare/master...${BRANCH}?expand=1"
 
           # Create a normal PR for human review if one doesn't already exist.
           existing_pr=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number // empty')
           if [ -n "$existing_pr" ]; then
             echo "PR #$existing_pr already exists for $BRANCH"
-          else
-            gh pr create \
-              --title "chore: weekly message center posts refresh $DATE" \
-              --body "Automated weekly message center data update. This PR contains externally derived content and requires human review." \
-              --head "$BRANCH" \
-              --base master
-            echo "✅ PR created"
+            exit 0
+          fi
+
+          if create_output=$(gh pr create \
+            --title "chore: weekly message center posts refresh $DATE" \
+            --body "Automated weekly message center data update. This PR contains externally derived content and requires human review." \
+            --head "$BRANCH" \
+            --base master 2>&1); then
+            echo "✅ PR created: $create_output"
+            exit 0
           fi
+
+          echo "$create_output"
+
+          # The refreshed content is already safely on the branch. A repo policy that
+          # forbids Actions from opening PRs must not destroy the week's refresh — warn
+          # and hand the maintainer a one-click link instead of failing the job.
+          if printf '%s' "$create_output" | grep -qi "not permitted to create or approve pull requests"; then
+            echo "::warning::Branch $BRANCH was pushed, but this repo blocks GitHub Actions from opening pull requests. Open it manually: $COMPARE_URL"
+            {
+              echo "### ⚠️ Message center refresh needs a manual PR"
+              echo ""
+              echo "The refreshed content was pushed to \`$BRANCH\`, but GitHub Actions is not"
+              echo "permitted to open pull requests in this repository."
+              echo ""
+              echo "**Open the PR:** $COMPARE_URL"
+              echo ""
+              echo "**Permanent fix:** Settings → Actions → General → enable"
+              echo "\"Allow GitHub Actions to create and approve pull requests\", or set an"
+              echo "\`AUTOMATION_PAT\` secret with \`pull-requests: write\`."
+            } >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          echo "::error::Failed to open the message center PR for $BRANCH."
+          exit 1