From aeeea19c6a860b5ed6714d6a57866597f18a6472 Mon Sep 17 00:00:00 2001 From: Ian Flores Siaca <18703558+ian-flores@users.noreply.github.com> Date: Tue, 8 Sep 2026 09:14:55 -0700 Subject: [PATCH 1/2] ci: file a tracking issue when a scheduled smoke run fails A scheduled run has no PR author to notice it, and the Workbench nightly was red for five consecutive days without surfacing anywhere. connect-integration.yml already posts to Slack on failure, but its webhook secret is Connect-specific and the convention was never extended to the other suites. Adds a notify-on-scheduled-failure composite action and wires it into the status jobs of the Workbench, Connect, Package Manager and mock-IdP workflows. It files one tracking issue per workflow and comments on it for repeat failures, so weeks of nightly breakage produce one issue rather than a pile. Uses the built-in GITHUB_TOKEN, so no new secret is needed. De-duplication matches the issue title exactly against the open issues, compared client-side rather than through --search: the search index tokenises, is only eventually consistent, and reads a colon as a qualifier separator. Closing the issue is how you ask for a fresh one. Gated on failure() and github.event_name == 'schedule', so pull-request failures stay quiet. The nightly-failure label has to be created manually. The action degrades to filing without it and emits a warning. Closes #632 --- .../notify-on-scheduled-failure/action.yml | 79 +++++++++++++++++++ .github/workflows/connect-smoke.yml | 16 ++++ .github/workflows/mock-idp-e2e.yml | 16 ++++ .github/workflows/packagemanager-smoke.yml | 16 ++++ .github/workflows/workbench-smoke.yml | 16 ++++ 5 files changed, 143 insertions(+) create mode 100644 .github/actions/notify-on-scheduled-failure/action.yml diff --git a/.github/actions/notify-on-scheduled-failure/action.yml b/.github/actions/notify-on-scheduled-failure/action.yml new file mode 100644 index 000000000..bc71e7ff1 --- /dev/null +++ b/.github/actions/notify-on-scheduled-failure/action.yml @@ -0,0 +1,79 @@ +name: Notify on scheduled failure +description: >- + Files a de-duplicated GitHub issue (or comments on the existing one) when a scheduled workflow + run fails, so a silent nightly failure gets surfaced instead of disappearing. Only call this + from a step gated on `if: failure() && github.event_name == 'schedule'` -- it does not check + the event name itself, so a caller gated on push/pull_request would file issues the author + already sees in their own run. + +inputs: + workflow-name: + description: >- + Human-readable name of the failing workflow. Also the de-duplication key: it forms the + issue title, and an existing open issue with that exact title is commented on rather than + duplicated. Renaming it orphans any existing tracking issue. + required: true + github-token: + description: "Token with `issues: write` on this repository. Pass `secrets.GITHUB_TOKEN`." + required: true + +runs: + using: composite + steps: + - name: File or update the nightly failure issue + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + GH_REPO: ${{ github.repository }} + WORKFLOW_NAME: ${{ inputs.workflow-name }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + + RUN_DATE=$(date -u '+%Y-%m-%d %H:%M UTC') + TITLE="Nightly failure: ${WORKFLOW_NAME}" + + # Which jobs in this run actually failed. Best-effort: an empty result just + # means the body omits the "Failed jobs" line. + FAILED_JOBS=$(gh api "repos/${GH_REPO}/actions/runs/${GITHUB_RUN_ID}/jobs" --paginate \ + --jq '[.jobs[] | select(.conclusion == "failure") | .name] | join(", ")' 2>/dev/null || echo "") + + # De-duplicate on an exact title match against the open issues, compared here + # rather than handed to GitHub's search index: `--search` tokenises its input, + # is only eventually consistent, and reads a colon as a qualifier separator -- + # none of which a de-duplication key can tolerate. `--state open` is what makes + # closing the issue the way to ask for a fresh one on the next failure. + EXISTING=$(gh issue list --state open --limit 200 --json number,title \ + | jq -r --arg t "${TITLE}" 'map(select(.title == $t)) | .[0].number // empty') + + # Built with printf into a file so no line carries the leading indentation of + # this script; four leading spaces would render the whole body as a code block. + BODY_FILE="${RUNNER_TEMP:-/tmp}/nightly-failure-body.md" + { + if [ -n "${EXISTING}" ]; then + printf 'The scheduled run of **%s** failed again on %s.\n\n' "${WORKFLOW_NAME}" "${RUN_DATE}" + else + printf 'The scheduled run of **%s** failed on %s.\n\n' "${WORKFLOW_NAME}" "${RUN_DATE}" + fi + printf 'Run: %s\n' "${RUN_URL}" + if [ -n "${FAILED_JOBS}" ]; then + printf '\nFailed jobs: %s\n' "${FAILED_JOBS}" + fi + if [ -z "${EXISTING}" ]; then + printf '\n%s\n' "This issue was filed automatically and is reused for subsequent nightly failures of this workflow. Close it once the underlying failure is fixed, so the next failure opens a fresh issue instead of reviving this one." + fi + } > "${BODY_FILE}" + + if [ -n "${EXISTING}" ]; then + echo "Existing open tracking issue #${EXISTING}; commenting instead of filing a duplicate." + gh issue comment "${EXISTING}" --body-file "${BODY_FILE}" + exit 0 + fi + + echo "No open tracking issue found; filing a new one." + if ! gh issue create --title "${TITLE}" --body-file "${BODY_FILE}" \ + --label nightly-failure 2>"${RUNNER_TEMP:-/tmp}/notify-create-err.log"; then + echo "::warning::Could not file the issue with the 'nightly-failure' label; retrying without it. Create that label in the repository to restore tagging." + cat "${RUNNER_TEMP:-/tmp}/notify-create-err.log" + gh issue create --title "${TITLE}" --body-file "${BODY_FILE}" + fi diff --git a/.github/workflows/connect-smoke.yml b/.github/workflows/connect-smoke.yml index fdff46435..2a9efe0e0 100644 --- a/.github/workflows/connect-smoke.yml +++ b/.github/workflows/connect-smoke.yml @@ -344,7 +344,13 @@ jobs: if: always() needs: [changes, set-matrix, connect-smoke] runs-on: ubuntu-latest + permissions: + contents: read + issues: write steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Check smoke test result run: | if [ "${{ needs.changes.result }}" = "failure" ] || [ "${{ needs.changes.result }}" = "cancelled" ]; then @@ -370,3 +376,13 @@ jobs: exit 1 fi echo "Connect smoke tests passed or were legitimately out of scope" + + # Nightly failures have no PR author to notice them; file (or update) a + # tracking issue instead of failing silently. Never fires on push/pull_request + # -- the author already sees those runs. + - name: Notify on scheduled failure + if: failure() && github.event_name == 'schedule' + uses: ./.github/actions/notify-on-scheduled-failure + with: + workflow-name: Connect Smoke Tests + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/mock-idp-e2e.yml b/.github/workflows/mock-idp-e2e.yml index 351de9e8c..f67c87075 100644 --- a/.github/workflows/mock-idp-e2e.yml +++ b/.github/workflows/mock-idp-e2e.yml @@ -249,7 +249,13 @@ jobs: if: always() needs: [changes, set-matrix, mock-idp-e2e] runs-on: ubuntu-latest + permissions: + contents: read + issues: write steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Check result run: | if [ "${{ needs.changes.result }}" = "failure" ] || [ "${{ needs.changes.result }}" = "cancelled" ]; then @@ -273,3 +279,13 @@ jobs: exit 1 fi echo "Mock-IdP E2E tests passed or were legitimately out of scope" + + # Nightly failures have no PR author to notice them; file (or update) a + # tracking issue instead of failing silently. Never fires on push/pull_request + # -- the author already sees those runs. + - name: Notify on scheduled failure + if: failure() && github.event_name == 'schedule' + uses: ./.github/actions/notify-on-scheduled-failure + with: + workflow-name: Mock-IdP E2E Tests + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/packagemanager-smoke.yml b/.github/workflows/packagemanager-smoke.yml index 46d9d7a6c..dadf6eb06 100644 --- a/.github/workflows/packagemanager-smoke.yml +++ b/.github/workflows/packagemanager-smoke.yml @@ -350,7 +350,13 @@ jobs: if: always() needs: [changes, set-matrix, packagemanager-smoke] runs-on: ubuntu-latest + permissions: + contents: read + issues: write steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Check smoke test result run: | if [ "${{ needs.changes.result }}" = "failure" ] || [ "${{ needs.changes.result }}" = "cancelled" ]; then @@ -372,3 +378,13 @@ jobs: exit 1 fi echo "Package Manager smoke tests passed or were legitimately out of scope" + + # Nightly failures have no PR author to notice them; file (or update) a + # tracking issue instead of failing silently. Never fires on push/pull_request + # -- the author already sees those runs. + - name: Notify on scheduled failure + if: failure() && github.event_name == 'schedule' + uses: ./.github/actions/notify-on-scheduled-failure + with: + workflow-name: Package Manager Smoke Tests + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/workbench-smoke.yml b/.github/workflows/workbench-smoke.yml index a64ec9ec0..b415e9e4d 100644 --- a/.github/workflows/workbench-smoke.yml +++ b/.github/workflows/workbench-smoke.yml @@ -444,7 +444,13 @@ jobs: if: always() needs: [changes, set-matrix, workbench-smoke] runs-on: ubuntu-latest + permissions: + contents: read + issues: write steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Check smoke test result run: | if [ "${{ needs.changes.result }}" = "failure" ] || [ "${{ needs.changes.result }}" = "cancelled" ]; then @@ -466,3 +472,13 @@ jobs: exit 1 fi echo "Workbench smoke tests passed or were legitimately out of scope" + + # Nightly failures have no PR author to notice them; file (or update) a + # tracking issue instead of failing silently. Never fires on push/pull_request + # -- the author already sees those runs. + - name: Notify on scheduled failure + if: failure() && github.event_name == 'schedule' + uses: ./.github/actions/notify-on-scheduled-failure + with: + workflow-name: Workbench Smoke Tests + github-token: ${{ secrets.GITHUB_TOKEN }} From bc9437b9c0b838dac632e4ba14d19fa09266d8a9 Mon Sep 17 00:00:00 2001 From: Ian Flores Siaca <18703558+ian-flores@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:47:56 -0700 Subject: [PATCH 2/2] fix(ci): never let the de-duplication lookup silence the notifier The step runs under set -euo pipefail, so a transient gh API error -- or a runner image without jq -- aborted it before anything was filed. That is the exact silence this action exists to prevent. The lookup is now non-fatal and degrades to an empty result, which files a duplicate issue instead. A duplicate is noisy; silence is invisible. Raises the open-issue scan from 200 to 500 so an older still-open tracking issue is not missed and mistaken for absent, which would also produce a duplicate. --- .../actions/notify-on-scheduled-failure/action.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/actions/notify-on-scheduled-failure/action.yml b/.github/actions/notify-on-scheduled-failure/action.yml index bc71e7ff1..b24bd91e8 100644 --- a/.github/actions/notify-on-scheduled-failure/action.yml +++ b/.github/actions/notify-on-scheduled-failure/action.yml @@ -43,8 +43,18 @@ runs: # is only eventually consistent, and reads a colon as a qualifier separator -- # none of which a de-duplication key can tolerate. `--state open` is what makes # closing the issue the way to ask for a fresh one on the next failure. - EXISTING=$(gh issue list --state open --limit 200 --json number,title \ - | jq -r --arg t "${TITLE}" 'map(select(.title == $t)) | .[0].number // empty') + # + # The whole lookup is deliberately non-fatal. This step runs under `set -e`, so + # without the trailing `|| true` a transient `gh` API error -- or a runner image + # that ships without `jq` -- would abort the step and file nothing, which is the + # exact silence this action exists to prevent. Degrading to an empty result + # files a duplicate issue instead. A duplicate is noisy; silence is invisible. + # + # The limit is high enough that an older still-open tracking issue is not missed + # and mistaken for absent, which would also produce a duplicate. + EXISTING=$(gh issue list --state open --limit 500 --json number,title 2>/dev/null \ + | jq -r --arg t "${TITLE}" 'map(select(.title == $t)) | .[0].number // empty' \ + || true) # Built with printf into a file so no line carries the leading indentation of # this script; four leading spaces would render the whole body as a code block.