From 8e4605ff546cc63f85569d915aa9d1140cf380da Mon Sep 17 00:00:00 2001 From: Joel Varty Date: Fri, 30 Oct 2026 11:29:30 -0400 Subject: [PATCH 01/25] CI: live sync test on throwaway instances, gating promotion to latest (PROD-3016) Leases throwaway instances, syncs a source to a target with the CLI under test, runs verify, checks that a second sync would change nothing, and always deletes the instances. - ci/leases.mjs: dependency-free lease client (create and wait, release by label prefix, sweep of left-behind leases), with retries. - ci/permutation/run.mjs: the first slice of the test. Writes a results grid to the job summary and a JSON artifact. - sync-permutation.yml: quick tier on same-repo PRs, beta tier on v* tags (tests the published package), nightly tier with a sweep, and manual runs. Credentials come only from a GitHub Environment by OIDC, so fork PRs get none. Every identifier is an Environment secret, so nothing about the infrastructure appears in the repo or the public logs. - npm-publish.yml: promote to latest requires a passing tag run. - ci/README.md: when it runs, how to trigger it, what it checks. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/npm-publish.yml | 19 +++ .github/workflows/sync-permutation.yml | 201 +++++++++++++++++++++++++ ci/README.md | 74 +++++++++ ci/leases.mjs | 171 +++++++++++++++++++++ ci/permutation/run.mjs | 117 ++++++++++++++ 5 files changed, 582 insertions(+) create mode 100644 .github/workflows/sync-permutation.yml create mode 100644 ci/README.md create mode 100644 ci/leases.mjs create mode 100644 ci/permutation/run.mjs diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index e50da78..98031bd 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -68,6 +68,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + # Reads the sync permutation runs for the gate below. + actions: read # Uses the trusted publisher's OIDC credentials; needs "Allow npm dist-tag" # enabled on the package's trusted publishing configuration on npmjs.com. id-token: write @@ -104,6 +106,23 @@ jobs: - name: Check the version is published run: npm view "@agility/cli@$VERSION" version | grep -qx "$VERSION" || { echo "@agility/cli@$VERSION is not on npm."; exit 1; } + # Nothing reaches `latest` untested on dev: the tag's sync permutation run + # (sync-permutation.yml, triggered by the same tag push) must have passed. If it failed + # for a reason unrelated to the CLI, re-run it from the Actions tab and promote again. + - name: Require a passing sync permutation run + if: env.DIST_TAG == 'latest' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + SHA=$(gh api "repos/$REPO/commits/v$VERSION" --jq .sha) || { echo "No tag v$VERSION."; exit 1; } + PASSED=$(gh api "repos/$REPO/actions/workflows/sync-permutation.yml/runs?event=push&head_sha=$SHA&status=success&per_page=1" --jq .total_count) + if [ "$PASSED" = "0" ]; then + echo "No passing 'Sync permutation test' run for v$VERSION ($SHA). See the Actions tab." + exit 1 + fi + echo "v$VERSION passed the sync permutation test." + - name: Promote run: npm dist-tag add "@agility/cli@$VERSION" "$DIST_TAG" diff --git a/.github/workflows/sync-permutation.yml b/.github/workflows/sync-permutation.yml new file mode 100644 index 0000000..88a161c --- /dev/null +++ b/.github/workflows/sync-permutation.yml @@ -0,0 +1,201 @@ +name: Sync permutation test + +# The live end-to-end sync test (epic PROD-3015, pipeline PROD-3016): lease throwaway dev +# instances, sync between them with the CLI under test, verify, and always delete them. +# +# This is the "live" kind of integration test that integration-tests.yml warns about. It is safe +# to run here because: +# - Credentials come only from the `cli-dev-live` GitHub Environment, by OIDC. Pull requests +# from forks get no OIDC token and no Environment, and the `if:` below skips them anyway. +# - The identity can only lease and delete its own throwaway test instances. +# - Every lease is labelled with this run and has a short TTL. The `release` job deletes them +# whatever happens to the rest of the run, and the leasing service deletes anything a +# dead runner leaves behind once the TTL runs out. +# +# Tiers (the same jobs; the tier sets TTLs and timeouts, and later slices add scenarios to it): +# quick every pull request +# beta every v* tag (the version npm-publish.yml just published under `beta`). Its result +# gates promotion to `latest` (npm-publish.yml, promote job) +# nightly every night against whatever is deployed to dev, plus a sweep for left-behind leases + +on: + pull_request: + push: + tags: + - "v*" + schedule: + - cron: "0 7 * * *" # 07:00 UTC, 03:00 Eastern + workflow_dispatch: + inputs: + tier: + description: "Tier" + type: choice + options: [quick, beta, nightly] + default: quick + cli_version: + description: "Test this published @agility/cli version instead of building this branch" + required: false + default: "" + +concurrency: + group: sync-permutation-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + id-token: write + +env: + TIER: ${{ inputs.tier || (github.event_name == 'schedule' && 'nightly') || (github.ref_type == 'tag' && 'beta') || 'quick' }} + # Published version under test: the dispatch input, or the tag's version on a tag push. + CLI_VERSION: ${{ inputs.cli_version || (github.ref_type == 'tag' && github.ref_name) || '' }} + # Every lease of this run (and of this attempt, so a re-run gets fresh instances) starts with this. + LEASE_PREFIX: cli-perm-${{ github.run_id }}-${{ github.run_attempt }}- + GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + +jobs: + lease: + name: Lease dev instances + # Same-repo PRs only: forks and Dependabot get no credentials. + if: >- + github.event_name != 'pull_request' || + (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') + runs-on: ubuntu-latest + environment: cli-dev-live + timeout-minutes: 20 + # Endpoints, IDs and names are cli-dev-live Environment *secrets*, not variables: this repo's + # Actions logs are public, and GitHub prints variables and step inputs there but masks secrets. + env: + LEASE_API_URL: ${{ secrets.LEASE_API_URL }} + LEASE_API_SCOPE: ${{ secrets.LEASE_API_SCOPE }} + outputs: + source_guid: ${{ steps.lease.outputs.source_guid }} + target_guid: ${{ steps.lease.outputs.target_guid }} + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: ci + + - uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + allow-no-subscriptions: true + + - name: Lease API token + run: | + TOKEN=$(az account get-access-token --scope "$LEASE_API_SCOPE" --tenant "${{ secrets.AZURE_TENANT_ID }}" --query accessToken -o tsv) + echo "::add-mask::$TOKEN" + echo "LEASE_API_TOKEN=$TOKEN" >> "$GITHUB_ENV" + + # Layer 4 of teardown: anything an earlier run left behind (its release job failed or never ran). + - name: Sweep left-behind leases + if: env.TIER == 'nightly' + run: node ci/leases.mjs sweep cli-perm- --older-than-hours 6 + + # TTL = the run's budget plus a margin, so the reaper is the backstop if every other layer fails. + - name: Lease source and target + id: lease + run: | + case "$TIER" in quick) TTL=60 ;; beta) TTL=120 ;; *) TTL=240 ;; esac + node ci/leases.mjs create "$LEASE_PREFIX" source:blog target:blank --ttl "$TTL" --timeout 900 + + test: + name: Sync and verify + needs: lease + runs-on: ubuntu-latest + environment: cli-dev-live + timeout-minutes: 50 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "24" + cache: "npm" + + # The CLI under test: this checkout, built; or a published version from npm. + - name: Build this branch + if: env.CLI_VERSION == '' + run: | + npm ci + npm run build + echo "CLI=node $GITHUB_WORKSPACE/dist/index.js" >> "$GITHUB_ENV" + + - name: Install @agility/cli@${{ env.CLI_VERSION }} from npm + if: env.CLI_VERSION != '' + run: | + V="${CLI_VERSION#v}" + # A tag push starts this workflow and npm-publish.yml together; wait for the publish. + for i in $(seq 1 40); do + npm view "@agility/cli@$V" version >/dev/null 2>&1 && break + echo "waiting for @agility/cli@$V on npm ($i/40)"; sleep 15 + done + npm install --prefix "$RUNNER_TEMP/cli" "@agility/cli@$V" + echo "CLI=node $RUNNER_TEMP/cli/node_modules/@agility/cli/dist/index.js" >> "$GITHUB_ENV" + + - uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + allow-no-subscriptions: true + + # One token that can read and write every test instance. + - name: Test account token + run: | + PAT=$(az keyvault secret show --vault-name "${{ secrets.TOKEN_VAULT }}" --name "${{ secrets.TOKEN_SECRET }}" --query value -o tsv) + echo "::add-mask::$PAT" + echo "AGILITY_TOKEN=$PAT" >> "$GITHUB_ENV" + + - name: Run the permutation test + env: + SOURCE_GUID: ${{ needs.lease.outputs.source_guid }} + TARGET_GUID: ${{ needs.lease.outputs.target_guid }} + OUT_DIR: ${{ runner.temp }}/permutation-out + run: node ci/permutation/run.mjs + + # Only the JSON results: anyone can download a public repo's artifacts, so the CLI's raw + # logs, pulls and mapping files stay on the runner. Re-run locally to debug from those. + - name: Keep results + if: always() + uses: actions/upload-artifact@v4 + with: + name: sync-permutation-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/permutation-out/*.json + retention-days: 30 + if-no-files-found: ignore + + # Layer 2 of teardown. Runs after success, failure and cancellation alike, and deletes every + # lease of this attempt by label, including any the lease job created before it failed. + release: + name: Release leases + needs: [lease, test] + if: >- + always() && ( + github.event_name != 'pull_request' || + (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') + ) + runs-on: ubuntu-latest + environment: cli-dev-live + timeout-minutes: 10 + # Endpoints, IDs and names are cli-dev-live Environment *secrets*, not variables: this repo's + # Actions logs are public, and GitHub prints variables and step inputs there but masks secrets. + env: + LEASE_API_URL: ${{ secrets.LEASE_API_URL }} + LEASE_API_SCOPE: ${{ secrets.LEASE_API_SCOPE }} + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: ci + + - uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + allow-no-subscriptions: true + + - name: Release every lease of this run + run: | + TOKEN=$(az account get-access-token --scope "$LEASE_API_SCOPE" --tenant "${{ secrets.AZURE_TENANT_ID }}" --query accessToken -o tsv) + echo "::add-mask::$TOKEN" + LEASE_API_TOKEN="$TOKEN" node ci/leases.mjs release "$LEASE_PREFIX" --reason "run ${{ github.run_id }} finished: ${{ needs.test.result }}" diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000..b513f7e --- /dev/null +++ b/ci/README.md @@ -0,0 +1,74 @@ +# Live sync test (`sync-permutation.yml`) + +An end-to-end test of `agility sync` against real instances. Each run leases throwaway instances, +syncs a source to a target with the CLI under test, checks the result, and always deletes the +instances afterwards. + +The unit tests (`test.yml`) and the hermetic sync tests (`integration-tests.yml`) need no +credentials and run on every PR, forks included. This test needs credentials, so it runs only where +they're available. + +## When it runs + +| Trigger | Tier | CLI under test | +| ---------------------------------------------------------------- | ----------- | ------------------------------------------------------------- | +| A pull request from a branch in this repo | `quick` | the PR's build. The result shows as a check on the PR | +| A `v*` tag push (each release, published to npm `beta`) | `beta` | the published package, installed from npm | +| Nightly at 07:00 UTC | `nightly` | `main`. Also deletes any instances an earlier run left behind | +| By hand (Actions tab → _Sync permutation test_ → _Run workflow_) | your choice | the branch you pick, or a published version | + +Pull requests from forks and from Dependabot skip it, because they get no credentials. + +From a terminal: + +```bash +gh workflow run sync-permutation.yml -f tier=quick +gh workflow run sync-permutation.yml -f tier=beta -f cli_version=1.2.3 # test a published version +gh run rerun # re-run with fresh instances +``` + +## What it checks + +Today (the first slice): + +- **First sync:** `sync` exits 0 and its `--jsonSummary` reports no failed items. +- **verify:** `agility verify` reports that the target matches the source (exit 0). +- **Nothing left to sync:** an immediate `sync --preflight` plans no creates, updates or conflicts. + +Each check is a row in the run's job summary. The JSON results are kept as an artifact for 30 days. +Later slices add generated sources, rounds of changes on both sides, more target types, and +reverse, round-trip and chained syncs. + +## Promotion to `latest` + +`npm-publish.yml` refuses to promote a version to `latest` unless that version's tag run of this +test passed. If a tag run failed for a reason unrelated to the CLI, re-run it, then promote again. + +## Cleanup + +Instances are deleted in three ways, so a run can't leave them behind: + +1. The `release` job runs after every run, including failed and cancelled ones, and deletes every + instance labelled with that run. +2. Every instance expires shortly after the run's time budget and is then deleted automatically. +3. The nightly run deletes any instance from an earlier run that is more than 6 hours old. + +## Running it locally + +The scripts need no dependencies beyond Node 20+. With `LEASE_API_URL` and `LEASE_API_TOKEN` set (ask +the CLI maintainers), and `AGILITY_TOKEN` set to a token that can write to the leased instances: + +```bash +npm run build +node ci/leases.mjs create my-run- source:blog target:blank # prints both instance GUIDs +SOURCE_GUID= TARGET_GUID= CLI="node $PWD/dist/index.js" node ci/permutation/run.mjs +node ci/leases.mjs release my-run- +``` + +Results land in `./permutation-out`. The CLI's own logs are under `permutation-out/work/agility-files/logs`. + +## Configuration + +Everything that identifies our infrastructure lives in the `cli-dev-live` GitHub Environment as +**secrets**, not in this repo and not as variables. The Actions logs of a public repo are public, +and GitHub masks secrets in them but prints variables. diff --git a/ci/leases.mjs b/ci/leases.mjs new file mode 100644 index 0000000..c0569a3 --- /dev/null +++ b/ci/leases.mjs @@ -0,0 +1,171 @@ +#!/usr/bin/env node +// Test-instance lease client for CI. +// +// Leases throwaway instances for the live sync test and tears them down. Plain Node (>= 20, global +// fetch), no dependencies. +// +// Configuration (set as GitHub Environment variables, never committed): +// LEASE_API_URL the lease endpoint's base URL +// LEASE_API_TOKEN a bearer token for it (in GitHub Actions: OIDC, then `az account get-access-token`) +// +// node ci/leases.mjs create :