ci: use the CLI against the dev platform, weekly and after each release

.github/workflows/ci.yml

@@ -70,6 +70,28 @@ jobs:
             } >> "$GITHUB_STEP_SUMMARY"
           fi
 
+  # Nightly, a quick look that the CLI still talks to the real platform: the test team's
+  # token works and the answers still decode. The whole flow runs weekly, in
+  # platform-e2e.yml.
+  platform-smoke:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    needs: [verify]
+    runs-on: ubuntu-latest
+    env:
+      STEADYBIT_URL: https://platform.dev.steadybit.com
+      STEADYBIT_TOKEN: ${{ secrets.STEADYBIT_E2E_TOKEN }}
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - run: go build -o steadybit ./cmd/steadybit
+      - name: Read the test team and its latest runs
+        run: |
+          test "$(./steadybit team get -k CLI --jq .key)" = CLI
+          ./steadybit execution list --team CLI --limit 1 -t json >/dev/null
+          ./steadybit environment list -t json --jq 'length' >/dev/null
+
   docker-build:
     if: github.event_name != 'schedule'
     # Images are pushed from here, so a tag whose tests fail must not publish one.
@@ -311,3 +333,10 @@ jobs:
           steadybit --version
           test "$(steadybit --version)" = "${GITHUB_REF_NAME#v}"
           test "${{ steps.setup.outputs.version }}" = "${GITHUB_REF_NAME#v}"
+
+  # A stable release is used against the platform right away, as installed by the action.
+  platform-e2e:
+    if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-')
+    needs: verify-action
+    uses: ./.github/workflows/platform-e2e.yml
+    secrets: inherit

.github/workflows/platform-e2e.yml

@@ -0,0 +1,52 @@
+name: Platform E2E
+
+# Uses the CLI against the dev platform the way a pipeline does (e2e/platform.sh), with
+# the team token of the test team CLI-E2E. Only wait steps, and everything it creates is
+# deleted afterwards.
+on:
+  # Weekly, Tuesday evening, so a change on the platform side shows within a week. Cron
+  # is UTC: 19:00 in Berlin in summer, 18:00 in winter.
+  schedule:
+    - cron: '0 17 * * 2'
+  workflow_dispatch:
+    inputs:
+      from-source:
+        description: Build the CLI from this branch instead of installing the latest release
+        type: boolean
+        default: false
+  # After each stable release, from ci.yml, on the binary users now get.
+  workflow_call:
+  # A change to the test itself is run before it is merged. Pull requests from forks get
+  # no secrets, so it only runs for this repository's own branches.
+  pull_request:
+    paths:
+      - e2e/platform.sh
+      - .github/workflows/platform-e2e.yml
+
+# The runs of one team get in each other's way, so one at a time.
+concurrency:
+  group: platform-e2e
+
+jobs:
+  platform-e2e:
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    env:
+      STEADYBIT_URL: https://platform.dev.steadybit.com
+      STEADYBIT_TOKEN: ${{ secrets.STEADYBIT_E2E_TOKEN }}
+      STEADYBIT_E2E_TEAM: CLI
+      STEADYBIT_E2E_ENVIRONMENT: Global
+    steps:
+      - uses: actions/checkout@v7
+      # The latest release through the action, as a pipeline installs it.
+      - if: ${{ !inputs.from-source }}
+        uses: ./
+      - if: ${{ inputs.from-source }}
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - if: ${{ inputs.from-source }}
+        run: |
+          go build -o "$RUNNER_TEMP/bin/steadybit" ./cmd/steadybit
+          echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
+      - run: e2e/platform.sh

CONTRIBUTING.md

@@ -38,14 +38,15 @@ go test ./...
 
 ## Tests
 
-Tests sit at three levels. Put a test at the lowest one that can hold it; the levels get
+Tests sit at four levels. Put a test at the lowest one that can hold it; the levels get
 slower and harder to debug as you go down this list.
 
 | Level     | Tool                               | Covers                                                  |
 | --------- | ---------------------------------- | ------------------------------------------------------- |
 | Unit      | `go test`                          | A single function, no I/O                               |
 | Command   | `go test` + `internal/platformtest` | A command end to end against a fake platform           |
 | Container | `e2e/run.sh` + expect              | Only what needs a real process                          |
+| Platform  | `e2e/platform.sh`                  | A pipeline's use of the CLI against the dev platform    |
 
 `internal/platformtest` starts an `httptest` server whose endpoints a test declares,
 records every request, and points the configuration at it:
@@ -65,6 +66,18 @@ docker build -t steadybit/cli:under-test .
 docker run --rm -v "$PWD/e2e:/e2e" --entrypoint sh steadybit/cli:under-test /e2e/run.sh
 ```
 
+The platform test uses the released CLI the way a pipeline does, against the dev
+platform with the team token of the test team CLI-E2E (key `CLI`): experiments applied
+from files, run in parallel with a report, checked for drift, canceled by SIGTERM,
+refused while another runs, and found by an `execution list` gate. It only uses wait
+steps and deletes what it creates. CI runs it weekly, after each stable release, and on
+pull requests that change it; a nightly job only checks that the token and the API
+still work. To run it by hand, with `steadybit` on the `PATH`:
+
+```sh
+STEADYBIT_URL=https://platform.dev.steadybit.com STEADYBIT_TOKEN=<token of team CLI> e2e/platform.sh
+```
+
 ### Output compatibility
 
 Users keep the files `get` writes in Git, so the YAML and JSON the CLI writes must stay

e2e/platform.sh

@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2026 Steadybit GmbH
+
+# Uses the CLI against a real platform, the way a pipeline does: experiments applied from
+# files, run in parallel with a report, checked for drift, canceled by the SIGTERM a
+# canceled job sends, refused while another runs, and found again by a gate. Everything
+# only waits, belongs to one test team, and is deleted afterwards, also when a check fails.
+#
+# Needs STEADYBIT_TOKEN (a team token) and STEADYBIT_URL, and `steadybit` on the PATH.
+# STEADYBIT_E2E_TEAM and STEADYBIT_E2E_ENVIRONMENT name the team and its environment.
+#
+# Other test suites run experiments on the same platform at the same time, so every run
+# here allows running in parallel, except the one whose refusal is the point of its check.
+
+set -uo pipefail
+
+: "${STEADYBIT_TOKEN:?a team access token is needed}"
+TEAM=${STEADYBIT_E2E_TEAM:-CLI}
+ENVIRONMENT=${STEADYBIT_E2E_ENVIRONMENT:-Global}
+# Every experiment of this suite has a name starting with this, which is how a later run
+# finds and deletes what a crashed one left behind.
+MARK=cli-e2e-ci
+RUN=${GITHUB_RUN_ID:-local-$$}
+
+work=$(mktemp -d)
+cd "$work" || exit 1
+failures=0
+
+check() {
+  description=$1
+  shift
+  if "$@"; then
+    echo "  ok    $description"
+  else
+    echo "  FAIL  $description"
+    failures=$((failures + 1))
+  fi
+}
+
+# Runs a command and checks its exit status; on a mismatch, shows what it printed.
+exits_with() {
+  expected=$1
+  shift
+  "$@" >out.log 2>&1
+  actual=$?
+  [ "$actual" -eq "$expected" ] || {
+    echo "        expected exit $expected, got $actual from: $*"
+    tail -n 20 out.log | sed 's/^/        /'
+    return 1
+  }
+}
+
+experiment() { # file name duration
+  mkdir -p "$(dirname "$1")"
+  cat >"$1" <<EOF
+# Written by e2e/platform.sh; only waits.
+name: $MARK-$RUN-$2
+team: $TEAM
+environment: $ENVIRONMENT
+lanes:
+  - steps:
+      - type: wait
+        ignoreFailure: false
+        parameters:
+          duration: $3
+EOF
+}
+
+key_of() { sed -n 's/^key: //p' "$1"; }
+
+# Waits until the experiment has a run in one of the states, for up to a minute.
+until_run_is() { # key state...
+  key=$1
+  shift
+  for _ in $(seq 30); do
+    [ "$(steadybit execution list --key "$key" --state "$@" --limit 1 --jq length 2>/dev/null)" = "1" ] && return 0
+    sleep 2
+  done
+  return 1
+}
+
+# Deletes every experiment of this suite in the team, this run's and any a crashed run
+# left, after canceling what still runs.
+cleanup() {
+  echo "cleanup"
+  rm -rf sweep
+  steadybit export --team "$TEAM" -d sweep >/dev/null 2>&1 || true
+  for file in $(grep -l "^name: $MARK-" sweep/experiments/*.yaml 2>/dev/null); do
+    key=$(key_of "$file")
+    [ -n "$key" ] || continue
+    for run in $(steadybit execution list --key "$key" --state CREATED PREPARED RUNNING --limit 0 --jq '.[].id' 2>/dev/null); do
+      steadybit execution cancel -i "$run" >/dev/null 2>&1
+    done
+    for _ in $(seq 10); do
+      steadybit experiment delete -k "$key" >/dev/null 2>&1 && break
+      sleep 3
+    done
+    steadybit experiment get -k "$key" >/dev/null 2>&1 && echo "  could not delete $key" || echo "  deleted $key"
+  done
+  rm -rf "$work"
+}
+trap cleanup EXIT
+
+echo "steadybit $(steadybit --version) against ${STEADYBIT_URL:-https://platform.steadybit.com}, team $TEAM"
+
+experiment experiments/a.yml a 5s
+experiment experiments/b.yml b 8s
+experiment long.yml long 90s
+
+check "apply creates the experiments and writes their keys into the files" sh -c '
+  steadybit experiment apply -f experiments -f long.yml >/dev/null &&
+  grep -q "^key: " experiments/a.yml && grep -q "^key: " experiments/b.yml && grep -q "^key: " long.yml &&
+  grep -q "^# Written by" experiments/a.yml
+'
+A=$(key_of experiments/a.yml)
+LONG=$(key_of long.yml)
+
+check "diff finds no drift right after apply" exits_with 0 steadybit experiment diff -f experiments
+sed -i.bak 's/duration: 8s/duration: 9s/' experiments/b.yml && rm -f experiments/b.yml.bak
+check "diff exits with 2 once a file changed" exits_with 2 steadybit experiment diff -f experiments
+check "apply updates the experiment" exits_with 0 steadybit experiment apply -f experiments
+
+check "a parallel run completes both experiments and reports them" sh -c '
+  steadybit experiment run -f experiments --yes --parallel 2 --allowParallel --report report.xml >run.log 2>&1 || { tail -n 20 run.log; exit 1; }
+  [ "$(grep -c "<testsuite " report.xml)" -eq 2 ] && grep -q "\[" run.log
+'
+
+# A canceled job sends SIGTERM: the run it started is canceled, and the CLI exits with 143.
+steadybit experiment run -k "$LONG" --yes --allowParallel >term.log 2>&1 &
+pid=$!
+if until_run_is "$LONG" RUNNING; then
+  kill -TERM "$pid"
+  wait "$pid"
+  status=$?
+  check "SIGTERM exits with 143" test "$status" -eq 143
+  check "SIGTERM cancels the run" until_run_is "$LONG" CANCELED
+else
+  kill -TERM "$pid" 2>/dev/null
+  check "the long run started" false
+fi
+
+# While one experiment runs, the platform accepts another and cancels it right away;
+# --no-wait has to notice instead of passing. This run alone does not allow running in
+# parallel: its refusal is what is checked.
+check "a run started with --no-wait begins" exits_with 0 steadybit experiment run -k "$LONG" --yes --no-wait --allowParallel
+until_run_is "$LONG" RUNNING
+check "--no-wait fails on a run the platform refused" exits_with 1 steadybit experiment run -k "$A" --yes --no-wait
+check "execution list --fail-on-match gates on that canceled run" exits_with 1 \
+  steadybit execution list --key "$A" --state CANCELED --ended-from "$(date -u +%F)" --limit 1 --fail-on-match
+check "execution list prints the platform's runs as JSON" sh -c "
+  [ \"\$(steadybit execution list --team $TEAM --limit 2 --jq length 2>/dev/null)\" -ge 1 ]
+"
+
+echo
+if [ "$failures" -eq 0 ]; then
+  echo "all platform checks passed"
+else
+  echo "$failures platform check(s) failed"
+fi
+exit "$failures"