From e5af7c27e5541968ac87d51f3f8a53ec55c0cb4e Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Tue, 29 Sep 2026 16:49:32 +0200 Subject: [PATCH 1/2] ci: use the CLI against the dev platform, weekly and after each release Nothing checked that the released CLI, installed by the action, works against a real platform: the tests use a fake one, and the container tests run without one. e2e/platform.sh uses it the way a pipeline does, with the team token of the test team CLI-E2E: experiments applied from files (keys written back), drift found by diff, a parallel run with a JUnit report, SIGTERM canceling the run with exit 143, --no-wait failing on a run the platform refused, and an execution list gate. Only wait steps, and everything it creates is deleted afterwards, also what an earlier crashed run left. The Platform E2E workflow runs it weekly on Tuesday evenings with the latest release, after each stable release from ci.yml, by hand (optionally from source), and on pull requests that change it. A nightly job in ci.yml only reads the test team and its runs, to see early that the token and the API still work. --- .github/workflows/ci.yml | 29 ++++++ .github/workflows/platform-e2e.yml | 51 ++++++++++ CONTRIBUTING.md | 15 ++- e2e/platform.sh | 157 +++++++++++++++++++++++++++++ 4 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/platform-e2e.yml create mode 100755 e2e/platform.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7ba724..12ce1d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,28 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" fi + # Nightly, a quick look that the CLI still talks to the real platform: the test team's + # token works and the answers still decode. The whole flow runs weekly, in + # platform-e2e.yml. + platform-smoke: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + needs: [verify] + runs-on: ubuntu-latest + env: + STEADYBIT_URL: https://platform.dev.steadybit.com + STEADYBIT_TOKEN: ${{ secrets.STEADYBIT_E2E_TOKEN }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - run: go build -o steadybit ./cmd/steadybit + - name: Read the test team and its latest runs + run: | + test "$(./steadybit team get -k CLI --jq .key)" = CLI + ./steadybit execution list --team CLI --limit 1 -t json >/dev/null + ./steadybit environment list -t json --jq 'length' >/dev/null + docker-build: if: github.event_name != 'schedule' # Images are pushed from here, so a tag whose tests fail must not publish one. @@ -311,3 +333,10 @@ jobs: steadybit --version test "$(steadybit --version)" = "${GITHUB_REF_NAME#v}" test "${{ steps.setup.outputs.version }}" = "${GITHUB_REF_NAME#v}" + + # A stable release is used against the platform right away, as installed by the action. + platform-e2e: + if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') + needs: verify-action + uses: ./.github/workflows/platform-e2e.yml + secrets: inherit diff --git a/.github/workflows/platform-e2e.yml b/.github/workflows/platform-e2e.yml new file mode 100644 index 0000000..50b42d9 --- /dev/null +++ b/.github/workflows/platform-e2e.yml @@ -0,0 +1,51 @@ +name: Platform E2E + +# Uses the CLI against the dev platform the way a pipeline does (e2e/platform.sh), with +# the team token of the test team CLI-E2E. Only wait steps, and everything it creates is +# deleted afterwards. +on: + # Weekly, Tuesday evening, so a change on the platform side shows within a week. + schedule: + - cron: '0 20 * * 2' + workflow_dispatch: + inputs: + from-source: + description: Build the CLI from this branch instead of installing the latest release + type: boolean + default: false + # After each stable release, from ci.yml, on the binary users now get. + workflow_call: + # A change to the test itself is run before it is merged. Pull requests from forks get + # no secrets, so it only runs for this repository's own branches. + pull_request: + paths: + - e2e/platform.sh + - .github/workflows/platform-e2e.yml + +# The runs of one team get in each other's way, so one at a time. +concurrency: + group: platform-e2e + +jobs: + platform-e2e: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + env: + STEADYBIT_URL: https://platform.dev.steadybit.com + STEADYBIT_TOKEN: ${{ secrets.STEADYBIT_E2E_TOKEN }} + STEADYBIT_E2E_TEAM: CLI + STEADYBIT_E2E_ENVIRONMENT: Global + steps: + - uses: actions/checkout@v7 + # The latest release through the action, as a pipeline installs it. + - if: ${{ !inputs.from-source }} + uses: ./ + - if: ${{ inputs.from-source }} + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - if: ${{ inputs.from-source }} + run: | + go build -o "$RUNNER_TEMP/bin/steadybit" ./cmd/steadybit + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + - run: e2e/platform.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 40ed970..eff2592 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,7 +38,7 @@ go test ./... ## Tests -Tests sit at three levels. Put a test at the lowest one that can hold it; the levels get +Tests sit at four levels. Put a test at the lowest one that can hold it; the levels get slower and harder to debug as you go down this list. | Level | Tool | Covers | @@ -46,6 +46,7 @@ slower and harder to debug as you go down this list. | Unit | `go test` | A single function, no I/O | | Command | `go test` + `internal/platformtest` | A command end to end against a fake platform | | Container | `e2e/run.sh` + expect | Only what needs a real process | +| Platform | `e2e/platform.sh` | A pipeline's use of the CLI against the dev platform | `internal/platformtest` starts an `httptest` server whose endpoints a test declares, records every request, and points the configuration at it: @@ -65,6 +66,18 @@ docker build -t steadybit/cli:under-test . docker run --rm -v "$PWD/e2e:/e2e" --entrypoint sh steadybit/cli:under-test /e2e/run.sh ``` +The platform test uses the released CLI the way a pipeline does, against the dev +platform with the team token of the test team CLI-E2E (key `CLI`): experiments applied +from files, run in parallel with a report, checked for drift, canceled by SIGTERM, +refused while another runs, and found by an `execution list` gate. It only uses wait +steps and deletes what it creates. CI runs it weekly, after each stable release, and on +pull requests that change it; a nightly job only checks that the token and the API +still work. To run it by hand, with `steadybit` on the `PATH`: + +```sh +STEADYBIT_URL=https://platform.dev.steadybit.com STEADYBIT_TOKEN= e2e/platform.sh +``` + ### Output compatibility Users keep the files `get` writes in Git, so the YAML and JSON the CLI writes must stay diff --git a/e2e/platform.sh b/e2e/platform.sh new file mode 100755 index 0000000..db22eda --- /dev/null +++ b/e2e/platform.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2026 Steadybit GmbH + +# Uses the CLI against a real platform, the way a pipeline does: experiments applied from +# files, run in parallel with a report, checked for drift, canceled by the SIGTERM a +# canceled job sends, refused while another runs, and found again by a gate. Everything +# only waits, belongs to one test team, and is deleted afterwards, also when a check fails. +# +# Needs STEADYBIT_TOKEN (a team token) and STEADYBIT_URL, and `steadybit` on the PATH. +# STEADYBIT_E2E_TEAM and STEADYBIT_E2E_ENVIRONMENT name the team and its environment. + +set -uo pipefail + +: "${STEADYBIT_TOKEN:?a team access token is needed}" +TEAM=${STEADYBIT_E2E_TEAM:-CLI} +ENVIRONMENT=${STEADYBIT_E2E_ENVIRONMENT:-Global} +# Every experiment of this suite has a name starting with this, which is how a later run +# finds and deletes what a crashed one left behind. +MARK=cli-e2e-ci +RUN=${GITHUB_RUN_ID:-local-$$} + +work=$(mktemp -d) +cd "$work" || exit 1 +failures=0 + +check() { + description=$1 + shift + if "$@"; then + echo " ok $description" + else + echo " FAIL $description" + failures=$((failures + 1)) + fi +} + +# Runs a command and checks its exit status; on a mismatch, shows what it printed. +exits_with() { + expected=$1 + shift + "$@" >out.log 2>&1 + actual=$? + [ "$actual" -eq "$expected" ] || { + echo " expected exit $expected, got $actual from: $*" + tail -n 20 out.log | sed 's/^/ /' + return 1 + } +} + +experiment() { # file name duration + mkdir -p "$(dirname "$1")" + cat >"$1" </dev/null)" = "1" ] && return 0 + sleep 2 + done + return 1 +} + +# Deletes every experiment of this suite in the team, this run's and any a crashed run +# left, after canceling what still runs. +cleanup() { + echo "cleanup" + rm -rf sweep + steadybit export --team "$TEAM" -d sweep >/dev/null 2>&1 || true + for file in $(grep -l "^name: $MARK-" sweep/experiments/*.yaml 2>/dev/null); do + key=$(key_of "$file") + [ -n "$key" ] || continue + for run in $(steadybit execution list --key "$key" --state CREATED PREPARED RUNNING --limit 0 --jq '.[].id' 2>/dev/null); do + steadybit execution cancel -i "$run" >/dev/null 2>&1 + done + for _ in $(seq 10); do + steadybit experiment delete -k "$key" >/dev/null 2>&1 && break + sleep 3 + done + steadybit experiment get -k "$key" >/dev/null 2>&1 && echo " could not delete $key" || echo " deleted $key" + done + rm -rf "$work" +} +trap cleanup EXIT + +echo "steadybit $(steadybit --version) against ${STEADYBIT_URL:-https://platform.steadybit.com}, team $TEAM" + +experiment experiments/a.yml a 5s +experiment experiments/b.yml b 8s +experiment long.yml long 90s + +check "apply creates the experiments and writes their keys into the files" sh -c ' + steadybit experiment apply -f experiments -f long.yml >/dev/null && + grep -q "^key: " experiments/a.yml && grep -q "^key: " experiments/b.yml && grep -q "^key: " long.yml && + grep -q "^# Written by" experiments/a.yml +' +A=$(key_of experiments/a.yml) +LONG=$(key_of long.yml) + +check "diff finds no drift right after apply" exits_with 0 steadybit experiment diff -f experiments +sed -i.bak 's/duration: 8s/duration: 9s/' experiments/b.yml && rm -f experiments/b.yml.bak +check "diff exits with 2 once a file changed" exits_with 2 steadybit experiment diff -f experiments +check "apply updates the experiment" exits_with 0 steadybit experiment apply -f experiments + +check "a parallel run completes both experiments and reports them" sh -c ' + steadybit experiment run -f experiments --yes --parallel 2 --report report.xml >run.log 2>&1 || { tail -n 20 run.log; exit 1; } + [ "$(grep -c "term.log 2>&1 & +pid=$! +if until_run_is "$LONG" RUNNING; then + kill -TERM "$pid" + wait "$pid" + status=$? + check "SIGTERM exits with 143" test "$status" -eq 143 + check "SIGTERM cancels the run" until_run_is "$LONG" CANCELED +else + kill -TERM "$pid" 2>/dev/null + check "the long run started" false +fi + +# While one experiment runs, the platform accepts another and cancels it right away; +# --no-wait has to notice instead of passing. +check "a run started with --no-wait begins" exits_with 0 steadybit experiment run -k "$LONG" --yes --no-wait --allowParallel +until_run_is "$LONG" RUNNING +check "--no-wait fails on a run the platform refused" exits_with 1 steadybit experiment run -k "$A" --yes --no-wait +check "execution list --fail-on-match gates on that canceled run" exits_with 1 \ + steadybit execution list --key "$A" --state CANCELED --ended-from "$(date -u +%F)" --limit 1 --fail-on-match +check "execution list prints the platform's runs as JSON" sh -c " + [ \"\$(steadybit execution list --team $TEAM --limit 2 --jq length 2>/dev/null)\" -ge 1 ] +" + +echo +if [ "$failures" -eq 0 ]; then + echo "all platform checks passed" +else + echo "$failures platform check(s) failed" +fi +exit "$failures" From 07fed642b5244037d959c5ac049c55efca2677b3 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Tue, 29 Sep 2026 16:57:49 +0200 Subject: [PATCH 2/2] ci: the platform test allows parallel runs and starts at 19:00 in Berlin Other test suites run experiments on dev in the evening, so every run of the platform test allows running in parallel, now explicitly, except the one whose refusal its check is about. The weekly run moves to 17:00 UTC: 19:00 in Berlin in summer, 18:00 in winter, as cron has no time zone. --- .github/workflows/platform-e2e.yml | 5 +++-- e2e/platform.sh | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/platform-e2e.yml b/.github/workflows/platform-e2e.yml index 50b42d9..915eb44 100644 --- a/.github/workflows/platform-e2e.yml +++ b/.github/workflows/platform-e2e.yml @@ -4,9 +4,10 @@ name: Platform E2E # the team token of the test team CLI-E2E. Only wait steps, and everything it creates is # deleted afterwards. on: - # Weekly, Tuesday evening, so a change on the platform side shows within a week. + # Weekly, Tuesday evening, so a change on the platform side shows within a week. Cron + # is UTC: 19:00 in Berlin in summer, 18:00 in winter. schedule: - - cron: '0 20 * * 2' + - cron: '0 17 * * 2' workflow_dispatch: inputs: from-source: diff --git a/e2e/platform.sh b/e2e/platform.sh index db22eda..b04b06d 100755 --- a/e2e/platform.sh +++ b/e2e/platform.sh @@ -9,6 +9,9 @@ # # Needs STEADYBIT_TOKEN (a team token) and STEADYBIT_URL, and `steadybit` on the PATH. # STEADYBIT_E2E_TEAM and STEADYBIT_E2E_ENVIRONMENT name the team and its environment. +# +# Other test suites run experiments on the same platform at the same time, so every run +# here allows running in parallel, except the one whose refusal is the point of its check. set -uo pipefail @@ -119,7 +122,7 @@ check "diff exits with 2 once a file changed" exits_with 2 steadybit experiment check "apply updates the experiment" exits_with 0 steadybit experiment apply -f experiments check "a parallel run completes both experiments and reports them" sh -c ' - steadybit experiment run -f experiments --yes --parallel 2 --report report.xml >run.log 2>&1 || { tail -n 20 run.log; exit 1; } + steadybit experiment run -f experiments --yes --parallel 2 --allowParallel --report report.xml >run.log 2>&1 || { tail -n 20 run.log; exit 1; } [ "$(grep -c "