diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7ba724..12ce1d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,28 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" fi + # Nightly, a quick look that the CLI still talks to the real platform: the test team's + # token works and the answers still decode. The whole flow runs weekly, in + # platform-e2e.yml. + platform-smoke: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + needs: [verify] + runs-on: ubuntu-latest + env: + STEADYBIT_URL: https://platform.dev.steadybit.com + STEADYBIT_TOKEN: ${{ secrets.STEADYBIT_E2E_TOKEN }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - run: go build -o steadybit ./cmd/steadybit + - name: Read the test team and its latest runs + run: | + test "$(./steadybit team get -k CLI --jq .key)" = CLI + ./steadybit execution list --team CLI --limit 1 -t json >/dev/null + ./steadybit environment list -t json --jq 'length' >/dev/null + docker-build: if: github.event_name != 'schedule' # Images are pushed from here, so a tag whose tests fail must not publish one. @@ -311,3 +333,10 @@ jobs: steadybit --version test "$(steadybit --version)" = "${GITHUB_REF_NAME#v}" test "${{ steps.setup.outputs.version }}" = "${GITHUB_REF_NAME#v}" + + # A stable release is used against the platform right away, as installed by the action. + platform-e2e: + if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') + needs: verify-action + uses: ./.github/workflows/platform-e2e.yml + secrets: inherit diff --git a/.github/workflows/platform-e2e.yml b/.github/workflows/platform-e2e.yml new file mode 100644 index 0000000..915eb44 --- /dev/null +++ b/.github/workflows/platform-e2e.yml @@ -0,0 +1,52 @@ +name: Platform E2E + +# Uses the CLI against the dev platform the way a pipeline does (e2e/platform.sh), with +# the team token of the test team CLI-E2E. Only wait steps, and everything it creates is +# deleted afterwards. +on: + # Weekly, Tuesday evening, so a change on the platform side shows within a week. Cron + # is UTC: 19:00 in Berlin in summer, 18:00 in winter. + schedule: + - cron: '0 17 * * 2' + workflow_dispatch: + inputs: + from-source: + description: Build the CLI from this branch instead of installing the latest release + type: boolean + default: false + # After each stable release, from ci.yml, on the binary users now get. + workflow_call: + # A change to the test itself is run before it is merged. Pull requests from forks get + # no secrets, so it only runs for this repository's own branches. + pull_request: + paths: + - e2e/platform.sh + - .github/workflows/platform-e2e.yml + +# The runs of one team get in each other's way, so one at a time. +concurrency: + group: platform-e2e + +jobs: + platform-e2e: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + env: + STEADYBIT_URL: https://platform.dev.steadybit.com + STEADYBIT_TOKEN: ${{ secrets.STEADYBIT_E2E_TOKEN }} + STEADYBIT_E2E_TEAM: CLI + STEADYBIT_E2E_ENVIRONMENT: Global + steps: + - uses: actions/checkout@v7 + # The latest release through the action, as a pipeline installs it. + - if: ${{ !inputs.from-source }} + uses: ./ + - if: ${{ inputs.from-source }} + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - if: ${{ inputs.from-source }} + run: | + go build -o "$RUNNER_TEMP/bin/steadybit" ./cmd/steadybit + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + - run: e2e/platform.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 40ed970..eff2592 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,7 +38,7 @@ go test ./... ## Tests -Tests sit at three levels. Put a test at the lowest one that can hold it; the levels get +Tests sit at four levels. Put a test at the lowest one that can hold it; the levels get slower and harder to debug as you go down this list. | Level | Tool | Covers | @@ -46,6 +46,7 @@ slower and harder to debug as you go down this list. | Unit | `go test` | A single function, no I/O | | Command | `go test` + `internal/platformtest` | A command end to end against a fake platform | | Container | `e2e/run.sh` + expect | Only what needs a real process | +| Platform | `e2e/platform.sh` | A pipeline's use of the CLI against the dev platform | `internal/platformtest` starts an `httptest` server whose endpoints a test declares, records every request, and points the configuration at it: @@ -65,6 +66,18 @@ docker build -t steadybit/cli:under-test . docker run --rm -v "$PWD/e2e:/e2e" --entrypoint sh steadybit/cli:under-test /e2e/run.sh ``` +The platform test uses the released CLI the way a pipeline does, against the dev +platform with the team token of the test team CLI-E2E (key `CLI`): experiments applied +from files, run in parallel with a report, checked for drift, canceled by SIGTERM, +refused while another runs, and found by an `execution list` gate. It only uses wait +steps and deletes what it creates. CI runs it weekly, after each stable release, and on +pull requests that change it; a nightly job only checks that the token and the API +still work. To run it by hand, with `steadybit` on the `PATH`: + +```sh +STEADYBIT_URL=https://platform.dev.steadybit.com STEADYBIT_TOKEN= e2e/platform.sh +``` + ### Output compatibility Users keep the files `get` writes in Git, so the YAML and JSON the CLI writes must stay diff --git a/e2e/platform.sh b/e2e/platform.sh new file mode 100755 index 0000000..b04b06d --- /dev/null +++ b/e2e/platform.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2026 Steadybit GmbH + +# Uses the CLI against a real platform, the way a pipeline does: experiments applied from +# files, run in parallel with a report, checked for drift, canceled by the SIGTERM a +# canceled job sends, refused while another runs, and found again by a gate. Everything +# only waits, belongs to one test team, and is deleted afterwards, also when a check fails. +# +# Needs STEADYBIT_TOKEN (a team token) and STEADYBIT_URL, and `steadybit` on the PATH. +# STEADYBIT_E2E_TEAM and STEADYBIT_E2E_ENVIRONMENT name the team and its environment. +# +# Other test suites run experiments on the same platform at the same time, so every run +# here allows running in parallel, except the one whose refusal is the point of its check. + +set -uo pipefail + +: "${STEADYBIT_TOKEN:?a team access token is needed}" +TEAM=${STEADYBIT_E2E_TEAM:-CLI} +ENVIRONMENT=${STEADYBIT_E2E_ENVIRONMENT:-Global} +# Every experiment of this suite has a name starting with this, which is how a later run +# finds and deletes what a crashed one left behind. +MARK=cli-e2e-ci +RUN=${GITHUB_RUN_ID:-local-$$} + +work=$(mktemp -d) +cd "$work" || exit 1 +failures=0 + +check() { + description=$1 + shift + if "$@"; then + echo " ok $description" + else + echo " FAIL $description" + failures=$((failures + 1)) + fi +} + +# Runs a command and checks its exit status; on a mismatch, shows what it printed. +exits_with() { + expected=$1 + shift + "$@" >out.log 2>&1 + actual=$? + [ "$actual" -eq "$expected" ] || { + echo " expected exit $expected, got $actual from: $*" + tail -n 20 out.log | sed 's/^/ /' + return 1 + } +} + +experiment() { # file name duration + mkdir -p "$(dirname "$1")" + cat >"$1" </dev/null)" = "1" ] && return 0 + sleep 2 + done + return 1 +} + +# Deletes every experiment of this suite in the team, this run's and any a crashed run +# left, after canceling what still runs. +cleanup() { + echo "cleanup" + rm -rf sweep + steadybit export --team "$TEAM" -d sweep >/dev/null 2>&1 || true + for file in $(grep -l "^name: $MARK-" sweep/experiments/*.yaml 2>/dev/null); do + key=$(key_of "$file") + [ -n "$key" ] || continue + for run in $(steadybit execution list --key "$key" --state CREATED PREPARED RUNNING --limit 0 --jq '.[].id' 2>/dev/null); do + steadybit execution cancel -i "$run" >/dev/null 2>&1 + done + for _ in $(seq 10); do + steadybit experiment delete -k "$key" >/dev/null 2>&1 && break + sleep 3 + done + steadybit experiment get -k "$key" >/dev/null 2>&1 && echo " could not delete $key" || echo " deleted $key" + done + rm -rf "$work" +} +trap cleanup EXIT + +echo "steadybit $(steadybit --version) against ${STEADYBIT_URL:-https://platform.steadybit.com}, team $TEAM" + +experiment experiments/a.yml a 5s +experiment experiments/b.yml b 8s +experiment long.yml long 90s + +check "apply creates the experiments and writes their keys into the files" sh -c ' + steadybit experiment apply -f experiments -f long.yml >/dev/null && + grep -q "^key: " experiments/a.yml && grep -q "^key: " experiments/b.yml && grep -q "^key: " long.yml && + grep -q "^# Written by" experiments/a.yml +' +A=$(key_of experiments/a.yml) +LONG=$(key_of long.yml) + +check "diff finds no drift right after apply" exits_with 0 steadybit experiment diff -f experiments +sed -i.bak 's/duration: 8s/duration: 9s/' experiments/b.yml && rm -f experiments/b.yml.bak +check "diff exits with 2 once a file changed" exits_with 2 steadybit experiment diff -f experiments +check "apply updates the experiment" exits_with 0 steadybit experiment apply -f experiments + +check "a parallel run completes both experiments and reports them" sh -c ' + steadybit experiment run -f experiments --yes --parallel 2 --allowParallel --report report.xml >run.log 2>&1 || { tail -n 20 run.log; exit 1; } + [ "$(grep -c "term.log 2>&1 & +pid=$! +if until_run_is "$LONG" RUNNING; then + kill -TERM "$pid" + wait "$pid" + status=$? + check "SIGTERM exits with 143" test "$status" -eq 143 + check "SIGTERM cancels the run" until_run_is "$LONG" CANCELED +else + kill -TERM "$pid" 2>/dev/null + check "the long run started" false +fi + +# While one experiment runs, the platform accepts another and cancels it right away; +# --no-wait has to notice instead of passing. This run alone does not allow running in +# parallel: its refusal is what is checked. +check "a run started with --no-wait begins" exits_with 0 steadybit experiment run -k "$LONG" --yes --no-wait --allowParallel +until_run_is "$LONG" RUNNING +check "--no-wait fails on a run the platform refused" exits_with 1 steadybit experiment run -k "$A" --yes --no-wait +check "execution list --fail-on-match gates on that canceled run" exits_with 1 \ + steadybit execution list --key "$A" --state CANCELED --ended-from "$(date -u +%F)" --limit 1 --fail-on-match +check "execution list prints the platform's runs as JSON" sh -c " + [ \"\$(steadybit execution list --team $TEAM --limit 2 --jq length 2>/dev/null)\" -ge 1 ] +" + +echo +if [ "$failures" -eq 0 ]; then + echo "all platform checks passed" +else + echo "$failures platform check(s) failed" +fi +exit "$failures"