test(release): qualify signed published artifacts on clean installs and upgrades

.github/workflows/ci.yaml

@@ -38,12 +38,17 @@ on:
       # Feature catalog and the website gitlink feed the features sync gate (spec 024).
       - "FEATURES.md"
       - "website"
-  # Manual full-suite run on any branch (spec 023 OD-7).
+  # An empty dispatch runs the full suite (spec 023 OD-7). A tag runs the release consumer.
   workflow_dispatch:
+    inputs:
+      published_release:
+        description: Run only release qualification for this published tag (for example v1.0.0)
+        required: false
+        type: string
 # Cancel superseded runs so rapid pushes to the same ref don't waste
 # runner minutes on stale commits.
 concurrency:
-  group: ci-${{ github.ref }}
+  group: ci-${{ inputs.published_release && format('published-{0}', inputs.published_release) || 'suite' }}-${{ github.ref }}
   cancel-in-progress: true
 
 # The Go module mirror and checksum database have repeatedly dropped HTTP/2
@@ -69,6 +74,15 @@ permissions:
 # its own kind cluster, so the slowest suite — not the sum of all of
 # them — sets the wall-clock time.
 jobs:
+  published-release:
+    name: qualify a published release
+    if: github.event_name == 'workflow_dispatch' && inputs.published_release != ''
+    permissions:
+      contents: read
+    uses: ./.github/workflows/qualify-release.yaml
+    with:
+      release_tag: ${{ inputs.published_release }}
+
   # Path-filter gate. Computes which parts of the tree a change touches so the
   # heavy jobs below can skip when they can't be affected (a web-only PR drops
   # from ~22 jobs to a handful). Any shared/CI-config path (this workflow, the
@@ -79,6 +93,7 @@ jobs:
   # selection table to this job's summary.
   changes:
     name: detect changes
+    if: github.event_name != 'workflow_dispatch' || inputs.published_release == ''
     runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:
@@ -849,6 +864,7 @@ jobs:
   # separate: not part of `make lint`, and not in the `report` job's needs.
   telemetry-default-gate:
     name: telemetry default gate
+    if: github.event_name != 'workflow_dispatch' || inputs.published_release == ''
     runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:
@@ -2189,6 +2205,9 @@ jobs:
         uses: ./.github/actions/e2e-images
         with:
           arch: ${{ matrix.arch }}
+      - name: verify release consumer provenance and cluster ownership guards
+        if: matrix.arch == 'amd64'
+        run: bash test/e2e/release-consumer-checks.sh
       - name: bring up cluster on the PREVIOUS release
         run: ./deploy/kind/upgrade.sh up "$CLUSTER" e2e
       - name: install Argo CD for the populated GitOps upgrade
@@ -2683,7 +2702,7 @@ jobs:
         e2e-postgres,
         e2e-kind-ingress-smoke,
       ]
-    if: always()
+    if: always() && (github.event_name != 'workflow_dispatch' || inputs.published_release == '')
     runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:

.github/workflows/qualify-release.yaml

@@ -0,0 +1,37 @@
+name: published release qualification
+
+on:
+  workflow_call:
+    inputs:
+      release_tag:
+        description: Published Gameplane version tag
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  consumer:
+    name: verify published chart and images
+    runs-on: ubuntu-latest
+    timeout-minutes: 50
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/go-cache
+        with:
+          key-suffix: e2e
+      - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+      - uses: helm/kind-action@ea822c7bf1874f08e5a491df991c228edffc3376 # v1.15.2
+        with:
+          install_only: true
+      - uses: oras-project/setup-oras@5e2f2482b30b0cb3e14e67f89a80c88c659a7c63 # v2.0.3
+      - uses: sigstore/cosign-installer@756fe2d284c7292c4e9b62c8a28f1e5629c59532 # v4.1.3
+        with:
+          cosign-release: 'v3.1.2'
+      - name: verify, install and upgrade using published chart and images
+        env:
+          RELEASE_TAG: ${{ inputs.release_tag }}
+        run: bash deploy/kind/qualify-published-release.sh "$RELEASE_TAG"

.github/workflows/release.yaml

@@ -271,6 +271,15 @@ jobs:
           cosign public-key --key env://COSIGN_PRIVATE_KEY --outfile "$RUNNER_TEMP/cosign.pub"
           cosign verify --key "$RUNNER_TEMP/cosign.pub" --new-bundle-format=false "$REF"
 
+  consumer:
+    name: qualify published artifacts as an anonymous consumer
+    needs: chart
+    permissions:
+      contents: read
+    uses: ./.github/workflows/qualify-release.yaml
+    with:
+      release_tag: ${{ github.ref_name }}
+
   # Publish the GitHub Release page for the tag, with the matching CHANGELOG
   # section as the notes. beta.3's page was created by hand and beta.4's was
   # forgotten entirely — this makes the page part of the release, not a
@@ -281,7 +290,7 @@ jobs:
     timeout-minutes: 15 # applied by extension per D-A, pending maintainer confirmation
     permissions:
       contents: write
-    needs: [images, chart]
+    needs: consumer
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: extract changelog section

changelog/unreleased/906-published-release-check.md

@@ -0,0 +1,5 @@
+# Published release verification
+
+Release qualification now verifies the signed Helm chart and component image digests as an
+anonymous consumer. It exercises a clean install and a populated upgrade from the previous
+supported release. [PR #906](https://github.com/GameplanePanel/Gameplane/pull/906)

deploy/kind/qualify-published-release.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Release CI runs this on a fresh amd64 runner without a registry login.
+set -euo pipefail
+source deploy/kind/release-consumer-lib.sh
+tag=${1:?pass the published release tag}
+[[ $tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$ ]] || {
+  echo "expected a published vMAJOR.MINOR.PATCH tag, optionally with a prerelease suffix" >&2
+  exit 2
+}
+version=${tag#v}
+owner="${GITHUB_REPOSITORY_OWNER,,}"
+registry="ghcr.io/$owner"
+scratch="$(mktemp -d)"
+export DOCKER_CONFIG="$scratch/docker" HELM_REGISTRY_CONFIG="$scratch/helm.json"
+export KUBECONFIG="$scratch/kubeconfig" KIND_EXPERIMENTAL_PROVIDER=docker
+owned_node=''
+cleanup() {
+  local status=$?
+  delete_consumer_cluster || status=1
+  rm -rf -- "$scratch"
+  exit "$status"
+}
+trap cleanup EXIT
+export GAMEPLANE_PUBLISHED_CHART="$scratch/gameplane" GAMEPLANE_RELEASE_DIGESTS="$scratch/images.tsv"
+export GAMEPLANE_PREVIOUS_RELEASE_DIGESTS="$scratch/previous-images.tsv"
+mkdir -p "$DOCKER_CONFIG"
+cosign initialize
+
+chart="$registry/charts/gameplane"
+digest="$(oras resolve "$chart:$version")"
+verify_release_signature "$chart" "$digest"
+helm pull "oci://$chart@$digest" --untar --untardir "$scratch"
+test "$(yq -r '.version' "$GAMEPLANE_PUBLISHED_CHART/Chart.yaml")" = "$version"
+test "$(yq -r '.appVersion' "$GAMEPLANE_PUBLISHED_CHART/Chart.yaml")" = "$version"
+printf 'Verified chart `%s@%s`\n' "$chart" "$digest" >> "$GITHUB_STEP_SUMMARY"
+
+# Read the publisher's complete matrix, including optional images. Keep the
+# signed index and its amd64 manifest/config IDs for containerd comparisons.
+git fetch --quiet --no-tags --depth=1 origin "refs/tags/$tag"
+revision=$(git rev-parse --verify 'FETCH_HEAD^{commit}')
+git show "$revision:.github/workflows/release.yaml" > "$scratch/published-release.yaml"
+printf 'Release `%s` at Git commit `%s`\n' "$tag" "$revision" >> "$GITHUB_STEP_SUMMARY"
+components="$(yq -r '.jobs.images.strategy.matrix.include[].component' "$scratch/published-release.yaml")"
+test -n "$components"
+while read -r component; do
+  verify_release_image "$registry/gameplane/$component" "$version" "$revision" "$GAMEPLANE_RELEASE_DIGESTS"
+done <<< "$components"
+
+run_id=$(cat /proc/sys/kernel/random/uuid)
+fresh="gp-fresh-$run_id"
+create_consumer_cluster "$fresh"
+helm install gameplane "$GAMEPLANE_PUBLISHED_CHART" --kube-context "kind-$fresh" \
+  --namespace gameplane-system --create-namespace --set ingress.enabled=false \
+  --set defaultModuleSource.enabled=false --wait --timeout 6m
+bash deploy/kind/verify-release-runtime.sh "kind-$fresh" "$GAMEPLANE_RELEASE_DIGESTS" api operator web
+delete_consumer_cluster
+
+# The regular upgrade fixture seeds the old API database and game PVC,
+# exercises the CRD hooks, and checks their contents after the upgrade.
+export GAMEPLANE_E2E_CLUSTER="gp-upgrade-$run_id" GAMEPLANE_E2E_REUSE_CLUSTER=1
+export GAMEPLANE_E2E_CONTEXT="kind-$GAMEPLANE_E2E_CLUSTER"
+create_consumer_cluster "$GAMEPLANE_E2E_CLUSTER"
+deploy/kind/upgrade.sh install "$GAMEPLANE_E2E_CLUSTER"
+(cd test/e2e && go test -tags=e2e -timeout 25m -v -run '^TestUpgrade_FromPreviousRelease$' .)

deploy/kind/release-consumer-lib.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Classic keyed cosign verification checks the digest. Also check the
+# repository named in the signed payload before trusting a component.
+verify_release_signature() {
+  local repository=$1 digest=$2
+  if ! cosign verify --key signing/cosign.pub --new-bundle-format=false "$repository@$digest" |
+    jq -e --arg repo "$repository" --arg digest "$digest" 'any(.[];
+      .critical.identity["docker-reference"] == $repo and
+      .critical.image["docker-manifest-digest"] == $digest)' >/dev/null; then
+    echo "Missing valid release signature for $repository@$digest" >&2
+    return 1
+  fi
+}
+
+# The config blob is addressed through the verified release index.
+verify_release_config() {
+  jq -e --arg arch "$1" --arg revision "$2" --arg version "$3" '
+    .os == "linux" and .architecture == $arch and
+    .config.Labels["org.opencontainers.image.revision"] == $revision and
+    .config.Labels["org.opencontainers.image.version"] == $version' >/dev/null
+}
+
+verify_release_image() {
+  local image=$1 version=$2 revision=$3 digest_file=$4
+  local index ref index_json arch manifest config
+  index=$(oras resolve "$image:$version")
+  ref="$image@$index"
+  verify_release_signature "$image" "$index"
+  docker pull --platform linux/amd64 "$ref"
+  printf '\n- `%s`\n' "$ref" >> "$GITHUB_STEP_SUMMARY"
+  index_json=$(oras manifest fetch "$ref")
+  for arch in amd64 arm64; do
+    manifest=$(jq -er --arg arch "$arch" '
+      [.manifests[] | select(.platform.os == "linux" and .platform.architecture == $arch) | .digest]
+      | if length == 1 then .[0] else error("expected one linux/" + $arch + " manifest") end' <<< "$index_json")
+    config=$(oras manifest fetch "$image@$manifest" | jq -er '.config.digest')
+    if ! oras blob fetch --output - "$image@$config" | verify_release_config "$arch" "$revision" "$version"; then
+      echo "$image@$config must identify linux/$arch, version $version and Git commit $revision" >&2
+      return 1
+    fi
+    printf '  - linux/%s manifest `%s`, config `%s`\n' "$arch" "$manifest" "$config" >> "$GITHUB_STEP_SUMMARY"
+    if [[ $arch == amd64 ]]; then
+      printf '%s\t%s\t%s\t%s\t%s\n' "${image##*/}" "$image:$version" "$index" "$manifest" "$config" >> "$digest_file"
+    fi
+  done
+}
+
+# Each consumer cluster has one Docker control-plane node. Capture its
+# immutable container ID only after its CA matches the private kubeconfig
+# written by this creation. A concurrent name replacement stays untouched.
+create_consumer_cluster() {
+  local name=$1 existing id node_ca kube_ca
+  [[ -z $owned_node ]] || return 1
+  existing=$(kind get clusters) || return 1
+  if grep -Fxq -- "$name" <<< "$existing"; then
+    echo "Kind cluster $name already exists" >&2
+    return 1
+  fi
+  kind create cluster --name "$name" --kubeconfig "$KUBECONFIG" --wait 90s || return 1
+  id=$(docker inspect --type container --format '{{.Id}}' "$name-control-plane") || return 1
+  [[ $id =~ ^[a-f0-9]{64}$ ]] || return 1
+  node_ca=$(docker exec "$id" cat /etc/kubernetes/pki/ca.crt | base64 -w0) || return 1
+  kube_ca=$(kubectl config view --kubeconfig "$KUBECONFIG" --context "kind-$name" \
+    --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}') || return 1
+  if [[ -z $node_ca || $node_ca != "$kube_ca" ]]; then
+    echo "Control-plane container for $name does not match its kubeconfig" >&2
+    return 1
+  fi
+  owned_node=$id
+}
+
+delete_consumer_cluster() {
+  if [[ -n $owned_node ]]; then
+    docker rm --force --volumes "$owned_node" >/dev/null || return 1
+    owned_node=''
+  fi
+}

deploy/kind/upgrade.sh

@@ -12,6 +12,7 @@
 #
 # Usage:
 #   deploy/kind/upgrade.sh up   [cluster-name] [tag]
+#   deploy/kind/upgrade.sh install <existing-kind-cluster> [tag]
 #   deploy/kind/upgrade.sh down [cluster-name]
 
 set -euo pipefail
@@ -35,44 +36,65 @@ REPO="$(cd "${HERE}/../.." && pwd)"
 FROM_VERSION="${GAMEPLANE_UPGRADE_FROM:-0.3.0}"
 # Releases up to v0.3.0 were published before the repo moved to the
 # GameplanePanel org, so they live under the original owner on GHCR.
-CHART_REF="oci://ghcr.io/valgulnecron/charts/gameplane"
+FROM_REGISTRY="ghcr.io/valgulnecron"
+CHART_REF="oci://$FROM_REGISTRY/charts/gameplane"
 
 need() { command -v "$1" >/dev/null 2>&1 || { echo "missing: $1" >&2; exit 1; }; }
 
 case "${ACTION}" in
-up)
+up|install)
     need kind
     need kubectl
     need helm
     need docker
 
-    if kind get clusters | grep -qx "${CLUSTER}"; then
-        echo "removing pre-existing cluster ${CLUSTER} for a clean slate"
-        kind delete cluster --name "${CLUSTER}" || true
+    if [[ "$ACTION" == up ]]; then
+        # Kind's default is a single control-plane node. Creation rejects
+        # collisions and leaves the existing cluster and its volumes alone.
+        kind create cluster --name "${CLUSTER}" --wait 90s
+    elif [[ -z ${2:-} ]]; then
+        echo "install requires the name of the existing Kind cluster" >&2
+        exit 2
     fi
-    echo "creating kind cluster ${CLUSTER} (single-node)"
-    kind create cluster --name "${CLUSTER}" --wait 90s --config - <<EOF
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-  - role: control-plane
-EOF
 
     kubectl cluster-info --context "kind-${CLUSTER}" >/dev/null
 
     # Load the NEW (working-tree) images now, so the upgrade the test performs
     # later needs no network. The OLD release's images are pulled from GHCR by
     # the install below — that is the point of the test.
-    echo "loading working-tree gameplane-test/{operator,api,agent,sentinel}:${TAG} into kind"
-    for img in operator api agent sentinel; do
-        if ! docker image inspect "gameplane-test/${img}:${TAG}" >/dev/null 2>&1; then
-            echo "  missing local image gameplane-test/${img}:${TAG} — building"
-            docker build -t "gameplane-test/${img}:${TAG}" -f "${REPO}/${img}/Dockerfile" "${REPO}"
-        fi
-        kind load docker-image "gameplane-test/${img}:${TAG}" --name "${CLUSTER}"
-    done
+    if [[ -z "${GAMEPLANE_PUBLISHED_CHART:-}" ]]; then
+        echo "loading working-tree gameplane-test/{operator,api,agent,sentinel}:${TAG} into kind"
+        for img in operator api agent sentinel; do
+            if ! docker image inspect "gameplane-test/${img}:${TAG}" >/dev/null 2>&1; then
+                echo "  missing local image gameplane-test/${img}:${TAG} — building"
+                docker build -t "gameplane-test/${img}:${TAG}" -f "${REPO}/${img}/Dockerfile" "${REPO}"
+            fi
+            kind load docker-image "gameplane-test/${img}:${TAG}" --name "${CLUSTER}"
+        done
+    fi
 
-    echo "installing PUBLISHED chart ${CHART_REF} --version ${FROM_VERSION}"
+    chart_args=(--version "$FROM_VERSION")
+    if [[ -n ${GAMEPLANE_PUBLISHED_CHART:-} ]]; then
+        # Resolve the documented predecessor in its original registry. Record
+        # what should be running before the seeded game and database migrate.
+        source "$HERE/release-consumer-lib.sh"
+        git -C "$REPO" fetch --quiet --no-tags --depth=1 origin "refs/tags/v$FROM_VERSION"
+        previous_commit=$(git -C "$REPO" rev-parse --verify 'FETCH_HEAD^{commit}')
+        repository=${CHART_REF#oci://}
+        digest=$(oras resolve "$repository:$FROM_VERSION")
+        verify_release_signature "$repository" "$digest"
+        CHART_REF="$CHART_REF@$digest"
+        chart_args=()
+        metadata=$(helm show chart "$CHART_REF")
+        [[ $(yq -r '.version' <<< "$metadata") == "$FROM_VERSION" &&
+           $(yq -r '.appVersion' <<< "$metadata") == "$FROM_VERSION" ]]
+        printf '\nPrevious chart `%s@%s`, version `%s`, Git commit `%s`\n' \
+            "$repository" "$digest" "$FROM_VERSION" "$previous_commit" >> "$GITHUB_STEP_SUMMARY"
+        for component in api operator agent; do
+            verify_release_image "$FROM_REGISTRY/gameplane/$component" "$FROM_VERSION" "$previous_commit" "$GAMEPLANE_PREVIOUS_RELEASE_DIGESTS"
+        done
+    fi
+    echo "installing PUBLISHED chart ${CHART_REF}, version ${FROM_VERSION}"
     # Value overrides mirror e2e.sh's, and are all about the harness rather
     # than the version under test:
     #   - api memory 1Gi: argon2id uses ~64Mi per hash and the 256Mi default
@@ -85,7 +107,8 @@ EOF
     # Note what is NOT overridden: image.registry/image.tag. The old release
     # must pull its own published GHCR images, or this tests nothing.
     helm install gameplane "${CHART_REF}" \
-        --version "${FROM_VERSION}" \
+        --kube-context "kind-${CLUSTER}" \
+        "${chart_args[@]}" \
         --namespace gameplane-system --create-namespace \
         --set "ingress.enabled=false" \
         --set "web.enabled=false" \
@@ -108,7 +131,7 @@ down)
     ;;
 
 *)
-    echo "usage: $0 {up|down} [cluster-name] [tag]" >&2
+    echo "usage: $0 {up|install|down} [cluster-name] [tag]" >&2
     exit 2
     ;;
 esac