release: stage and verify signed packages

.github/workflows/promote-release.yml

@@ -0,0 +1,196 @@
+name: Promote Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: Published signed release tag (vMAJOR.MINOR.PATCH or vMAJOR.MINOR.PATCH-rcNUMBER)
+        required: true
+        type: string
+      build_run_id:
+        description: Successful Release workflow run containing the image digests
+        required: true
+        type: string
+
+permissions:
+  contents: read
+  actions: read
+  packages: write
+
+jobs:
+  promote:
+    runs-on: ubuntu-latest
+    environment: orchestrator-production
+    steps:
+    - name: Check out the release tag
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.tag }}
+
+    - name: Validate signed release and build run
+      id: preflight
+      env:
+        GH_TOKEN: ${{ github.token }}
+        TAG: ${{ inputs.tag }}
+        BUILD_RUN_ID: ${{ inputs.build_run_id }}
+      run: |
+        set -euo pipefail
+        [[ "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?$ ]]
+        [[ "${BUILD_RUN_ID}" =~ ^[0-9]+$ ]]
+
+        tag_sha=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${TAG}" --jq .sha)
+        test "$(git rev-parse HEAD)" = "${tag_sha}"
+
+        run_json=$(gh run view "${BUILD_RUN_ID}" -R "${GITHUB_REPOSITORY}" \
+          --json databaseId,workflowName,event,headBranch,headSha,status,conclusion)
+        test "$(jq -r .databaseId <<<"${run_json}")" = "${BUILD_RUN_ID}"
+        test "$(jq -r .workflowName <<<"${run_json}")" = Release
+        test "$(jq -r .event <<<"${run_json}")" = push
+        test "$(jq -r .headBranch <<<"${run_json}")" = "${TAG}"
+        test "$(jq -r .headSha <<<"${run_json}")" = "${tag_sha}"
+        test "$(jq -r .status <<<"${run_json}")" = completed
+        test "$(jq -r .conclusion <<<"${run_json}")" = success
+
+        release_json=$(gh release view "${TAG}" -R "${GITHUB_REPOSITORY}" \
+          --json assets,isDraft,isPrerelease,tagName,targetCommitish)
+        test "$(jq -r .isDraft <<<"${release_json}")" = false
+        test "$(jq -r .tagName <<<"${release_json}")" = "${TAG}"
+        test "$(jq -r .targetCommitish <<<"${release_json}")" = "${tag_sha}"
+
+        version=${TAG#v}
+        prerelease=false
+        [[ "${TAG}" != *-rc* ]] || prerelease=true
+        test "$(jq -r .isPrerelease <<<"${release_json}")" = "${prerelease}"
+
+        expected=$(
+          {
+            script/release-artifacts expected-unsigned "${version}"
+            printf '%s\n' \
+              "orchestrator-${version}-linux-amd64.tar.gz.asc" \
+              "orchestrator-${version}-linux-arm64.tar.gz.asc" \
+              SHA256SUMS SHA256SUMS.asc
+          } | LC_ALL=C sort
+        )
+        actual=$(jq -r '.assets[].name' <<<"${release_json}" | LC_ALL=C sort)
+        test "${actual}" = "${expected}"
+
+        printf 'tag=%s\nversion=%s\nprerelease=%s\n' \
+          "${TAG}" "${version}" "${prerelease}" >>"${GITHUB_OUTPUT}"
+
+    - name: Verify signed release assets
+      env:
+        GH_TOKEN: ${{ github.token }}
+        TAG: ${{ steps.preflight.outputs.tag }}
+        SIGNING_FINGERPRINT: 653F85BB38256DF8A96206C3E8CA2E8D8217C97E
+        PUBLIC_KEY_URL: https://repo.proxysql.com/ProxySQL/repo_pub_key
+      run: |
+        set -euo pipefail
+        verification_root=$(mktemp -d)
+        trap 'rm -rf "${verification_root}"' EXIT
+        assets=${verification_root}/assets
+        export GNUPGHOME=${verification_root}/gnupg
+        mkdir -m 700 "${assets}" "${GNUPGHOME}"
+
+        gh release download "${TAG}" -R "${GITHUB_REPOSITORY}" --dir "${assets}"
+        curl -fSLo "${verification_root}/repo_pub_key" "${PUBLIC_KEY_URL}"
+        primary_fingerprints=$(
+          gpg --batch --show-keys --with-colons --with-fingerprint \
+            "${verification_root}/repo_pub_key" |
+            awk -F: '
+              $1 == "pub" {want_primary_fingerprint=1; next}
+              want_primary_fingerprint && $1 == "fpr" {
+                print $10
+                want_primary_fingerprint=0
+              }
+            '
+        )
+        test "${primary_fingerprints}" = "${SIGNING_FINGERPRINT}"
+        gpg --batch --import "${verification_root}/repo_pub_key" >/dev/null 2>&1
+
+        cd "${assets}"
+        expected_checksum_names=$(
+          "${GITHUB_WORKSPACE}/script/release-artifacts" expected-unsigned \
+            "${{ steps.preflight.outputs.version }}"
+        )
+        test "$(wc -l <SHA256SUMS)" -eq 14
+        actual_checksum_names=$(
+          sed -nE 's/^[0-9a-f]{64}  ([^/]+)$/\1/p' SHA256SUMS | LC_ALL=C sort
+        )
+        test "${actual_checksum_names}" = "${expected_checksum_names}"
+        signature_status=$(gpg --batch --status-fd=1 \
+          --verify SHA256SUMS.asc SHA256SUMS 2>/dev/null)
+        awk -v expected="${SIGNING_FINGERPRINT}" '
+          $1 == "[GNUPG:]" && $2 == "VALIDSIG" &&
+            ($3 == expected || $NF == expected) {valid=1}
+          END {exit valid ? 0 : 1}
+        ' <<<"${signature_status}"
+        sha256sum -c SHA256SUMS
+
+    - name: Download image digests from the selected build
+      uses: actions/download-artifact@v4
+      with:
+        run-id: ${{ inputs.build_run_id }}
+        pattern: digests-*
+        path: /tmp/digests
+        merge-multiple: true
+        github-token: ${{ github.token }}
+
+    - name: Validate image digests
+      run: |
+        set -euo pipefail
+        mapfile -t digests < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | LC_ALL=C sort)
+        test "${#digests[@]}" -eq 2
+        test "${digests[0]}" != "${digests[1]}"
+        for digest in "${digests[@]}"; do
+          [[ "${digest}" =~ ^[0-9a-f]{64}$ ]]
+        done
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to GHCR
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Extract image metadata
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: ghcr.io/proxysql/orchestrator
+        tags: |
+          type=semver,pattern={{version}},value=${{ steps.preflight.outputs.tag }}
+          type=semver,pattern={{major}}.{{minor}},value=${{ steps.preflight.outputs.tag }}
+          type=raw,value=latest,enable=${{ steps.preflight.outputs.prerelease == 'false' }}
+
+    - name: Promote image digests
+      working-directory: /tmp/digests
+      env:
+        METADATA_JSON: ${{ steps.meta.outputs.json }}
+      run: |
+        set -euo pipefail
+        mapfile -t digests < <(find . -maxdepth 1 -type f -printf '%f\n' | LC_ALL=C sort)
+        mapfile -t tags < <(jq -r '.tags[]' <<<"${METADATA_JSON}")
+        tag_args=()
+        sources=()
+        for tag in "${tags[@]}"; do
+          tag_args+=(-t "${tag}")
+        done
+        for digest in "${digests[@]}"; do
+          sources+=("ghcr.io/proxysql/orchestrator@sha256:${digest}")
+        done
+        docker buildx imagetools create "${tag_args[@]}" "${sources[@]}"
+
+    - name: Verify multi-architecture image
+      env:
+        VERSION: ${{ steps.preflight.outputs.version }}
+      run: |
+        set -euo pipefail
+        actual=$(docker buildx imagetools inspect --raw \
+          "ghcr.io/proxysql/orchestrator:${VERSION}" |
+          jq -r '.manifests[].platform | "\(.os)/\(.architecture)"' |
+          LC_ALL=C sort -u)
+        expected=$'linux/amd64\nlinux/arm64'
+        test "${actual}" = "${expected}"

.github/workflows/release.yml

@@ -6,11 +6,11 @@ on:
       - 'v*'
 
 permissions:
-  contents: write
+  contents: read
   packages: write
 
 jobs:
-  build-and-release:
+  build-packages:
     strategy:
       fail-fast: false
       matrix:
@@ -55,13 +55,47 @@ jobs:
         cp /tmp/orchestrator-release/*.rpm    dist/ 2>/dev/null || true
         ls -la dist/
 
-    - name: Upload release assets
-      uses: softprops/action-gh-release@v2
+    - name: Upload unsigned package stage
+      uses: actions/upload-artifact@v4
+      with:
+        name: orchestrator-packages-${{ matrix.goarch }}
+        path: dist/*
+        if-no-files-found: error
+        retention-days: 14
+
+  validate-package-stage:
+    needs: build-packages
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Download package stages
+      uses: actions/download-artifact@v4
+      with:
+        pattern: orchestrator-packages-*
+        path: dist
+        merge-multiple: true
+
+    - name: Validate unsigned package stage
+      env:
+        TAG_NAME: ${{ github.ref_name }}
+        EXPECTED_SHA: ${{ github.sha }}
+      run: |
+        set -euo pipefail
+        version="${TAG_NAME#v}"
+        test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}"
+        script/release-artifacts validate-unsigned "${version}" dist
+        script/release-artifacts validate-packages "${version}" dist
+        script/release-artifacts write-unsigned-checksums \
+          "${version}" dist unsigned-SHA256SUMS
+
+    - name: Upload unsigned manifest
+      uses: actions/upload-artifact@v4
       with:
-        files: dist/*
-        generate_release_notes: true
-        draft: false
-        prerelease: ${{ contains(github.ref_name, 'rc') }}
+        name: orchestrator-unsigned-manifest
+        path: unsigned-SHA256SUMS
+        if-no-files-found: error
+        retention-days: 14
 
   docker-build:
     strategy:
@@ -111,48 +145,4 @@ jobs:
         name: digests-${{ matrix.arch }}
         path: /tmp/digests/*
         if-no-files-found: error
-        retention-days: 1
-
-  docker-merge:
-    needs: docker-build
-    runs-on: ubuntu-latest
-    steps:
-    - name: Download digests
-      uses: actions/download-artifact@v4
-      with:
-        pattern: digests-*
-        path: /tmp/digests
-        merge-multiple: true
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
-    - name: Login to GHCR
-      uses: docker/login-action@v3
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Extract metadata
-      id: meta
-      uses: docker/metadata-action@v5
-      with:
-        images: ghcr.io/proxysql/orchestrator
-        tags: |
-          type=semver,pattern={{version}}
-          type=semver,pattern={{major}}.{{minor}}
-          type=raw,value=latest,enable=${{ !contains(github.ref_name, 'rc') }}
-
-    - name: Create manifest list and push
-      working-directory: /tmp/digests
-      run: |
-        docker buildx imagetools create \
-          $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-          $(printf 'ghcr.io/proxysql/orchestrator@sha256:%s ' *)
-
-    - name: Inspect image
-      env:
-        VERSION: ${{ steps.meta.outputs.version }}
-      run: |
-        docker buildx imagetools inspect "ghcr.io/proxysql/orchestrator:${VERSION}"
+        retention-days: 14

docs/download.md

@@ -3,6 +3,10 @@
 `orchestrator` is released as open source and is available at [GitHub](https://github.com/proxysql/orchestrator).
 Find official releases in https://github.com/proxysql/orchestrator/releases
 
+See [Package signature verification](package-signatures.md) before installing
+a downloaded release artifact. Package signing will begin with a future
+release; the verification page records the rollout status and signing key.
+
 `orchestrator` packages can be found in https://packagecloud.io/github/orchestrator
 
 For developers: `orchestrator` is go-gettable. Issue:

docs/install.md

@@ -2,12 +2,18 @@
 
 For production deployments, see [Orchestrator deployment](deployment.md). The following text walks you through the manual way of installation and the necessary configuration to make it work.
 
+Before installing a release artifact, follow [Package signature
+verification](package-signatures.md). Signature availability depends on the
+release version documented there.
+
 The following assumes you will be using the same machine for both the `orchestrator` binary and the MySQL backend.
 If not, replace `127.0.0.1` with appropriate host name. Replace `orch_backend_password` with your own super secret password.
 
 #### Which package do I need?
 
-Each release publishes three package variants (as `.deb`, `.rpm`, and `.tar.gz`, for both `amd64` and `arm64`):
+Each release publishes three package variants for both `amd64` and `arm64`.
+All three have `.deb` and `.rpm` packages; only the full `orchestrator` variant
+has a `.tar.gz` archive:
 
 - **`orchestrator`** — the full server install. Contains the `orchestrator` binary, the web UI resources, sample config files, and the systemd unit. This is what you want on the host(s) that will run the `orchestrator` service with the HTTP API and web interface.
 - **`orchestrator-cli`** — the `orchestrator` binary only. Pick this when you want to run `orchestrator` from the command line (CLI mode) or as an HTTP API server without the web UI, and you do not need the sample configs or systemd unit.