From e0ec1acd5c417d67ac835ddad35c6a0f1ef3a3da Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 10 Nov 2026 20:31:19 +0100 Subject: [PATCH] CI: release automation for the sdist, GitHub release and PyPI Like on master, pushing a release tag now does the release: - the "release" job (.github/workflows/release.yml) builds the sdist, checks that it has the version of the tag and that it can be installed, attests its build provenance and drafts the GitHub release with the sdist and all standalone binaries attached, - the "pypi" job uploads the sdist to PyPI via trusted publishing (gated by the "pypi" environment). Every release asset comes with the sigstore bundle of its provenance attestation as .sigstore.jsonl, for offline verification: the binary jobs now hand their bundles over together with the binaries. The sdist is no longer built and GPG-signed locally, so scripts/sdist-sign is removed. The upload to PyPI is done by CI, so scripts/upload-pypi is removed, too. The release checklist is updated accordingly. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 107 +++++++++++++++- .github/workflows/release.yml | 224 ++++++++++++++++++++++++++++++++++ docs/binaries/00_README.txt | 11 +- docs/development.rst | 44 +++---- scripts/sdist-sign | 20 --- scripts/upload-pypi | 18 --- 6 files changed, 353 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/release.yml delete mode 100755 scripts/sdist-sign delete mode 100755 scripts/upload-pypi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75cd27f1f3..fa13fc38c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -253,11 +253,32 @@ jobs: ls -l artifacts/ - name: Attest binaries provenance (${{ matrix.binary }}) + id: attest-binaries if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 with: subject-path: 'artifacts/*' + - name: Name the sigstore bundle after the binaries (${{ matrix.binary }}) + # The attestation is also stored in the GitHub attestations API, but + # fetching it from there needs a recent gh and a GitHub token - so the + # sigstore bundle rides along with the binaries and the release job + # attaches it to the release as .sigstore.jsonl, just like it does + # for the sdist, see .github/workflows/release.yml. + # + # One attestation covers both the single-file binary and the .tgz, so the + # same bundle is stored under both names - one for each asset. + if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} + env: + BUNDLE_PATH: ${{ steps.attest-binaries.outputs.bundle-path }} + run: | + set -euxo pipefail + # the glob is expanded before the loop body creates any file: + for asset in artifacts/*; do + cp "$BUNDLE_PATH" "$asset.sigstore.jsonl" + done + ls -l artifacts/ + - name: Upload binaries (${{ matrix.binary }}) if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -487,6 +508,31 @@ jobs: ;; esac + - name: Attest provenance + id: attest-binaries + if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries + uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + with: + subject-path: 'artifacts/*' + + - name: Name the sigstore bundle after the binaries + # The sigstore bundle rides along with the binaries and the release job + # attaches it to the release as .sigstore.jsonl - see the same step + # in native_tests for the details. One attestation covers both the + # single-file binary and the .tgz, so the same bundle is stored under both + # names - one for each asset. This is also why the attestation is made + # before the artifact is uploaded and not after it. + if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries + env: + BUNDLE_PATH: ${{ steps.attest-binaries.outputs.bundle-path }} + run: | + set -euxo pipefail + # the glob is expanded before the loop body creates any file: + for asset in artifacts/*; do + cp "$BUNDLE_PATH" "$asset.sigstore.jsonl" + done + ls -l artifacts/ + - name: Upload artifacts if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -495,8 +541,61 @@ jobs: path: artifacts/* if-no-files-found: ignore - - name: Attest provenance - if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries - uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + + release: + # Build the sdist and draft the GitHub release with the binaries the jobs + # above built for this tag, see .github/workflows/release.yml. + # + # vm_tests is continue-on-error (a flaky BSD VM must not stop a release), so + # this waits for it, but only requires native_tests to have succeeded. A + # binary that did not get built is warned about while drafting the release. + if: ${{ !cancelled() && startsWith(github.ref, 'refs/tags/') && needs.native_tests.result == 'success' }} + needs: [native_tests, vm_tests] + + permissions: + contents: write + id-token: write + attestations: write + + uses: ./.github/workflows/release.yml + + pypi: + # Upload the sdist the release job built to PyPI. + # + # This job can not live in release.yml with the rest of the release code: + # PyPI trusted publishing does not work from a reusable workflow, see + # https://docs.pypi.org/trusted-publishers/troubleshooting/ + # + # One-time setup, so that no API token has to be stored anywhere: + # - on pypi.org, add a trusted publisher to the "borgbackup" project: + # owner "borgbackup", repository "borg", workflow "ci.yml", + # environment "pypi". + # - create the "pypi" environment in the repository settings. Configuring + # required reviewers for it makes this (irreversible) upload wait for an + # approval, which is the last chance to stop a release. + if: ${{ startsWith(github.ref, 'refs/tags/') && needs.release.result == 'success' }} + needs: [release] + + runs-on: ubuntu-24.04 + timeout-minutes: 30 + + environment: + name: pypi + url: https://pypi.org/project/borgbackup/${{ github.ref_name }} + + permissions: + contents: read + id-token: write # trusted publishing + + steps: + - name: Get the sdist built by the release job + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - subject-path: 'artifacts/*' + name: sdist + path: dist + + - name: What we are about to upload + run: ls -l dist/ + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..33b7e47bc3 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,224 @@ +# Release automation: build the source distribution and create a GitHub release +# with it and the standalone binaries. +# +# This is called by ci.yml when a tag is pushed, and it deliberately runs as +# part of that same workflow run: that is where the binaries for the tag are +# built, and artifacts can only be downloaded within the run that created them. +# +# The GitHub release is created as a *draft* on purpose: +# - the release notes want a human, +# - and the binaries should be tried out before the release becomes visible. +# Publishing the draft is a single click in the GitHub UI. +# +# Every release asset has a build provenance attestation, and the sigstore bundle +# of each attestation is attached to the release as .sigstore.jsonl, for +# offline verification. The sdist is attested here, the binaries by the jobs in +# ci.yml that build them - those hand their bundles over in the artifact that +# carries the binaries. +# +# The upload to PyPI is *not* done here, but by the "pypi" job in ci.yml: PyPI +# trusted publishing can not be used from a reusable workflow, see +# https://docs.pypi.org/trusted-publishers/troubleshooting/ - so that job has to +# live in a workflow that is triggered by an event. This job hands the sdist +# over to it as a workflow artifact. + +name: Release + +on: + workflow_call: + +permissions: + contents: read + +env: + # The standalone binaries expected for a release: for every platform the + # single-file binary and, as a .tgz, the single-directory variant. See the + # "binary" entries of the native_tests matrix and the "artifact_prefix" + # entries of the vm_tests matrix in ci.yml - keep in sync. + EXPECTED_ASSETS: >- + borg-linux-glibc235-x86_64-gh + borg-linux-glibc235-x86_64-gh.tgz + borg-linux-glibc235-arm64-gh + borg-linux-glibc235-arm64-gh.tgz + borg-macos-15-arm64-gh + borg-macos-15-arm64-gh.tgz + borg-macos-15-x86_64-gh + borg-macos-15-x86_64-gh.tgz + borg-freebsd-14-x86_64-gh + borg-freebsd-14-x86_64-gh.tgz + +jobs: + github_release: + name: Draft the GitHub release + runs-on: ubuntu-24.04 + timeout-minutes: 30 + + permissions: + contents: write # to create the release + id-token: write # to attest the sdist + attestations: write + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Just fetching one commit is not enough for setuptools-scm, so we fetch all. + fetch-depth: 0 + fetch-tags: true + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + + - name: Install Linux packages + run: | + sudo apt-get update + sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev pkg-config build-essential + sudo apt-get install -y libxxhash-dev + + - name: Build the sdist + run: | + set -euxo pipefail + python -m pip install --upgrade pip build twine + # only a sdist: we do not publish wheels, they would be platform specific. + python -m build --sdist + twine check dist/* + ls -l dist/ + + - name: Check that the sdist has the version of the tag + # If the checkout is modified, setuptools-scm does not use the tag as the + # version, but silently guesses the next one and appends .devN. Such a + # release must not be uploaded to PyPI. + env: + TAG: ${{ github.ref_name }} + run: | + set -euxo pipefail + python - "$TAG" dist/borgbackup-*.tar.gz <<'EOF' + import sys + from packaging.utils import parse_sdist_filename + from packaging.version import Version + tag, sdist = sys.argv[1], sys.argv[2] + _, version = parse_sdist_filename(sdist.rpartition("/")[2]) + if version != Version(tag): + raise SystemExit(f"sdist version {version} is not the tag version {tag} - modified checkout?") + print(f"sdist version {version} is the tag version.") + EOF + + - name: Check that the sdist is complete and installable + # A release that cannot be installed from PyPI is the worst kind of + # release, and nothing else in CI ever installs borg from a sdist or + # without the development requirements. + run: | + set -euxo pipefail + python -m venv "$RUNNER_TEMP/venv-sdist" + "$RUNNER_TEMP/venv-sdist/bin/pip" install --upgrade pip + "$RUNNER_TEMP/venv-sdist/bin/pip" install dist/borgbackup-*.tar.gz + "$RUNNER_TEMP/venv-sdist/bin/borg" --version + "$RUNNER_TEMP/venv-sdist/bin/borg" --help > /dev/null + + - name: Attest the sdist provenance + id: attest-sdist + uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + with: + subject-path: 'dist/*.tar.gz' + + - name: Name the sigstore bundle after the sdist + # The attestation is also stored in the GitHub attestations API, but + # fetching it from there needs a recent gh and a GitHub token - so attach + # the sigstore bundle to the release as .sigstore.jsonl, which + # allows offline/token-less verification with generic sigstore tooling + # (cosign, gh attestation verify --bundle). + env: + BUNDLE_PATH: ${{ steps.attest-sdist.outputs.bundle-path }} + run: | + set -euxo pipefail + sdist=$(basename dist/borgbackup-*.tar.gz) + cp "$BUNDLE_PATH" "dist/$sdist.sigstore.jsonl" + + - name: Download the binaries built for this tag + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + path: assets + # the binary artifacts are all named like the binaries they contain + pattern: 'borg-*-gh' + merge-multiple: true + + - name: Check that no binary is missing + run: | + set -uo pipefail + ls -l assets/ || true + missing="" + for asset in $EXPECTED_ASSETS; do + test -f "assets/$asset" || missing="$missing $asset" + done + if [ -n "$missing" ]; then + # not an error: vm_tests is continue-on-error, so e.g. a flaky FreeBSD + # VM should not stop the release - but do not lose it silently either. + echo "::warning::binaries missing from this release:$missing" + fi + + - name: Create the draft release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: | + set -euxo pipefail + # 1.4.0b1, 1.4.0rc1 and friends are pre-releases, 1.4.1 is not. + prerelease="" + case "$TAG" in *a*|*b*|*rc*) prerelease="--prerelease" ;; esac + cat > release-notes.md <\`. + + Every attestation is additionally attached as a [sigstore](https://www.sigstore.dev/) + bundle \`.sigstore.jsonl\`, so it can be verified offline and without a + GitHub token, e.g. with cosign: + + \`\`\` + cosign verify-blob borgbackup-$TAG.tar.gz \\ + --bundle borgbackup-$TAG.tar.gz.sigstore.jsonl \\ + --certificate-identity https://github.com/borgbackup/borg/.github/workflows/release.yml@refs/tags/$TAG \\ + --certificate-oidc-issuer https://token.actions.githubusercontent.com + \`\`\` + + The binaries are built by \`ci.yml\`, not by \`release.yml\`, so their + \`--certificate-identity\` is + \`https://github.com/borgbackup/borg/.github/workflows/ci.yml@refs/tags/$TAG\`. + EOF + mkdir -p assets # there may not have been any artifact to download + if gh release view "$TAG" > /dev/null 2>&1; then + # a re-run of this job: keep the (possibly already edited) release and + # just replace its assets. + gh release upload "$TAG" --clobber dist/*.tar.gz dist/*.sigstore.jsonl $(find assets -type f | sort) + else + gh release create "$TAG" \ + --draft $prerelease \ + --title "borg $TAG" \ + --notes-file release-notes.md \ + dist/*.tar.gz dist/*.sigstore.jsonl $(find assets -type f | sort) + fi + gh release view "$TAG" --json isDraft,isPrerelease,assets + + - name: Keep the sdist for the PyPI upload + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: sdist + path: dist/*.tar.gz + if-no-files-found: error diff --git a/docs/binaries/00_README.txt b/docs/binaries/00_README.txt index c6bab7f215..a1ff70b169 100644 --- a/docs/binaries/00_README.txt +++ b/docs/binaries/00_README.txt @@ -56,7 +56,7 @@ To check the GPG signature, download both the file and the corresponding signature (*.asc file) and then (on the shell) type, for example: gpg --recv-keys 9F88FB52FAF7B393 - gpg --verify borgbackup.tar.gz.asc borgbackup.tar.gz + gpg --verify borg-linux64.asc borg-linux64 The files are signed by: @@ -83,6 +83,15 @@ Practical example (Linux, 1.4.3 tag): If verification succeeds, gh prints a summary stating the subject (your file), that it was attested by GitHub Actions, and the job/workflow reference. +The source distribution (borgbackup-X.Y.Z.tar.gz, also on PyPI) is built on +GitHub, too, and has a provenance attestation that is verified the same way: + + gh attestation verify --repo borgbackup/borg --source-ref refs/tags/X.Y.Z borgbackup-X.Y.Z.tar.gz + +Its sigstore bundle is attached to the GitHub release as +borgbackup-X.Y.Z.tar.gz.sigstore.jsonl, so it can also be verified offline +(gh attestation verify --bundle) or with other sigstore tooling. + Installing ---------- diff --git a/docs/development.rst b/docs/development.rst index 509494bd3d..b7e84d4384 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -392,7 +392,6 @@ Checklist: git tag -s -m "tagged/signed release X.Y.Z" X.Y.Z - Push the release PR branch to GitHub, make a pull request. -- Also push the release tag. - Create a clean repo and use it for the following steps:: git clone borg borg-clean @@ -404,23 +403,22 @@ Checklist: - Optional: run tox and/or binary builds on all supported platforms via vagrant, check for test failures. This is now optional as we do platform testing and binary building on GitHub. -- Create sdist, sign it, upload release to (test) PyPi: - - :: - - scripts/sdist-sign X.Y.Z - scripts/upload-pypi X.Y.Z test - scripts/upload-pypi X.Y.Z - - Note: the signature is not uploaded to PyPi any more, but we upload it to - github releases. - -- When GitHub CI looks good on the release PR, merge it and then check "Actions": - GitHub will create binary assets after the release PR is merged within the - CI testing of the merge. Check the "Upload binaries" step on Ubuntu (AMD/Intel - and ARM64) and macOS (Intel and ARM64), fetch the ZIPs with the binaries. -- Unpack the ZIPs and test the binaries, upload the binaries to the GitHub - release page (borg-OS-SPEC-ARCH-gh and borg-OS-SPEC-ARCH-gh.tgz). +- When GitHub CI looks good on the release PR, merge it and push the release tag. + + Pushing the tag makes CI build the standalone binaries and then release: + the ``release`` job (``.github/workflows/release.yml``) builds the sdist, + checks that it has the version of the tag and that it can be installed, + attests its provenance and drafts the GitHub release with the sdist and all + standalone binaries attached. Each of these assets comes with the sigstore + bundle of its provenance attestation, as ``.sigstore.jsonl``, so that + it can be verified offline. The ``pypi`` job then uploads the sdist to PyPI. + + If the ``pypi`` environment has required reviewers configured, the upload to + PyPI waits for an approval - that is the last chance to stop it. Also watch + out for a warning while the release is drafted: it means that a binary is + missing because its build did not succeed. +- Download the binaries from the drafted release and test them. +- Review the release notes and publish the drafted GitHub release. - Close the release milestone on GitHub. - `Update borgbackup.org `_ with the @@ -430,13 +428,3 @@ Checklist: - Mailing list. - Mastodon / BlueSky / X (aka Twitter). - IRC channel (change ``/topic``). - -- Create a GitHub release, include: - - - pypi dist package and signature - - Standalone binaries (see above for how to create them). - - - For macOS binaries **with** FUSE support, document the macFUSE version - in the README of the binaries. macFUSE uses a kernel extension that needs - to be compatible with the code contained in the binary. - - A link to ``CHANGES.rst``. diff --git a/scripts/sdist-sign b/scripts/sdist-sign deleted file mode 100755 index 81dd923658..0000000000 --- a/scripts/sdist-sign +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -R=$1 - -if [ "$R" = "" ]; then - echo "Usage: sdist-sign 1.2.3" - exit -fi - -if [ "$QUBES_GPG_DOMAIN" = "" ]; then - GPG=gpg -else - GPG=qubes-gpg-client-wrapper -fi - -python -m build - -D=dist/borgbackup-$R.tar.gz - -$GPG --detach-sign --local-user "Thomas Waldmann" --armor --output $D.asc $D diff --git a/scripts/upload-pypi b/scripts/upload-pypi deleted file mode 100755 index 0098e718bc..0000000000 --- a/scripts/upload-pypi +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -R=$1 - -if [ "$R" = "" ]; then - echo "Usage: upload-pypi 1.2.3 [test]" - exit -fi - -if [ "$2" = "test" ]; then - export TWINE_REPOSITORY=testborgbackup -else - export TWINE_REPOSITORY=borgbackup -fi - -D=dist/borgbackup-$R.tar.gz - -twine upload $D