diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4f6810b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,324 @@ +name: Release + +# Tags only. A release is a deliberate act with a name, not a side effect of +# merging — and `docs/releases.md` says only tagged releases are supported, which +# has to be true of what this workflow will sign. +on: + push: + tags: ["v*.*.*"] + workflow_dispatch: + inputs: + tag: + description: "Existing tag to re-publish (recovery only)" + required: true + type: string + +# Least privilege, then widened per job. `id-token: write` is what makes the +# keyless signing below possible: cosign exchanges the job's OIDC identity for a +# short-lived certificate, so this project holds no signing key that could leak, +# expire, or need rotating. +permissions: + contents: read + +env: + REGISTRY: ghcr.io + +jobs: + # The tag is checked before anything is built or published. A release whose + # version files disagree, or whose CHANGELOG has no entry, is a release an + # operator cannot reason about — and once an artifact is signed and pushed it + # is public, so the only place to catch it is here. + verify: + name: verify the tag is releasable + runs-on: ubuntu-latest + outputs: + version: ${{ steps.resolve.outputs.version }} + tag: ${{ steps.resolve.outputs.tag }} + # Every publishing job checks out *this*, never the dispatch input. The + # input is a string a human typed; this is a ref that has been proved to + # exist and to be a release tag. + ref: ${{ steps.resolve.outputs.ref }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + # Deliberately *not* the dispatch input: this checkout exists to look + # up whether that input names a real tag, so taking it on faith first + # would be checking the ref against itself. Full history, because a + # shallow clone has no tags to check against. + fetch-depth: 0 + # Nothing here pushes with git; the registry and release steps use + # their own tokens. Leaving the checkout credential in .git/config + # would put a write-capable token inside every artifact built from + # this tree. + persist-credentials: false + + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + with: + enable-cache: true + + # The gate `on: push: tags:` gives for free, and that `workflow_dispatch` + # does not. Without it a recovery run with `tag: main` would check out a + # branch, pass every check below (main declares the current version), and + # publish untagged code as that release — over the image an operator has + # already verified. So the input is validated as an existing release tag + # before anything is checked out from it, and every later job uses the + # resolved ref rather than the string. + - name: The ref is an existing release tag + id: resolve + env: + REF_NAME: ${{ inputs.tag || github.ref_name }} + run: | + set -euo pipefail + # Anchored, so a ref like `v1.2.3-attacker` or `refs/heads/v1.2.3` is + # refused rather than trimmed into something that looks valid. + if ! printf '%s' "$REF_NAME" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "'$REF_NAME' is not a vX.Y.Z release tag" >&2 + exit 1 + fi + if ! git rev-parse --verify --quiet "refs/tags/$REF_NAME" > /dev/null; then + echo "refs/tags/$REF_NAME does not exist; tag the commit first" >&2 + exit 1 + fi + # Now, and only now, move the working tree to the tag so the checks + # below read the code that would actually be published. + git checkout --detach "refs/tags/$REF_NAME" + printf 'tag=%s\n' "$REF_NAME" >> "$GITHUB_OUTPUT" + printf 'ref=refs/tags/%s\n' "$REF_NAME" >> "$GITHUB_OUTPUT" + printf 'version=%s\n' "${REF_NAME#v}" >> "$GITHUB_OUTPUT" + + - name: The declared versions agree with each other and with the tag + env: + VERSION: ${{ steps.resolve.outputs.version }} + run: | + set -euo pipefail + uv run python scripts/check_version.py + declared="$(uv run python -c \ + 'import tomllib,pathlib;print(tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]["version"])')" + if [ "$declared" != "$VERSION" ]; then + echo "tag v$VERSION does not match the declared version $declared" >&2 + exit 1 + fi + + - name: The CHANGELOG documents this version + env: + VERSION: ${{ steps.resolve.outputs.version }} + run: | + set -euo pipefail + if ! grep -qF "## [$VERSION]" CHANGELOG.md; then + echo "CHANGELOG.md has no '## [$VERSION]' entry; the release notes come from it" >&2 + exit 1 + fi + + images: + name: build, sign, and attest the role images + needs: verify + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # Keyless signing and provenance attestation both need the job's OIDC + # identity; neither needs a stored key. + id-token: write + attestations: write + strategy: + matrix: + role: [api, engine] + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + # The ref the verify job proved is a release tag, never the raw input. + ref: ${{ needs.verify.outputs.ref }} + # Nothing here pushes with git; the registry and release steps use + # their own tokens. Leaving the checkout credential in .git/config + # would put a write-capable token inside every artifact built from + # this tree. + persist-credentials: false + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Built here rather than promoted from the CI job: CI builds to verify + # deployment invariants and throws the image away, and promoting an + # untagged artifact across workflows would mean signing something whose + # provenance is a second workflow's word for it. + - name: Build and push + id: build + env: + IMAGE: ${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.role }} + VERSION: ${{ needs.verify.outputs.version }} + ROLE: ${{ matrix.role }} + run: | + set -euo pipefail + image="$(echo "$IMAGE" | tr '[:upper:]' '[:lower:]')" + docker build -f "deploy/docker/$ROLE.Dockerfile" \ + -t "$image:$VERSION" -t "$image:latest" . + docker push "$image:$VERSION" + docker push "$image:latest" + digest="$(docker inspect --format='{{index .RepoDigests 0}}' "$image:$VERSION" | cut -d@ -f2)" + printf 'image=%s\n' "$image" >> "$GITHUB_OUTPUT" + printf 'digest=%s\n' "$digest" >> "$GITHUB_OUTPUT" + + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + # Signed by digest, never by tag. A tag is a mutable pointer the registry + # can move; a digest is the artifact. Verifying a signature on a tag would + # prove something about a name rather than about the bytes that run. + - name: Sign the image + env: + REFERENCE: ${{ steps.build.outputs.image }}@${{ steps.build.outputs.digest }} + run: cosign sign --yes "$REFERENCE" + + - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0 + with: + image: ${{ steps.build.outputs.image }}@${{ steps.build.outputs.digest }} + format: spdx-json + artifact-name: ${{ matrix.role }}-sbom.spdx.json + output-file: ${{ matrix.role }}-sbom.spdx.json + + # The SBOM is attached to the image and signed with it, so "does this + # release contain the library in that advisory?" is answerable from the + # registry without rebuilding anything. + - name: Attach and sign the SBOM + env: + REFERENCE: ${{ steps.build.outputs.image }}@${{ steps.build.outputs.digest }} + ROLE: ${{ matrix.role }} + run: | + set -euo pipefail + cosign attest --yes --predicate "$ROLE-sbom.spdx.json" --type spdxjson "$REFERENCE" + + - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ steps.build.outputs.image }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ matrix.role }}-sbom + path: ${{ matrix.role }}-sbom.spdx.json + + chart: + name: package and sign the Helm chart + needs: [verify, images] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + # The ref the verify job proved is a release tag, never the raw input. + ref: ${{ needs.verify.outputs.ref }} + # Nothing here pushes with git; the registry and release steps use + # their own tokens. Leaving the checkout credential in .git/config + # would put a write-capable token inside every artifact built from + # this tree. + persist-credentials: false + + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + # The chart's own version moves independently of the application's, so it + # is read from the chart rather than assumed to be the tag. `appVersion` is + # the one the verify job pinned to the tag. + - name: Package and push + id: package + # Every expansion goes through the environment rather than into the shell + # text: `github.actor` and `github.repository` are attacker-influenceable + # in the general case, and a `run:` block that interpolates them is a + # shell-injection sink even when this repository's values are benign. + env: + VERSION: ${{ needs.verify.outputs.version }} + REGISTRY_USER: ${{ github.actor }} + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + chart_version="$(helm show chart deploy/helm/icebergsst | awk '/^version:/ {print $2}')" + helm package deploy/helm/icebergsst --destination dist + printf '%s' "$REGISTRY_TOKEN" | helm registry login "$REGISTRY" \ + --username "$REGISTRY_USER" --password-stdin + repository="$(echo "oci://$REGISTRY/$REPOSITORY/charts" | tr '[:upper:]' '[:lower:]')" + helm push "dist/icebergsst-$chart_version.tgz" "$repository" 2>&1 | tee push.log + digest="$(awk '/^Digest:/ {print $2}' push.log)" + printf 'reference=%s\n' "${repository#oci://}/icebergsst@$digest" >> "$GITHUB_OUTPUT" + printf 'path=dist/icebergsst-%s.tgz\n' "$chart_version" >> "$GITHUB_OUTPUT" + + - name: Sign the chart + env: + REFERENCE: ${{ steps.package.outputs.reference }} + run: cosign sign --yes "$REFERENCE" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: chart + path: ${{ steps.package.outputs.path }} + + release: + name: publish the GitHub release + needs: [verify, images, chart] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + # The ref the verify job proved is a release tag, never the raw input. + ref: ${{ needs.verify.outputs.ref }} + # Nothing here pushes with git; the registry and release steps use + # their own tokens. Leaving the checkout credential in .git/config + # would put a write-capable token inside every artifact built from + # this tree. + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: dist + merge-multiple: true + + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + # A source archive that matches the tag, signed alongside the images. `git + # archive` rather than the auto-generated tarball: GitHub's is produced on + # demand and has changed byte-for-byte across platform upgrades, which makes + # a checksum published against it a promise this project cannot keep. + - name: Build and sign the source archive + env: + VERSION: ${{ needs.verify.outputs.version }} + run: | + set -euo pipefail + archive="dist/icebergsst-$VERSION-source.tar.gz" + git archive --format=tar.gz --prefix="icebergsst-$VERSION/" -o "$archive" HEAD + cosign sign-blob --yes --bundle "$archive.cosign.bundle" "$archive" + (cd dist && sha256sum ./* > SHA256SUMS) + + - name: Extract this version's release notes + id: notes + env: + VERSION: ${{ needs.verify.outputs.version }} + run: | + set -euo pipefail + # The CHANGELOG entry *is* the release notes (docs/releases.md), so they + # are never written twice and can never disagree. + awk -v version="## [$VERSION]" ' + index($0, version) == 1 { capturing = 1; next } + capturing && /^## \[/ { exit } + capturing { print } + ' CHANGELOG.md > notes.md + if [ ! -s notes.md ]; then + echo "no release notes found for $VERSION" >&2 + exit 1 + fi + + - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 + with: + tag_name: ${{ needs.verify.outputs.tag }} + body_path: notes.md + files: dist/* + fail_on_unmatched_files: true diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..69cc35b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,58 @@ +# Changelog + +Every release has an entry here, and the entry **is** the release notes — GitHub's release body is +generated from it rather than written twice. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); the version policy, support window, and +what counts as a breaking change are in [`docs/releases.md`](./docs/releases.md). + +Sections appear in this order when they apply, because that is the order an operator needs them: +**Breaking**, **Migrations**, **Operator actions**, then Added / Changed / Fixed / Security. + +## [Unreleased] + +### Added + +- Release policy, a signed release pipeline, and a version-consistency invariant (#147). Every + published artifact — both role images, the Helm chart, and the source archive — is signed with + cosign's keyless flow and carries an SBOM and GitHub build provenance, so an operator can verify + what they are running without this project holding a signing key. + +## [0.1.0] — unreleased + +The first tagged release. Everything below is the state of the project at the point a version +number started meaning something; earlier changes are in the git history, where they were never +claimed to be supported. + +### Migrations + +`0001` through `0015`. On a fresh database they apply in seconds. `0014` seeds the four response +targets, and `0015` backfills `notification_delivery.kind`; both are reversible, and no downgrade +in this range drops data an operator has entered. + +### Operator actions + +- Set `ICEBERG_MASTER_KEY` and `ICEBERG_FINGERPRINT_PEPPER_REF` before first start. Losing the + master key makes every stored credential ref undecryptable — see + [`runbooks/key-rotation.md`](./docs/runbooks/key-rotation.md). +- Mount SMB/NFS shares read-only into the **engine** only, if using the file-share connector + (#145). + +### Added + +- Confluence, Jira, and SMB/NFS file-share connectors, on a versioned connector SDK with a + conformance kit. +- Two-phase scans with API-authoritative leases, durable checkpoints, and incremental scanning + with per-scope cursors (ADR 0009, ADR 0013). +- Detection with versioned rule packs, analyst-editable suppressions, and confidence thresholds. +- Findings with fingerprint-stable identity, triage, exposure clusters, remediation evidence, and + ownership with routing rules and response targets (ADR 0006, 0011, 0012; #146). +- Opt-in credential liveness validation that never persists plaintext (ADR 0010). +- Notification channels with a transactional outbox, and escalation for findings that miss their + response target. +- OIDC authentication with RBAC, a server-rendered console under a strict CSP, and an + administrative audit trail. +- Coverage manifests: what a scan actually read, and where it could not. +- docker-compose for development and a Helm chart for production. + +[Unreleased]: https://github.com/IcebergAI/IcebergSST/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/IcebergAI/IcebergSST/releases/tag/v0.1.0 diff --git a/Makefile b/Makefile index 1810c36..c5b9cd5 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,8 @@ COMPOSE ?= docker compose -f deploy/compose/docker-compose.yml --env-file .env # Engine replica count for `make scale`. N ?= 2 -.PHONY: help sync hooks lint format type test check docs-check images images-verify \ +.PHONY: help sync hooks lint format type test check docs-check version-check \ + images images-verify \ helm-verify helm-template up down destroy \ migrate seed logs ps scale init-env secrets engine-token @@ -42,6 +43,13 @@ check: lint type test ## Everything CI runs docs-check: ## Verify repository-local Markdown links uv run python scripts/check_docs.py +# A release is one number applied to both images, the chart's appVersion, and +# every package (docs/releases.md). `make check` asserts the same thing through +# tests/test_release_invariants.py, so a half-bumped version fails on the pull +# request rather than on the tag. +version-check: ## Assert every shipped component declares the same version + uv run python scripts/check_version.py + # ─── Images ─────────────────────────────────────────────────────────────────── images: ## Build both role images diff --git a/docs/README.md b/docs/README.md index d4053de..f495f51 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,9 +12,11 @@ Design specification and reference docs. Start with [`../ARCHITECTURE.md`](../AR - [`secret-validation.md`](./secret-validation.md) — opt-in credential liveness contracts and controls - [`security.md`](./security.md) — threat model and mitigations - [`deployment.md`](./deployment.md) — docker-compose (dev) + Helm (prod) +- [`releases.md`](./releases.md) — versioning, support window, compatibility, upgrade and rollback - [`runbooks/production-install.md`](./runbooks/production-install.md) — production-oriented installation and go-live checks - [`runbooks/backup-restore.md`](./runbooks/backup-restore.md) — isolated recovery rehearsal +- [`runbooks/release.md`](./runbooks/release.md) — cutting a release, and verifying a published one - [`backlog.md`](./backlog.md) — milestones, epics, and issues (mirrors GitHub) ## Decision records (ADRs) diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 0000000..549bbd6 --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,127 @@ +# Releases, versions, and supported upgrades + +What a version number promises, how long a release is supported, and which upgrades are +rehearsed. The mechanics of *producing* a release — signing, SBOMs, provenance — are in +[`runbooks/release.md`](./runbooks/release.md); this page is the policy those mechanics serve. + +## One version, everywhere + +A release is a single number applied to everything the project ships: both role images, the +Helm chart's `appVersion`, and every workspace package. There is deliberately no per-component +versioning, because every component in a deployment comes from the same build and an operator +answering "what am I running?" should not have to answer it five times. + +`make version-check` asserts that agreement, and `tests/test_release_invariants.py` runs it in +the ordinary suite — so a half-bumped version fails on the pull request that introduced it rather +than during a release. + +The chart's own `version` is separate and moves independently: a change to a template is a chart +release even when the application did not change. It is the one number that is allowed to differ. + +## What a version number means + +Semantic versioning, applied to the things an operator actually depends on: + +| Change | Bump | +|---|---| +| A REST or engine-protocol field is removed or changes meaning | **major** | +| A connector SDK major (`CONNECTOR_SDK_VERSION`) | **major** | +| A configuration variable is removed, or its default changes behaviour | **major** | +| A migration cannot be reversed | **major** (see below) | +| A new endpoint, field, connector, rule pack, or setting | **minor** | +| A fix that changes no documented behaviour | **patch** | + +Two things are explicitly *not* breaking changes, because treating them as such would make every +release a major one: adding a field to an API response or a notification payload (documented as +"ignore keys you do not recognise" in [`api.md`](./api.md) and +[`notifications.md`](./notifications.md)), and adding a rule to a rule pack. A new rule finds more +secrets; that is the product working, not a compatibility break. + +## Support window + +| Line | Supported until | +|---|---| +| Current minor | — | +| Previous minor | 90 days after the next minor is released | +| Anything older | Not supported | + +"Supported" means: security fixes are backported, and an upgrade from it to current is rehearsed +in CI (below). It does not mean an uptime commitment — [`SUPPORT.md`](../SUPPORT.md) is the +authority on that, and this is self-hosted software. + +Only tagged releases are supported. An arbitrary commit on `main` may be perfectly good, but +nothing rehearses an upgrade from it and no artifact is signed for it. + +## Compatibility + +**Database.** The API owns the schema and is the only role that migrates (ADR 0002). A migration +runs before the new API starts — the Helm chart does it in a pre-upgrade Job +([`deployment.md`](./deployment.md)) — so during a rolling upgrade the *old* API may briefly run +against the *new* schema. Every migration must therefore be additive with respect to the previous +release: add a column, backfill, and only drop it a release later. A migration that removes or +narrows something the previous release reads is a major-version change, and the release notes say +so. + +**Engines.** An engine speaks the versioned engine protocol and carries a connector SDK major. +A fleet may run one version behind the API — that is what makes a rolling engine upgrade possible +— and the API refuses a lease to an engine whose connector SDK major it does not support +([`connector-sdk.md`](./connector-sdk.md)). Two engine minors apart is not tested and not +supported. + +**Configuration.** Every variable the deployment interpolates is documented in `.env.example`, and +`tests/test_deploy_invariants.py` fails if one is missing. A removed variable is a major change; a +new one always has a default that preserves current behaviour, or the release is a major. + +## Upgrades and rollback + +**Supported path:** previous minor → current, or any patch within a minor. Skipping a minor is not +rehearsed; upgrade through it. + +**Rollback is bounded by migrations, not by images.** Rolling an image back is trivial and rolling +a schema back is not, so every migration ships with a `downgrade()` and +`apps/api/tests/test_migrations.py` proves it reverses. That makes rollback *mechanically* +possible; it does not make it lossless. A downgrade that drops a column drops the data in it, and +a release whose downgrade is destructive says so in its notes under **Rollback**. Where the data +matters, restore from a backup taken before the upgrade +([`runbooks/backup-restore.md`](./runbooks/backup-restore.md)) rather than downgrading. + +The order that makes rollback survivable: + +1. Back up the database. Verify the backup restores, in a scratch database, before continuing. +2. Upgrade the API (which migrates), then the engines. +3. Roll back in the reverse order: engines first, then the API, then — only if the release notes + say the schema change is not backward-compatible — the migration. + +## Release notes + +Every release has an entry in [`CHANGELOG.md`](../CHANGELOG.md), and the entry is the release +notes: GitHub's release body is generated from it rather than written twice. Each entry carries, +in this order, whichever of these apply: + +- **Breaking** — what changed, and what an operator must do about it. +- **Migrations** — which run, roughly how long they take on a large table, and whether the + downgrade is lossless. +- **Operator actions** — new configuration, a required engine upgrade, a rule-pack change that + will move finding counts. +- **Added / Changed / Fixed / Security** — the ordinary sections. + +An entry that lists no migrations is a claim, not an omission: `test_release_invariants.py` +checks that a release adding a migration file also documents one. + +## Security releases + +A confirmed vulnerability is fixed on the current minor and backported to the supported previous +minor. The release is cut immediately rather than waiting for a planned one — a fix sitting on +`main` behind other work is a fix nobody is running. + +Notification is by GitHub Security Advisory on this repository, which is also what populates the +ecosystem's vulnerability databases; the CHANGELOG entry carries the advisory identifier. The +private reporting path and the handling expectations are in [`SECURITY.md`](../SECURITY.md). + +## Verifying what you are running + +Every published artifact is signed, and the signature is verifiable without trusting this project +with a key: images and charts are signed with cosign's keyless flow, and the build provenance is a +GitHub attestation. The exact commands are in [`runbooks/release.md`](./runbooks/release.md) — +including the SBOM, which is the practical way to answer "does this release contain the library in +that advisory?" without rebuilding it. diff --git a/docs/runbooks/release.md b/docs/runbooks/release.md new file mode 100644 index 0000000..5a16044 --- /dev/null +++ b/docs/runbooks/release.md @@ -0,0 +1,142 @@ +# Runbook: cutting a release, and verifying one + +Two audiences. The first half is for whoever tags a release; the second is for an operator who +wants to know that what they are about to run is what this project published. The policy behind +both — what a version number promises, how long it is supported — is in +[`../releases.md`](../releases.md). + +## Cutting a release + +Everything below happens on `main`, in one pull request, before any tag exists. + +1. **Bump the version.** Six files, all to the same number: the root `pyproject.toml`, the four + workspace packages, and `appVersion` in `deploy/helm/icebergsst/Chart.yaml`. Bump the chart's + own `version` too if any template changed. + + ``` + make version-check # fails naming every file that disagrees + ``` + +2. **Write the CHANGELOG entry.** Rename `## [Unreleased]` to `## [X.Y.Z] — YYYY-MM-DD` and add a + fresh `Unreleased` above it. The entry *is* the release notes — the workflow extracts it and + nothing is written twice — so it is the place to say what an operator has to do, not a list of + commit subjects. + + A release that adds a migration must have a **Migrations** section. `make check` fails + otherwise (`tests/test_release_invariants.py`), because "does this upgrade touch my database?" + is the question an operator asks first and the one a changelog most often fails to answer. + +3. **Merge, then tag the merge commit.** + + ``` + git switch main && git pull + git tag -a v0.2.0 -m "v0.2.0" + git push origin v0.2.0 + ``` + + The tag is what triggers `release.yml`. Its first job re-checks the version agreement and the + CHANGELOG entry *before* anything is built, because once an artifact is signed and pushed it is + public and a bad release has to be superseded rather than withdrawn. + +4. **Watch the workflow.** It publishes, in order: both role images (signed by digest, with an + SBOM and build provenance attached), the Helm chart, and a GitHub release carrying the source + archive, its signature bundle, the SBOMs, and `SHA256SUMS`. + +5. **Verify the release yourself**, using the operator section below. A release nobody has + verified is a signature nobody has tested. + +### If the workflow fails halfway + +Do not delete and re-push the tag: a moved tag is exactly the mutable-pointer problem signing +exists to solve, and anything already published is still signed against the old commit. Fix +forward — cut the next patch version — unless *nothing* was published, in which case +`workflow_dispatch` re-runs the whole thing for the same tag. + +That dispatch input is validated as an **existing** `vX.Y.Z` tag before anything is checked out +from it, and every publishing job then uses the resolved ref rather than the string. A recovery +path that accepted a branch name would let a manual run publish untagged code over a release an +operator had already verified — the one thing `on: push: tags:` gives for free. + +## Verifying a release + +None of this requires trusting IcebergSST with a signing key. The images are signed with cosign's +**keyless** flow: the signature is bound to the GitHub workflow identity that produced it, and the +certificate that proves it was issued for the length of one job and then expired. + +**The image:** + +``` +cosign verify \ + --certificate-identity-regexp '^https://github\.com/IcebergAI/IcebergSST/\.github/workflows/release\.yml@' \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + ghcr.io/icebergai/icebergsst/api:0.2.0 +``` + +The two `--certificate-*` flags are the check. Without them `cosign verify` confirms that +*somebody* signed the image, which is not a useful statement about anything. + +**The build provenance** — that it was built from this repository, by that workflow, from that +commit: + +``` +gh attestation verify oci://ghcr.io/icebergai/icebergsst/api:0.2.0 --repo IcebergAI/IcebergSST +``` + +**The SBOM**, which is how to answer "does this release contain the library in that advisory?" +without rebuilding it: + +``` +cosign download attestation ghcr.io/icebergai/icebergsst/api:0.2.0 \ + | jq -r '.payload | @base64d | fromjson | .predicate' > api-sbom.spdx.json +``` + +**The chart:** + +``` +cosign verify \ + --certificate-identity-regexp '^https://github\.com/IcebergAI/IcebergSST/\.github/workflows/release\.yml@' \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + ghcr.io/icebergai/icebergsst/charts/icebergsst:0.2.0 +``` + +**The source archive**, using the bundle published beside it: + +``` +cosign verify-blob \ + --bundle icebergsst-0.2.0-source.tar.gz.cosign.bundle \ + --certificate-identity-regexp '^https://github\.com/IcebergAI/IcebergSST/\.github/workflows/release\.yml@' \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + icebergsst-0.2.0-source.tar.gz +``` + +`SHA256SUMS` is published for convenience and is **not** a substitute for any of the above: it is +published by the same party as the files it describes, so it detects a corrupted download and +nothing else. + +### Pin by digest in production + +A tag can be moved; a digest cannot. Once a release is verified, deploy the digest: + +```yaml +image: + api: + repository: ghcr.io/icebergai/icebergsst/api + tag: "" # unset, so the digest is what resolves + digest: sha256:… +``` + +That also makes the next upgrade a deliberate act rather than something a re-pulled tag does on +its own during an unrelated pod restart. + +## Upgrading + +The supported paths, the migration compatibility rule, and what rollback can and cannot recover +are in [`../releases.md`](../releases.md). The order that makes it survivable is short enough to +repeat here: + +1. Back up the database, and **restore the backup into a scratch database** to prove it works + ([`backup-restore.md`](./backup-restore.md)). An unverified backup is a plan, not a backup. +2. Read the release's **Migrations** and **Operator actions** sections. +3. Upgrade the API — which runs the migration in a pre-upgrade Job — then the engines. +4. To roll back: engines first, then the API. Only touch the schema if the notes say the change + is not backward-compatible, and prefer restoring the backup where the data matters. diff --git a/scripts/check_version.py b/scripts/check_version.py new file mode 100644 index 0000000..c8d506d --- /dev/null +++ b/scripts/check_version.py @@ -0,0 +1,80 @@ +"""Assert that everything IcebergSST ships carries the same version. + +A release is one number applied to both role images, the chart's ``appVersion``, +and every workspace package (``docs/releases.md``). Nothing structural keeps those +five files in step — they are edited by hand, in a bump nobody enjoys doing — so +this is the check that a half-bumped version fails on the pull request that +introduced it rather than during a release. + +The chart's own ``version`` is deliberately *not* checked: a template change is a +chart release even when the application did not change, and it is the one number +allowed to differ. + +Run it as ``make version-check``; ``tests/test_release_invariants.py`` runs the +same function in the ordinary suite, so CI enforces it without a separate job. +""" + +import re +import sys +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +#: Every ``pyproject.toml`` whose version is part of the release. +PACKAGES = ( + "pyproject.toml", + "apps/api/pyproject.toml", + "apps/engine/pyproject.toml", + "packages/core/pyproject.toml", + "packages/connectors/pyproject.toml", + "packages/detect/pyproject.toml", +) + +CHART = "deploy/helm/icebergsst/Chart.yaml" + +#: Read with a regex rather than a YAML parser: the chart is the one file here +#: that is not TOML, and adding a YAML dependency to a script that runs in every +#: test session to read one scalar is not a trade worth making. +_APP_VERSION = re.compile(r"^appVersion:\s*\"?([^\"\s]+)\"?\s*$", re.MULTILINE) + + +def declared_versions(root: Path = ROOT) -> dict[str, str]: + """Every declared version, by the file that declares it.""" + versions: dict[str, str] = {} + for relative in PACKAGES: + data = tomllib.loads((root / relative).read_text(encoding="utf-8")) + versions[relative] = str(data["project"]["version"]) + + chart = (root / CHART).read_text(encoding="utf-8") + match = _APP_VERSION.search(chart) + if match is None: + raise ValueError(f"{CHART} declares no appVersion") + versions[f"{CHART} (appVersion)"] = match.group(1) + return versions + + +def disagreements(root: Path = ROOT) -> dict[str, str]: + """The files that do not match the majority version. Empty when they agree.""" + versions = declared_versions(root) + # The root package is the reference rather than the most common value: a bump + # that changed four files out of six should name the two that were missed, + # not silently redefine the release as whatever the stragglers say. + expected = versions["pyproject.toml"] + return {name: value for name, value in versions.items() if value != expected} + + +def main() -> int: + versions = declared_versions() + mismatched = disagreements() + if not mismatched: + print(f"version {versions['pyproject.toml']} is consistent across {len(versions)} files") + return 0 + print(f"expected {versions['pyproject.toml']} (from pyproject.toml), but:", file=sys.stderr) + for name, value in sorted(mismatched.items()): + print(f" {name}: {value}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_release_invariants.py b/tests/test_release_invariants.py new file mode 100644 index 0000000..54e342e --- /dev/null +++ b/tests/test_release_invariants.py @@ -0,0 +1,196 @@ +"""What has to be true before a tag can be pushed (#147). + +The release workflow re-checks all of this before it builds anything, because a +signed artifact is public the moment it exists and a bad release has to be +superseded rather than withdrawn. But a check that only runs on a tag is one +nobody sees until the release they were trying to cut fails — so the same +properties are asserted here, on every pull request, where the fix is cheap. + +Lives in the root suite because it reads across the whole workspace: six +`pyproject.toml` files, the chart, the CHANGELOG, and the migrations directory. +""" + +import importlib.util +import re +import subprocess +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +CHANGELOG = ROOT / "CHANGELOG.md" +MIGRATIONS = ROOT / "apps/api/src/iceberg_api/migrations/versions" +WORKFLOW = ROOT / ".github/workflows/release.yml" + +#: `## [1.2.3] — 2026-08-16`, or `## [Unreleased]`. +_ENTRY = re.compile(r"^## \[([^\]]+)\]", re.MULTILINE) + + +def _load(name: str) -> ModuleType: + """Import a `scripts/` module by path. + + `scripts/` is not a package and should not become one: the files in it are + commands `make` runs, not library code, and making it importable would invite + application code to depend on them. Loading by path keeps that one-way. + """ + path = ROOT / "scripts" / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +check_version = _load("check_version") + + +def _sections(version: str) -> str: + """One version's CHANGELOG entry, from its heading to the next one.""" + text = CHANGELOG.read_text(encoding="utf-8") + start = text.index(f"## [{version}]") + remainder = text[start + 1 :] + end = remainder.find("\n## [") + return remainder if end == -1 else remainder[:end] + + +# ─── One version, everywhere ────────────────────────────────────────────────── + + +def test_every_shipped_component_declares_the_same_version() -> None: + """A release is one number applied to both images, the chart, and every + package. Nothing structural keeps six hand-edited files in step, so this is + what fails on the pull request that half-bumps them rather than during the + release.""" + assert check_version.disagreements() == {} + + +def test_the_version_check_would_notice_a_disagreement(tmp_path: Path) -> None: + """A guard nobody has seen fail is a guard nobody should trust. Builds a + scratch tree where one package lags, and asserts it is named.""" + for relative in check_version.PACKAGES: + target = tmp_path / relative + target.parent.mkdir(parents=True, exist_ok=True) + version = "0.1.0" if relative != "packages/detect/pyproject.toml" else "0.0.9" + target.write_text(f'[project]\nname = "x"\nversion = "{version}"\n') + chart = tmp_path / "deploy/helm/icebergsst/Chart.yaml" + chart.parent.mkdir(parents=True, exist_ok=True) + chart.write_text('appVersion: "0.1.0"\n') + + assert list(check_version.disagreements(tmp_path)) == ["packages/detect/pyproject.toml"] + + +def test_the_chart_version_is_allowed_to_differ_from_the_app_version() -> None: + """The one number that moves independently: a template change is a chart + release even when the application did not change. Asserted so a future + "tidy-up" does not fold it into the check above.""" + assert not any("(version)" in name for name in check_version.declared_versions()) + + +# ─── The CHANGELOG is the release notes ─────────────────────────────────────── + + +def test_the_changelog_has_an_unreleased_section() -> None: + """Where the next release's notes accumulate. Without it, notes get written + at tag time from commit subjects — which is how a migration goes + undocumented.""" + assert "## [Unreleased]" in CHANGELOG.read_text(encoding="utf-8") + + +def test_every_changelog_version_is_a_semantic_version() -> None: + text = CHANGELOG.read_text(encoding="utf-8") + versions = [name for name in _ENTRY.findall(text) if name != "Unreleased"] + + assert versions, "the changelog documents no releases" + for version in versions: + assert re.fullmatch(r"\d+\.\d+\.\d+", version), version + + +def test_a_release_that_adds_a_migration_documents_it() -> None: + """ "Does this upgrade touch my database?" is the first question an operator + asks and the one a changelog most often fails to answer. + + Checked against the *current* version's entry, since that is the one whose + scope this working tree can still change. + """ + version = check_version.declared_versions()["pyproject.toml"] + if f"## [{version}]" not in CHANGELOG.read_text(encoding="utf-8"): + pytest.skip(f"{version} is not yet a changelog entry; nothing to check") + migrations = sorted(path.name for path in MIGRATIONS.glob("0*.py")) + + assert migrations, "no migrations found; the glob is looking in the wrong place" + assert "### Migrations" in _sections(version) + + +def test_the_release_workflow_verifies_before_it_publishes() -> None: + """Ordering is the whole safety property: once an artifact is signed and + pushed it is public. Both build jobs must depend on the verify job.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + + assert "needs: verify" in workflow + assert "needs: [verify, images]" in workflow + + +def test_only_an_existing_release_tag_can_be_published() -> None: + """`on: push: tags:` gives this for free; `workflow_dispatch` does not. + + Without the check, a recovery run with `tag: main` would check out a branch, + pass the version and CHANGELOG checks — `main` declares the current version — + and publish untagged code over an image an operator had already verified. + """ + workflow = WORKFLOW.read_text(encoding="utf-8") + + assert "grep -Eq '^v[0-9]+\\.[0-9]+\\.[0-9]+$'" in workflow + assert 'git rev-parse --verify --quiet "refs/tags/$REF_NAME"' in workflow + + +def test_no_publishing_job_checks_out_the_dispatch_input() -> None: + """The structural half of the check above. The input is a string a human + typed; `needs.verify.outputs.ref` is a ref that has been proved to be a + release tag — so validating it in one job is only worth anything if no later + job can reach past that job for the original.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + + # Exactly one mention: the verify job's own env, where it is validated. + assert workflow.count("inputs.tag") == 1, "the raw dispatch input is used more than once" + assert "ref: ${{ needs.verify.outputs.ref }}" in workflow + assert "tag_name: ${{ needs.verify.outputs.tag }}" in workflow + + +def test_every_release_action_is_pinned_to_a_commit() -> None: + """A tag is a mutable pointer its owner can move — the same reason images are + signed by digest. The trailing comment is the human-readable half; only the + SHA decides what runs.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + used = re.findall(r"uses:\s*(\S+)", workflow) + + assert used, "no actions found; the pattern is looking at the wrong thing" + unpinned = [ref for ref in used if not re.search(r"@[0-9a-f]{40}$", ref)] + assert unpinned == [], f"not pinned to a commit: {unpinned}" + + +def test_the_release_workflow_signs_by_digest_never_by_tag() -> None: + """Verifying a signature on a tag proves something about a name rather than + about the bytes that run.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + + assert "REFERENCE: ${{ steps.build.outputs.image }}@${{ steps.build.outputs.digest }}" in ( + workflow + ) + assert 'cosign sign --yes "$REFERENCE"' in workflow + + +def test_the_version_check_script_runs_as_a_command() -> None: + """`make version-check` is what a release engineer runs; this is the same + entry point, so a refactor that breaks the CLI fails here.""" + result = subprocess.run( # noqa: S603 # this interpreter, one repository path + [sys.executable, str(ROOT / "scripts/check_version.py")], + capture_output=True, + text=True, + check=False, + cwd=ROOT, + ) + + assert result.returncode == 0, result.stderr + assert "consistent" in result.stdout