diff --git a/.changes/home-of-nonroot-from-var-lib.md b/.changes/home-of-nonroot-from-var-lib.md new file mode 100644 index 0000000..7444068 --- /dev/null +++ b/.changes/home-of-nonroot-from-var-lib.md @@ -0,0 +1,6 @@ +--- +type: changed +--- +Home of `nonroot` from `/var/lib/sqlite` to `/home/nonroot`, so `.sqlite_history` and whatever else a program writes under `HOME` stay out of the data directory + +- The working directory stays `/var/lib/sqlite`, where the databases are diff --git a/.changes/sbom-document-in-var-lib-db.md b/.changes/sbom-document-in-var-lib-db.md new file mode 100644 index 0000000..d7ca954 --- /dev/null +++ b/.changes/sbom-document-in-var-lib-db.md @@ -0,0 +1,6 @@ +--- +type: fixed +--- +SBOM document in `/var/lib/db/sbom` without the fields the syft schema requires + +- `src/Dockerfile` pins the version of the syft schema diff --git a/.changes/sbom-document-that-listed-every-file.md b/.changes/sbom-document-that-listed-every-file.md new file mode 100644 index 0000000..a88521f --- /dev/null +++ b/.changes/sbom-document-that-listed-every-file.md @@ -0,0 +1,6 @@ +--- +type: fixed +--- +SBOM document that listed every file of a directory as an entry of its own, which made it very large for a package with many files such as time zone data + +- A directory is one entry diff --git a/.gitattributes b/.gitattributes index 8a01f67..e2545e1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,6 +3,7 @@ *.md diff=markdown CHANGELOG.md merge=union +/.changes export-ignore /.github export-ignore .editorconfig export-ignore .gitattributes export-ignore diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index b7ef44f..225e010 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -19,11 +19,11 @@ workflows: '.github/workflows/pull-request.yaml': - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' - - 'docker/login-action@v4.7.0' + - 'docker/login-action@v4.8.0' '.github/workflows/release.yaml': - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' - - 'docker/login-action@v4.7.0' + - 'docker/login-action@v4.8.0' '.github/workflows/tests.yaml': - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' @@ -49,9 +49,9 @@ dependencies: commit: 'sha1-c3c9e263c25d99ce0380d002d59b67737d91b0dc' owner_id: 5429470 repo_id: 241092383 - 'docker/login-action@v4.7.0': - ref: 'v4.7.0' - commit: 'sha1-81c6f93d4bc991e510b342a00b419b3bb4efc5c6' + 'docker/login-action@v4.8.0': + ref: 'v4.8.0' + commit: 'sha1-8aa5831fb8d48d8f76594c7ddde61ae58019f550' owner_id: 5429470 repo_id: 287743349 'docker/setup-buildx-action@v4.4.1': diff --git a/.github/workflows/clean-manifests.yaml b/.github/workflows/clean-manifests.yaml index 4a1ee32..1c42b5f 100644 --- a/.github/workflows/clean-manifests.yaml +++ b/.github/workflows/clean-manifests.yaml @@ -23,6 +23,7 @@ jobs: if: ${{ github.repository == 'peakimages/sqlite' }} permissions: contents: read + actions: write # disable this workflow after a failed run environment: name: cleanup deployment: false @@ -41,11 +42,19 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CLEANUP_TOKEN }} run: scripts/clean-manifests.sh --delete 200 - - name: Upload the list of deleted manifests + - name: Upload the lists of manifests if: ${{ always() }} uses: actions/upload-artifact@v7.0.2 with: - name: deleted-manifests - path: scripts/deleted-manifests.tsv + name: manifests + path: | + scripts/manifests-to-delete.tsv + scripts/deleted-manifests.tsv if-no-files-found: ignore retention-days: 90 + + - name: Disable this workflow after a failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ github.token }} + run: gh workflow disable clean-manifests.yaml --repo "$GITHUB_REPOSITORY" diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index d488618..3ce4d3a 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -67,7 +67,7 @@ jobs: path: ${{ runner.temp }}/images - name: Log in to ghcr.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f50019a..df1ebda 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -88,14 +88,14 @@ jobs: path: ${{ runner.temp }}/images - name: Log in to ghcr.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to docker.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -164,14 +164,14 @@ jobs: echo "tags=$tags" | tee -a "$GITHUB_OUTPUT" - name: Log in to ghcr.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to docker.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 5322ecc..a1142c1 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -35,9 +35,9 @@ jobs: - name: Verify the action lockfile env: GH_TOKEN: ${{ github.token }} - # https://github.com/github/gh-actions-lock/releases/tag/v0.4.0 + # https://github.com/github/gh-actions-lock/releases/tag/v0.4.1 run: | - gh extension install github/gh-actions-lock --pin v0.4.0 + gh extension install github/gh-actions-lock --pin v0.4.1 gh actions-lock --verify test: diff --git a/RELEASE.md b/RELEASE.md index 80e9246..619e6d2 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -3,7 +3,7 @@ Every release is tagged `v-v`, such as `v3.54.1-v1.4.0-beta.1`. > [!IMPORTANT] -> A push that changes `release.json` or `src/upstream.json` publishes every release of it whose tag does not exist yet. Check what a push publishes with `scripts/plan-version.sh` first. +> A push that changes `release.json` or `src/upstream.json` releases the versions they set, unless they are released already. Run `scripts/plan-version.sh` first to see what a push releases. ## Rules @@ -17,6 +17,7 @@ Every release is tagged `v-v`, such as `v3.54.1-v | `v1.2.x` | backports and hotfixes, cut from the newest tag of v1.2 | stable `1.2.x` versions | - `main` takes fixes only. A new minor or major reaches it as a merge of `next` or a release branch +- Merge `next` or a release branch into `main` only right before you release it, on the same day. To freeze a minor for longer, publish an RC on `next` or cut `release/v1.4` - `next` and the release branches publish prereleases only, their stable versions are released on `main` - Name a branch after what it changes: `fix/`, `feat/`, and `fix/-v1.2.2` for a backport, after the version that releases it - Squash merge pull requests from contributors and pull requests with a single change, such as SQLite updates @@ -35,11 +36,12 @@ Every release is tagged `v-v`, such as `v3.54.1-v > [!NOTE] > `next` publishes prereleases only. For its stable version, merge `next` into `main` with a merge commit and create the release on `main`. -1. Describe the changes under `## [Unreleased]` in `CHANGELOG.md`, they become the release notes -2. Check what the branch releases: +1. Describe the changes in change files with `scripts/change.sh`, they become the release notes, see [Change files](#change-files) +2. Check what the branch releases and the entries it lists: ```sh scripts/plan-version.sh + scripts/change.sh --notes ``` 3. Select the stage and the version, the script commits them and asks to push: @@ -57,6 +59,27 @@ Every release is tagged `v-v`, such as `v3.54.1-v > scripts/undo-release.sh > ``` +## Change files + +Every change to the image has a file in `.changes/`, `AGENTS.md` describes the types and the wording: + +```text +--- +type: fixed +commit: 1a2b3c4 +--- +Missing description and license on ghcr.io + +- The release index carries the labels of the image as annotations +``` + +- `scripts/change.sh` writes a file, it asks for what its flags leave out, and `--notes` prints the entries of the next release +- `scripts/release.sh` preselects the version the files need and writes them into `CHANGELOG.md`, the release workflow reads its notes from there +- A prerelease writes the files under `## [Unreleased]`, keeps them and adds its version to the files that have none +- A stable release writes the files under `## [v1.4.0] - ` and removes them +- `scripts/undo-release.sh` brings the files back as they were +- A new SQLite version needs no file, the release workflow writes its notes + ## Backport a fix `main` is on 1.3, and 1.1 and 1.2 still get fixes. A fix reaches each of them through a patch branch and a pull request of its own: @@ -130,8 +153,11 @@ Every release is tagged `v-v`, such as `v3.54.1-v ## Pull requests -The pull request workflow builds every pull request and publishes the image of the line `latest` of one from a branch of this repository as `ghcr.io/peakimages/sqlite:pr-`, versioned `-pr.`: pull request 1234 into `next` is `1.4.0-beta.1-pr.1234` after `1.4.0-beta.1` and `1.5.0-alpha.0-pr.1234` before the first alpha of 1.5.0, into `main` `1.3.4-pr.1234`. PHP's `version_compare` sorts it after its base and before the next prerelease or release. +The pull request workflow builds every pull request and publishes the image of the line `latest` of a pull request from a branch of this repository as `ghcr.io/peakimages/sqlite:pr-`. Its version is `-pr.`: +- Into `next`: `1.4.0-beta.1-pr.1234` after `1.4.0-beta.1`, or `1.5.0-alpha.0-pr.1234` before the first alpha of 1.5.0 +- Into `main`: `1.3.4-pr.1234` +- PHP's `version_compare` sorts it after its base and before the next prerelease or release - Pull requests from forks and Dependabot are built, not published - A push to a pull request from `main`, `next`, a release branch or a patch branch that changes `release.json` or `src/upstream.json` is not built, the release workflow builds it - ghcr.io only, no git tag and no GitHub release, the clean manifests workflow keeps the image while the pull request is open diff --git a/scripts/change.sh b/scripts/change.sh new file mode 100755 index 0000000..7c1c1e9 --- /dev/null +++ b/scripts/change.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -euo pipefail + +CHANGE_DIRECTORY=.changes +TYPES='added,changed,deprecated,removed,fixed,security,maintenance' +SECTIONS='Breaking Changes,Security,Added,Changed,Deprecated,Removed,Fixed,Maintenance' + +BOLD=$'\033[1m' GREEN=$'\033[32m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' + +heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } +ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} +confirm() { + local answer + printf '\n %s [y/N] ' "$*" + read -r answer + [[ "$answer" =~ ^[Yy]$ ]] +} + +cd "$(dirname "$0")/.." +trap 'tput cnorm 2> /dev/null || true' EXIT + +main() { + read_arguments "$@" + if [ -n "$notes" ]; then + print_notes + return + fi + [ -n "$kind" ] && [ -n "$summary" ] || ask_questions + write_change +} + +read_arguments() { + kind='' summary='' commit='' breaking='' notes='' details=() + while [ $# -gt 0 ]; do + case "$1" in + --notes) notes=true ;; + --breaking) breaking=true ;; + --type | --summary | --detail | --commit) + [ $# -ge 2 ] || die "$1 has no value, expected one" + case "$1" in + --type) kind="$2" ;; + --summary) summary="$2" ;; + --detail) details+=("$2") ;; + --commit) commit="$2" ;; + esac + shift + ;; + *) die "'$1' is not an option, expected --type, --summary, --detail, --breaking, --commit or --notes" ;; + esac + shift + done +} + +ask_questions() { + local selected detail types + [ -t 0 ] || die "--type and --summary are required without a terminal" + if [ -z "$kind" ]; then + heading "Select the type" + IFS=, read -ra types <<< "$TYPES" + selected="$(menu 0 "${types[@]}")" + kind="${types[$selected]}" + fi + if [ -z "$summary" ]; then + heading "Summary, one sentence with the heading" + printf ' ' + read -r summary + fi + heading "Details, an empty line ends them" + while printf ' - ' && read -r detail && [ -n "$detail" ]; do + details+=("$detail") + done + if confirm "Breaking change?"; then + breaking=true + fi + printf '\n' +} + +write_change() { + local path detail + [[ ",$TYPES," == *",$kind,"* ]] || die "the type is '$kind', expected ${TYPES//,/, }" + [ -n "$summary" ] || die "the summary is empty, expected one sentence" + if [ -n "$commit" ]; then + git rev-parse --verify --quiet "$commit^{commit}" > /dev/null || die "$commit is not a commit of this repository" + commit="$(git rev-parse --short "$commit")" + fi + mkdir -p "$CHANGE_DIRECTORY" + path="$CHANGE_DIRECTORY/$(file_name).md" + { + printf -- '---\ntype: %s\n' "$kind" + [ -z "$breaking" ] || printf 'breaking: true\n' + [ -z "$commit" ] || printf 'commit: %s\n' "$commit" + printf -- '---\n%s\n' "$summary" + [ "${#details[@]}" = 0 ] || printf '\n' + for detail in ${details[@]+"${details[@]}"}; do + printf -- '- %s\n' "$detail" + done + } > "$path" + ok "$path describes the $BOLD$kind$RESET change" +} + +print_notes() { + local section entries sections + compgen -G "$CHANGE_DIRECTORY/*.md" > /dev/null || die "$CHANGE_DIRECTORY has no change files" + IFS=, read -ra sections <<< "$SECTIONS" + for section in "${sections[@]}"; do + entries="$(awk -v section="$section" ' + FNR == 1 { fences = 0; type = ""; breaking = 0 } + fences < 2 { + if ($0 == "---") fences++ + else if ($1 == "type:") type = $2 + else if ($0 == "breaking: true") breaking = 1 + next + } + NF && tolower(section) == (breaking ? "breaking changes" : type) { + prefix = /^- / ? " " : "- " + print prefix $0 + } + ' "$CHANGE_DIRECTORY"/*.md)" + [ -z "$entries" ] || printf '### %s\n\n%s\n\n' "$section" "$entries" + done +} + +file_name() { + local stem name number=2 + stem="$(printf '%s' "$summary" | LC_ALL=C tr -cs '[:alnum:]' '\n' | LC_ALL=C tr '[:upper:]' '[:lower:]' | awk 'NF && ++count <= 6' | paste -sd - -)" + name="$stem" + while [ -e "$CHANGE_DIRECTORY/$name.md" ]; do + name="$stem-$number" + number=$((number + 1)) + done + echo "$name" +} + +menu() { + local selected="$1" options key i + shift + options=("$@") + tput civis >&2 2> /dev/null || true + while true; do + for i in "${!options[@]}"; do + if [ "$i" = "$selected" ]; then + printf ' %s❯ %s%s\n' "$GREEN" "${options[$i]}" "$RESET" >&2 + else + printf ' %s %s%s\n' "$DIM" "${options[$i]}" "$RESET" >&2 + fi + done + IFS= read -rsn1 key + if [ "$key" = $'\x1b' ]; then + IFS= read -rsn2 key + if [ "$key" = '[A' ] && [ "$selected" -gt 0 ]; then + selected=$((selected - 1)) + elif [ "$key" = '[B' ] && [ "$selected" -lt $((${#options[@]} - 1)) ]; then + selected=$((selected + 1)) + fi + elif [ -z "$key" ]; then + break + fi + printf '\033[%dA\033[J' "${#options[@]}" >&2 + done + tput cnorm >&2 2> /dev/null || true + echo "$selected" +} + +main "$@" diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh index 964f394..7423473 100755 --- a/scripts/clean-manifests.sh +++ b/scripts/clean-manifests.sh @@ -4,10 +4,14 @@ set -euo pipefail OPTION="$*" LIMIT="${2:-}" IMAGE=peakimages/sqlite +VARIANTS="" KEEP_TAG_PATTERN='^(latest|preview|[0-9]+(\.[0-9]+){0,2}|[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?)$' KEEP_TAG_EXCEPTIONS="" +MIN_AGE_DAYS=1 DOCKERHUB_USERNAME="${DOCKERHUB_USERNAME:-}" DOCKERHUB_TOKEN="${DOCKERHUB_TOKEN:-}" +GHCR_TOKEN="" +MANIFEST_TYPES='application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' TO_DELETE_FILE=scripts/manifests-to-delete.tsv DELETED_FILE=scripts/deleted-manifests.tsv WORK_DIR="$(mktemp -d)" @@ -28,14 +32,17 @@ trap 'rm -rf "$WORK_DIR"' EXIT main() { [[ "$OPTION" =~ ^(--delete [1-9][0-9]*)?$ ]] || die "'$OPTION' is not an option, expected --delete with a count or none" + [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_TOKEN" ] || die "DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to read docker.io/$IMAGE" + GHCR_TOKEN="$(gh auth token)" || die "gh is not signed in with a token that has read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" heading "Reading ghcr.io/$IMAGE and docker.io/$IMAGE" read_ghcr_versions read_dockerhub_tags + read_dockerhub_manifests_deleted_on_ghcr + read_dockerhub_untagged_manifests select_kept_tags read_manifests select_kept_manifests heading "Selecting what to delete" - find_dockerhub_only_manifests write_list delete_manifests verify_kept_manifests @@ -48,134 +55,186 @@ read_ghcr_versions() { list_ghcr_versions "state=deleted&page=$page" done | awk -v window="$window" 'NR <= window' > "$WORK_DIR/ghcr-deleted.tsv" cut -f 1 "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/on-ghcr.txt" - ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged, and $(count < "$WORK_DIR/ghcr-deleted.tsv") of the versions it deleted last are checked on docker.io, twice as many as the run deletes" + ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged" } read_dockerhub_tags() { - local token page last="" - token="$(registry_token docker.io pull)" - while :; do - page="$(curl -sSf --retry 3 -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$IMAGE/tags/list?n=1000${last:+&last=$last}" | jq -r '.tags // [] | .[]')" \ - || die "could not list the tags of docker.io/$IMAGE" - [ -n "$page" ] || break - echo "$page" >> "$WORK_DIR/dockerhub-tags.txt" - last="$(tail -n 1 <<< "$page")" + local hub url page + hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME" + url="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags?page_size=100" + : > "$WORK_DIR/hub-tags.tsv" + : > "$WORK_DIR/hub-images.tsv" + : > "$WORK_DIR/dockerhub-images.tsv" + while [ -n "$url" ]; do + page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" || die "could not list the tags of docker.io/$IMAGE from the Docker Hub API" + jq -r '.results[] | [.name, (.digest // ""), (.media_type // ""), (.tag_last_pushed // .last_updated // ""), (.full_size // 0)] | @tsv' <<< "$page" >> "$WORK_DIR/hub-tags.tsv" + jq -r '.results[] | .name as $tag | .images[]? | select(.digest) | [$tag, .digest, ([.os, .architecture] | join("/")), (.last_pushed // ""), (.size // 0)] | @tsv' <<< "$page" >> "$WORK_DIR/hub-images.tsv" + url="$(jq -r '.next // empty' <<< "$page")" done - request_manifests docker.io "$WORK_DIR/dockerhub-tags.txt" -f --head -o /dev/null -w '%header{docker-content-digest}\t%header{content-type}\t{}\n' \ - > "$WORK_DIR/dockerhub.tsv" || die "could not read the digests of the tags of docker.io/$IMAGE" - ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub.tsv") tags on $(cut -f 1 "$WORK_DIR/dockerhub.tsv" | sort -u | count) manifests" + awk -F '\t' '$2 == "" || $3 == "" { print $1 }' "$WORK_DIR/hub-tags.tsv" > "$WORK_DIR/old-tags.txt" + request_manifests docker.io "$WORK_DIR/old-tags.txt" -f --head -o /dev/null -w '{}\t%header{docker-content-digest}\t%header{content-type}\n' > "$WORK_DIR/old-tags.tsv" \ + || die "could not read the digests of the old tags of docker.io/$IMAGE, the Docker Hub API lists them without one" + awk -F '\t' -v OFS='\t' -v images="$WORK_DIR/dockerhub-images.tsv" ' + FILENAME == ARGV[1] { old_digest[$1] = $2; old_type[$1] = $3; next } + FILENAME == ARGV[2] { if ($2 == "") $2 = old_digest[$1]; if ($3 == "") $3 = old_type[$1]; root[$1] = $2; tags[$2] = ($2 in tags) ? tags[$2] "," $1 : $1; type[$2] = $3; if ($4 > pushed[$2]) pushed[$2] = $4; next } + $2 == root[$1] { platform[$2] = $3; size[$2] = $5; next } + !((root[$1], $2) in pair) { pair[root[$1], $2]; print root[$1], $2 > images } + !($2 in tags) && !seen[$2]++ { print $2, "", "", $4, $5, $3 } + END { for (d in tags) print d, type[d], tags[d], pushed[d], (type[d] ~ /index|list/ ? 0 : size[d] + 0), platform[d] }' \ + "$WORK_DIR/old-tags.tsv" "$WORK_DIR/hub-tags.tsv" "$WORK_DIR/hub-images.tsv" > "$WORK_DIR/dockerhub-manifests.tsv" + awk -F '\t' -v OFS='\t' '$3 != "" { n = split($3, tags, ","); for (i = 1; i <= n; i++) print $1, $2, tags[i] }' "$WORK_DIR/dockerhub-manifests.tsv" > "$WORK_DIR/dockerhub.tsv" + grep -q $'\tlatest$' "$WORK_DIR/dockerhub.tsv" || die "docker.io/$IMAGE lists no latest tag, expected the Docker Hub API to return every tag" + awk -F '\t' '$1 !~ /^sha256:/ || $4 == "" { exit 1 }' "$WORK_DIR/dockerhub-manifests.tsv" || die "docker.io/$IMAGE lists manifests without a digest or push date, expected both from the Docker Hub API" + ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub.tsv") tags on $(awk -F '\t' '$3 != ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) manifests, which reference $(awk -F '\t' '$3 == ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) more" +} + +read_dockerhub_manifests_deleted_on_ghcr() { + awk -F '\t' 'FILENAME == ARGV[1] { listed[$1]; next } !($1 in listed) && !seen[$1]++ { print $1 }' "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/ghcr-deleted.tsv" > "$WORK_DIR/deleted-on-ghcr.txt" + request_manifests docker.io "$WORK_DIR/deleted-on-ghcr.txt" --head -o /dev/null -w '%{http_code}\t{}\t%header{content-type}\n' > "$WORK_DIR/deleted-on-ghcr.tsv" \ + || die "could not check which of the versions that ghcr.io deleted last docker.io still has" + awk '$1 != 200 && $1 != 404 { exit 1 }' "$WORK_DIR/deleted-on-ghcr.tsv" || die "docker.io answered neither 200 nor 404 for a version that ghcr.io deleted last" + awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { created[$1] = $4; next } $1 == 200 { print $2, $3, "", created[$2], "", "" }' "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/deleted-on-ghcr.tsv" >> "$WORK_DIR/dockerhub-manifests.tsv" + ok "docker.io still has $(awk '$1 == 200' "$WORK_DIR/deleted-on-ghcr.tsv" | count) of the $(count < "$WORK_DIR/ghcr-deleted.tsv") versions that ghcr.io deleted last and no tag references" +} + +# The manifests endpoint is undocumented, Docker Hub uses it for image management, so it can change without notice. It only adds the untagged manifests that no tag references +read_dockerhub_untagged_manifests() { + local hub base url page key + hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME" + base="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/manifests?page_size=100" + url="$base" + : > "$WORK_DIR/listing.tsv" + while [ -n "$url" ]; do + if ! page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" \ + || ! jq -r '.manifests[] | [.manifest_digest, .media_type, (.tags // [] | join(",")), .last_pushed, (if (.media_type // "" | test("index|list")) then .size else .total_size end), ([.os, .arch] | map(select(. != null and . != "")) | join("/"))] | @tsv' <<< "$page" >> "$WORK_DIR/listing.tsv"; then + warn "the undocumented manifests listing of the Docker Hub API failed, the untagged manifests that no tag references wait for a later run" + return + fi + key="$(jq -r 'select(.manifests | length > 0) | .last_evaluated_key // empty' <<< "$page")" + url="${key:+$base&last_evaluated_key=$(jq -rn --arg key "$key" '$key | @uri')}" + done + awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { known[$1]; next } !($1 in known) && $3 == "" { print $1, $2, "", $4, $5, $6 }' "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/listing.tsv" > "$WORK_DIR/untagged.tsv" + cat "$WORK_DIR/untagged.tsv" >> "$WORK_DIR/dockerhub-manifests.tsv" + ok "the undocumented manifests listing adds $(count < "$WORK_DIR/untagged.tsv") untagged manifests that no tag references" } select_kept_tags() { { cut -f 3 "$WORK_DIR/ghcr-active.tsv" | tr ',' '\n' && cut -f 3 "$WORK_DIR/dockerhub.tsv"; } | sed '/^$/d' | sort -u > "$WORK_DIR/tags.txt" git ls-remote --tags "https://github.com/$IMAGE.git" | sed -n 's|.*refs/tags/||p' | sed 's/\^{}$//; p; s/^v//' > "$WORK_DIR/keep.txt" \ || die "could not list the git tags of github.com/$IMAGE" - gh pr list --repo "$IMAGE" --state open --limit 1000 --json number --jq '.[] | "pr-\(.number)"' >> "$WORK_DIR/keep.txt" \ + gh pr list --repo "$IMAGE" --state open --limit 1000 --json number --jq '.[] | "pr-\(.number)"' > "$WORK_DIR/pull-requests.txt" \ || die "gh could not list the open pull requests of $IMAGE" - tr ' ' '\n' <<< "$KEEP_TAG_EXCEPTIONS" >> "$WORK_DIR/keep.txt" + awk -v variants="$VARIANTS" '{ print; n = split(variants, variant, ","); for (i = 1; i <= n; i++) print $0 "-" variant[i] }' "$WORK_DIR/pull-requests.txt" >> "$WORK_DIR/keep.txt" + tr ',' '\n' <<< "$KEEP_TAG_EXCEPTIONS" >> "$WORK_DIR/keep.txt" { grep -E "$KEEP_TAG_PATTERN" "$WORK_DIR/tags.txt" || true grep -xF -f "$WORK_DIR/keep.txt" "$WORK_DIR/tags.txt" || true } | sort -u > "$WORK_DIR/kept-tags.txt" grep -qx latest "$WORK_DIR/kept-tags.txt" || die "found no latest tag, expected it among the tags to keep" - ok "keeping $(count < "$WORK_DIR/kept-tags.txt") tags: every tag KEEP_TAG_PATTERN matches or KEEP_TAG_EXCEPTIONS lists, every git tag of github.com/$IMAGE and the tags of open pull requests" + ok "keeping $(count < "$WORK_DIR/kept-tags.txt") tags: the ones KEEP_TAG_PATTERN matches or KEEP_TAG_EXCEPTIONS lists, the git tags of github.com/$IMAGE and the tags of open pull requests" } read_manifests() { - local digest tag mkdir "$WORK_DIR/manifests" request_manifests ghcr.io "$WORK_DIR/on-ghcr.txt" -f -o "$WORK_DIR/manifests/{}" -w '{}\t%{size_download}\n' > "$WORK_DIR/bytes.tsv" \ || die "could not read the manifests of ghcr.io/$IMAGE" - awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && !seen[$1]++ { print $1, $2, $3 }' \ - "$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-roots.tsv" - awk -F '\t' '$2 ~ /oci\.image\.index/ { print $1 }' "$WORK_DIR/dockerhub-roots.tsv" > "$WORK_DIR/dockerhub-indexes.txt" - request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the indexes kept tags point at on docker.io" - while IFS=$'\t' read -r digest tag; do - curl -sSf --retry 3 "https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags/$tag" | jq -r --arg root "$digest" '.images[] | [$root, .digest] | @tsv' \ - || die "could not read the manifests $tag points at on docker.io" - done < <(awk -F '\t' -v OFS='\t' '$2 !~ /oci\.image\.index/ { print $1, $3 }' "$WORK_DIR/dockerhub-roots.tsv") > "$WORK_DIR/children.tsv" + awk -F '\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && $2 ~ /oci\.image\.index/ && !seen[$1]++ { print $1 }' \ + "$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-indexes.txt" + request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the kept indexes that only docker.io has" manifest_fields '.manifests[]? | [manifest, .digest, (.platform | [.os, .architecture, .variant] | map(values) | join("/")), .annotations["vnd.docker.reference.digest"]]' \ - >> "$WORK_DIR/children.tsv" + > "$WORK_DIR/children.tsv" manifest_fields '(.config // empty), .layers[]? | [manifest, .digest, .size]' > "$WORK_DIR/blobs.tsv" manifest_fields '[manifest, if .manifests then "index" elif any(.layers[]?; .mediaType == "application/vnd.in-toto+json") then "attestation" else "image" end]' \ > "$WORK_DIR/kinds.tsv" - ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests of ghcr.io/$IMAGE and the $(count < "$WORK_DIR/dockerhub-roots.tsv") that kept tags point at on docker.io alone" + ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests on ghcr.io and the $(count < "$WORK_DIR/dockerhub-indexes.txt") kept indexes that only docker.io has, each with a GET that counts as a pull" } select_kept_manifests() { local kinds - kept_manifests "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/kept-ghcr.io.txt" - kept_manifests "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/kept-docker.io.txt" + manifests_of_kept_tags "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/kept-ghcr.io.txt" + manifests_of_kept_tags "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/kept-docker.io.txt" sort -u "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kept-docker.io.txt" > "$WORK_DIR/kept.txt" awk 'FILENAME == ARGV[1] { ghcr[$1]; next } !($1 in ghcr) { exit 1 }' "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/kept-ghcr.io.txt" \ - || die "manifests the kept tags point at are missing from the versions gh listed on ghcr.io, expected every one of them" - kinds="$(awk -F '\t' 'FILENAME == ARGV[1] { kept[$1]; next } ($1 in kept) { count[$2]++ } - END { printf "%d indexes, %d images and %d attestations", count["index"], count["image"], count["attestation"] }' "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kinds.tsv")" - ok "keeping $(count < "$WORK_DIR/kept-ghcr.io.txt") manifests on ghcr.io, $kinds, and $(count < "$WORK_DIR/kept-docker.io.txt") on docker.io: the kept tags and every manifest they point at" -} - -find_dockerhub_only_manifests() { - awk -F '\t' 'FILENAME == ARGV[1] || FILENAME == ARGV[2] { skip[$1]; next } !($1 in skip) && !seen[$1]++ { print $1 }' \ - "$WORK_DIR/kept.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" "$WORK_DIR/ghcr-deleted.tsv" > "$WORK_DIR/candidates.txt" - request_manifests docker.io "$WORK_DIR/candidates.txt" --head -o /dev/null -w '%{http_code}\t{}\t%header{content-type}\n' \ - > "$WORK_DIR/dockerhub-only.tsv" || die "could not check which manifests docker.io has" - awk '$1 != 200 && $1 != 404 { exit 1 }' "$WORK_DIR/dockerhub-only.tsv" \ - || die "could not check docker.io for the manifests ghcr.io does not have, expected 200 or 404" - ok "docker.io still has $(awk '$1 == 200' "$WORK_DIR/dockerhub-only.tsv" | count) of the $(count < "$WORK_DIR/candidates.txt") unused manifests ghcr.io does not have" + || die "ghcr.io lists no version for some kept manifests, expected one for each" + kinds="$(awk -F '\t' 'function amount(count, one, many) { return (count + 0) " " (count == 1 ? one : many) } + FILENAME == ARGV[1] { kept[$1]; next } ($1 in kept) { count[$2]++ } + END { print amount(count["index"], "index", "indexes") ", " amount(count["image"], "image", "images") ", " amount(count["attestation"], "attestation", "attestations") }' "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kinds.tsv")" + ok "keeping $(count < "$WORK_DIR/kept-ghcr.io.txt") manifests on ghcr.io ($kinds) and $(count < "$WORK_DIR/kept-docker.io.txt") on docker.io" } write_list() { - awk -F '\t' -v OFS='\t' 'function platform_of(digest) { return (reference[digest] in platform) ? platform[reference[digest]] : platform[digest] } + local cutoff + cutoff="$(jq -nr --argjson days "$MIN_AGE_DAYS" 'now - $days * 86400 | strftime("%Y-%m-%dT%H:%M:%SZ")')" + ok "keeping the $(awk -F '\t' -v cutoff="$cutoff" '$4 >= cutoff && !seen[$1]++' "$WORK_DIR/ghcr-active.tsv" | count) versions on ghcr.io and the $(awk -F '\t' -v cutoff="$cutoff" '$4 >= cutoff' "$WORK_DIR/dockerhub-manifests.tsv" | count) manifests on docker.io pushed since $cutoff, a running build may still need them" + awk -F '\t' -v OFS='\t' -v cutoff="$cutoff" 'function platform_of(digest) { return (reference[digest] in platform) ? platform[reference[digest]] : platform[digest] } FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { kind[$1] = $2; next } FILENAME == ARGV[3] { size[$1] = $2; next } FILENAME == ARGV[4] { size[$1] += $3; next } FILENAME == ARGV[5] { platform[$2] = $3; reference[$2] = $4; next } - FILENAME == ARGV[6] { tags[$1] = tags[$1] (tags[$1] == "" ? "" : ",") $3; next } - FILENAME == ARGV[7] { version[$1] = $2; created[$1] = $4; next } - FILENAME == ARGV[8] { if ($1 == 200) print "no", "yes", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } - !($1 in kept) { print "yes", "undefined", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ - "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ - "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k1,1 -k6,6 > "$WORK_DIR/rows.tsv" + FILENAME == ARGV[6] { + on_docker_hub[$1] + if (!($1 in kept) && !($1 in kind) && $4 != "" && $4 < cutoff && !seen[$1]++) + print "no", "yes", ($2 ~ /index|list/ ? "index" : ($6 == "unknown/unknown" ? "attestation" : "image")), ($6 == "unknown/unknown" ? "" : $6), $3, $4, $5, "", $1 + next + } + !($1 in kept) && $4 < cutoff && !seen[$1]++ { print "yes", (($1 in on_docker_hub) ? "yes" : "no"), kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ + "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/ghcr-active.tsv" \ + | sort -t $'\t' -k3,3r -k6,6 > "$WORK_DIR/sorted.tsv" + awk -F '\t' 'function emit(digest, list, n, i) { printed[digest]; print line[digest]; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) if ((list[i] in line) && !(list[i] in printed) && --parents[list[i]] == 0) emit(list[i]) } + FILENAME == ARGV[1] || FILENAME == ARGV[2] { if (!(($1, $2) in pair)) { pair[$1, $2]; children[$1] = children[$1] " " $2 } next } + { order[++rows] = $9; line[$9] = $0; type[$9] = $3 } + END { + for (i = 1; i <= rows; i++) if (order[i] in children) { n = split(children[order[i]], list, " "); for (j = 1; j <= n; j++) if (list[j] in line) parents[list[j]]++ } + for (i = 1; i <= rows; i++) if (type[order[i]] == "index" && !(order[i] in children)) emit(order[i]) + for (i = 1; i <= rows; i++) if (type[order[i]] == "image" && !(order[i] in parents) && !(order[i] in printed)) emit(order[i]) + for (i = 1; i <= rows; i++) if (!(order[i] in printed)) emit(order[i]) + }' "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-images.tsv" "$WORK_DIR/sorted.tsv" > "$WORK_DIR/rows.tsv" write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" - ok "$(awk -F '\t' 'END { print $9 " to delete, " $3 ", they free " $7 }' "$TO_DELETE_FILE")" - ok "$(awk -F '\t' '$1 == "no"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" - info "$TO_DELETE_FILE lists every manifest to delete with its type, platform, tags, date and size, indexes first" + ok "$(awk -F '\t' 'END { print ($9 + 0 ? $9 " to delete (" $3 "), they free " $7 : "nothing to delete") }' "$TO_DELETE_FILE")" + info "$TO_DELETE_FILE lists every manifest to delete in the order it deletes them, with its type, platform, tags, date and size" } delete_manifests() { - local on_ghcr type tags version digest rest name deleted ghcr docker_hub status + local on_ghcr on_docker_hub type tags version digest rest name deleted ghcr docker_hub status gone=0 if [ -z "$LIMIT" ]; then - info "nothing is deleted, a run with --delete and a count deletes that many from the top of the list" + info "a dry run deletes nothing, --delete with a count deletes that many manifests from the top of the list" return fi - [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_TOKEN" ] || die "DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to delete manifests on docker.io" heading "Deleting" - write_table /dev/null "$DELETED_FILE" - while IFS='|' read -r on_ghcr type tags version digest rest; do + head -n 1 "$TO_DELETE_FILE" > "$DELETED_FILE" + : > "$WORK_DIR/deleted.tsv" + while IFS='|' read -r on_ghcr on_docker_hub type tags version digest rest; do + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || die "the list names '$digest', expected a sha256 digest" name="$digest${tags:+ ($tags)}" - deleted="" ghcr=no docker_hub=no + deleted="" ghcr=no docker_hub=no status="" if [ "$on_ghcr" = yes ]; then delete_ghcr_version "$version" "$type $name" || continue deleted=ghcr.io ghcr=yes fi - status="$(delete_dockerhub_manifest "$digest")" - if [[ "$status" == 2?? ]]; then - deleted="${deleted:+$deleted and }docker.io" docker_hub=yes + if [ "$on_docker_hub" = yes ]; then + status="$(delete_dockerhub_manifest "$digest")" + if [[ "$status" == 2?? ]]; then + deleted="${deleted:+$deleted and }docker.io" docker_hub=yes + elif [ "$status" = 404 ]; then + info "docker.io had already deleted the $type $BOLD$name$RESET together with its index" + gone=$((gone + 1)) + fi fi if [ -n "$deleted" ]; then ok "deleted the $type $BOLD$name$RESET from $deleted" - printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" >> "$WORK_DIR/deleted.tsv" - write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" + printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" | tee -a "$WORK_DIR/deleted.tsv" >> "$DELETED_FILE" fi case "$status" in - 2?? | 404) ;; + "" | 2?? | 404) ;; 403 | 5??) warn "docker.io keeps the $type $name for now, it answered $status, the next run tries again" ;; *) die "could not delete the $type $name from docker.io, it answered $status" ;; esac - done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $3, $5, $8, $9, rest }') - ok "$(awk -F '\t' 'END { print "deleted " $9 ", " $3 ", they free " $7 }' "$DELETED_FILE")" - info "$DELETED_FILE lists what this run deleted and where, ghcr.io restores it for 30 days, docker.io deletes for good" + done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $2, $3, $5, $8, $9, rest }') + write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" + ok "$(awk -F '\t' 'END { print ($9 + 0 ? "deleted " $9 " (" $3 "), they free " $7 : "deleted nothing") }' "$DELETED_FILE")" + [ "$gone" = 0 ] || info "docker.io had already deleted $gone of the listed manifests together with their index, it deletes the children of an index that nothing else references" + info "$DELETED_FILE lists what this run deleted and where, ghcr.io can restore them for 30 days, docker.io deletes them permanently" } verify_kept_manifests() { @@ -184,9 +243,9 @@ verify_kept_manifests() { for registry in ghcr.io docker.io; do missing="$(request_manifests "$registry" "$WORK_DIR/kept-$registry.txt" --head -o /dev/null -w '%{http_code}\t{}\n' | awk '$1 != 200 { print $2 }')" \ || die "could not check the kept manifests on $registry" - [ -z "$missing" ] || die "$(count <<< "$missing") manifests the kept tags point at are gone from $registry, expected all of them: $(head -n 3 <<< "$missing" | paste -sd ' ' -)" + [ -z "$missing" ] || die "$(count <<< "$missing") kept manifests are gone from $registry: $(head -n 3 <<< "$missing" | paste -sd ' ' -)" done - ok "every kept tag and every manifest it points at is still on ghcr.io and docker.io" + ok "every kept manifest is still on ghcr.io and docker.io" } list_ghcr_versions() { @@ -194,7 +253,7 @@ list_ghcr_versions() { shift gh api "$@" "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions?per_page=100&$query" \ --jq '.[] | [.name, (.id | tostring), (.metadata.container.tags | join(",")), .created_at] | @tsv' 2> /dev/null \ - || die "gh could not list the versions of ghcr.io/$IMAGE with $query, it needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" + || die "gh could not list the versions of ghcr.io/$IMAGE, its token needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" } request_manifests() { @@ -204,9 +263,7 @@ request_manifests() { split -l 1000 "$list" "$WORK_DIR/chunk-" for chunk in "$WORK_DIR"/chunk-*; do token="$(registry_token "$registry" pull)" - xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" \ - -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \ - "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1 + xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" -H "Accept: $MANIFEST_TYPES" "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1 rm "$chunk" done } @@ -215,11 +272,11 @@ manifest_fields() { find "$WORK_DIR/manifests" -type f -exec jq -r 'def manifest: input_filename | sub(".*/"; ""); '"$1"' | @tsv' {} + } -kept_manifests() { +manifests_of_kept_tags() { awk -F '\t' 'function keep(digest, list, n, i) { if (digest in kept) return; kept[digest]; print digest; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) keep(list[i]) } - FILENAME == ARGV[1] { children[$1] = children[$1] " " $2; next } - FILENAME == ARGV[2] { tag[$1]; next } - { n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/kept-tags.txt" "$1" + FILENAME == ARGV[1] || FILENAME == ARGV[2] { children[$1] = children[$1] " " $2; next } + FILENAME == ARGV[3] { tag[$1]; next } + { n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-images.tsv" "$WORK_DIR/kept-tags.txt" "$1" } write_table() { @@ -227,22 +284,26 @@ write_table() { function amount(count, one, many) { return (count + 0) " " (count == 1 ? one : many) } BEGIN { split("B kB MB GB TB", units, " "); print "ghcr", "docker_hub", "type", "platform", "tags", "created", "size", "ghcr_id", "digest" } FILENAME == ARGV[1] { removed[$9]; types[$3]++; manifests++; if ($7 != "") $7 = human($7); print; next } - FILENAME == ARGV[2] { before += $2; if (!($1 in removed)) after += $2; next } - { blob[$2] = $3; if (!($1 in removed)) used[$2] } + FILENAME == ARGV[2] { bytes[$1] = $2; before += $2; if (!($1 in removed)) after += $2; next } + FILENAME == ARGV[3] { if ($1 in bytes) on_hub[$1]; size = ($1 in bytes) ? bytes[$1] : $5; hub_before += size; if (!($1 in removed)) hub_after += size; next } + { blob[$2] = $3; if (!($1 in removed)) used[$2]; if ($1 in on_hub) { hub_blob[$2] = $3; if (!($1 in removed)) hub_used[$2] } } END { for (digest in blob) { before += blob[digest]; if (digest in used) after += blob[digest] } + for (digest in hub_blob) { hub_before += hub_blob[digest]; if (digest in hub_used) hub_after += hub_blob[digest] } print "total", "", amount(types["index"], "index", "indexes") ", " amount(types["image"], "image", "images") ", " amount(types["attestation"], "attestation", "attestations"), - "", "", "", human(before - after) ", " human(before) " → " human(after) " on ghcr.io", "", amount(manifests, "manifest", "manifests") - }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" > "$2" + "", "", "", human(before - after) " on ghcr.io (" human(before) " → " human(after) ") and ~" human(hub_before - hub_after) " on docker.io (~" human(hub_before) " → ~" human(hub_after) ")", + "", amount(manifests, "manifest", "manifests") + }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/blobs.tsv" > "$2" } delete_ghcr_version() { local error + [[ "$1" =~ ^[0-9]+$ ]] || die "the version id of the $2 is '$1', expected a number from the GitHub API" until error="$(gh api -X DELETE "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions/$1" 2>&1 > /dev/null)"; do case "$error" in *"rate limit"*) sleep 60 ;; *downloads*) - warn "ghcr.io and docker.io keep the $2, it has more than 5,000 downloads on ghcr.io" + warn "ghcr.io and docker.io keep the $2, ghcr.io deletes no version with more than 5,000 downloads" return 1 ;; *) die "could not delete the $2 from ghcr.io: $error" ;; @@ -251,22 +312,38 @@ delete_ghcr_version() { sleep 1 } +# docker.io answers 202 to the deletion of any digest, so a HEAD tells first whether the manifest is still there delete_dockerhub_manifest() { - local status - while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $(registry_token docker.io pull,push,delete)" \ - "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do + local token status + token="$(registry_token docker.io pull,push,delete)" + status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' --head -H "Authorization: Bearer $token" -H "Accept: $MANIFEST_TYPES" "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" + [ "$status" = 200 ] || { + echo "$status" + return + } + while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do sleep 60 + token="$(registry_token docker.io pull,push,delete)" done echo "$status" } registry_token() { case "$1" in - ghcr.io) curl -sSf --retry 3 "https://ghcr.io/token?scope=repository:$IMAGE:$2" ;; - docker.io) curl -sSf --retry 3 ${DOCKERHUB_USERNAME:+-u "$DOCKERHUB_USERNAME:$DOCKERHUB_TOKEN"} "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:$2" ;; + ghcr.io) basic_auth "x:$GHCR_TOKEN" | curl -sSf --retry 3 -K - "https://ghcr.io/token?scope=repository:$IMAGE:$2" ;; + docker.io) basic_auth "$DOCKERHUB_USERNAME:$DOCKERHUB_TOKEN" | curl -sSf --retry 3 -K - "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:$2" ;; esac | jq -r .token } +basic_auth() { + printf 'header = "Authorization: Basic %s"\n' "$(printf '%s' "$1" | jq -Rrj @base64)" +} + +hub_token() { + jq -n --arg identifier "$DOCKERHUB_USERNAME" --arg secret "$DOCKERHUB_TOKEN" '{identifier: $identifier, secret: $secret}' \ + | curl -sSf --retry 3 -X POST -H 'Content-Type: application/json' --data-binary @- https://hub.docker.com/v2/auth/token | jq -er .access_token +} + count() { awk 'END { print NR }'; } main diff --git a/scripts/lint.sh b/scripts/lint.sh index 0624685..b696e30 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -8,8 +8,8 @@ HADOLINT=hadolint/hadolint:v2.15.1 SHELLCHECK=koalaman/shellcheck:v0.11.0 # https://github.com/mvdan/sh/releases/tag/v3.14.1 SHFMT=mvdan/shfmt:v3.14.1 -# https://github.com/jdx/jactionlint/releases/tag/v2.4.0 -JACTIONLINT=ghcr.io/jdx/jactionlint:2.4.0 +# https://github.com/jdx/jactionlint/releases/tag/v2.5.0 +JACTIONLINT=ghcr.io/jdx/jactionlint:2.5.0 # https://github.com/editorconfig-checker/editorconfig-checker/releases/tag/v4.0.2 EDITORCONFIG_CHECKER=mstruebing/editorconfig-checker:4.0.2 diff --git a/scripts/plan-version.sh b/scripts/plan-version.sh index 1675a45..c5287dd 100755 --- a/scripts/plan-version.sh +++ b/scripts/plan-version.sh @@ -59,9 +59,13 @@ print_build_version() { } fetch_origin() { - local current behind + local current behind fetched stale [ "$(git rev-parse --is-shallow-repository)" = false ] || die "the clone is shallow, run git fetch --unshallow first" - git fetch --quiet --prune --tags origin || die "could not fetch origin" + fetched="$(git fetch --porcelain --no-progress --prune --tags origin)" || { + stale="$(awk '$1 == "!" { sub("^refs/tags/", "", $4); print $4 }' <<< "$fetched" | paste -sd ' ' -)" + [ -z "$stale" ] || die "the local tag(s) $stale differ from origin, check them with git ls-remote --tags origin, then run git fetch --tags --force origin" + die "could not fetch origin" + } current="$(git branch --show-current)" if [ -n "$current" ] && behind="$(git rev-list --count "HEAD..origin/$current" 2> /dev/null)"; then [ "$behind" = 0 ] || die "$current is $behind commit(s) behind origin/$current, pull first" @@ -126,21 +130,21 @@ check_version() { } select_releases() { - local name sqlite tag previous older moves listed added="" + local name upstream tag previous older moves listed added="" releases= [ "$publish" = true ] || return 0 - while read -r name sqlite; do - tag="v${sqlite:+$sqlite-v}$version" + while read -r name upstream; do + tag="v${upstream:+$upstream-v}$version" if grep -qxF "$tag" <<< "$tags"; then info "$tag is published" continue fi previous= while read -r older; do - ! newer "$sqlite" "$older" || previous="$older" + ! newer "$upstream" "$older" || previous="$older" done < <(with_version "$version" <<< "$tags" | sed -nE 's/^v(.*)-v.*/\1/p') added+="$tag"$'\n' - releases+="$(jq -nc --arg name "$name" --arg version "$sqlite" --arg image "$version" --arg tag "$tag" --arg previous "$previous" \ + releases+="$(jq -nc --arg name "$name" --arg version "$upstream" --arg image "$version" --arg tag "$tag" --arg previous "$previous" \ '{$name, $version, $image, $tag, $previous}')"$'\n' done < <(upstream_lines) moves="$(moved_tags "$tags" "$tags"$'\n'"$added")" @@ -259,7 +263,7 @@ moved_tags() { } moving_tags() { - local tags releases latest preview base image sqlite prefix seen=" " + local tags releases latest preview base image upstream prefix seen=" " tags="$(cat)" releases="$(sed -nE 's/^v((.*)-v)?([^-]*)$/\3 \2/p' <<< "$tags" | sort -k1,1rV -k2,2rV)" latest="$(awk 'NR == 1 && NF { print "v" ($2 == "" ? "" : $2 "-v") $1 }' <<< "$releases")" @@ -273,10 +277,10 @@ moving_tags() { done < <(tag_versions <<< "$tags" | sed -n 's/-.*//p' | sort -uV) [ -z "$latest" ] || echo "latest $latest" [ -z "$preview" ] || echo "preview $preview" - while read -r image sqlite; do - prefix="$sqlite" + while read -r image upstream; do + prefix="$upstream" while [ -n "$prefix" ] && [[ "$seen" != *" $prefix "* ]]; do - echo "$prefix v$sqlite-v$image" + echo "$prefix v$upstream-v$image" seen+="$prefix " [[ "$prefix" == *.* ]] || break prefix="${prefix%.*}" diff --git a/scripts/release.sh b/scripts/release.sh index 7eaa44f..0a3642f 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -3,9 +3,10 @@ set -euo pipefail OPTION="$*" VERSION_FILE=release.json +CHANGE_DIRECTORY=.changes +SECTIONS='Breaking Changes,Security,Added,Changed,Deprecated,Removed,Fixed,Maintenance' ARTIFACT=image UPSTREAM=sqlite -BREAKING_CHANGE='^([a-z]+(\([^)]*\))?!|BREAKING[ -]CHANGE): ' WORK_DIR="$(mktemp -d)" BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' @@ -38,13 +39,14 @@ main() { select_version heading "Releasing v$new_version" update_changelog + update_change_files update_version commit push } check_repository() { - local counts behind ahead entries + local counts behind ahead entries fetched stale heading "Checking the repository" git rev-parse --is-inside-work-tree > /dev/null 2>&1 || die "$PWD is not a git repository" branch="$(git symbolic-ref --quiet --short HEAD)" || die "HEAD is detached, expected a branch" @@ -55,14 +57,18 @@ check_repository() { [[ ! "$branch" =~ ^(next|release/) ]] || channel=preview [ -z "$(git status --porcelain)" ] || die "the working tree is not clean, commit or stash first" [ "$(git rev-parse --is-shallow-repository)" = false ] || die "the clone is shallow, run git fetch --unshallow first" - git fetch --quiet --prune --tags origin || die "could not fetch origin" + fetched="$(git fetch --porcelain --no-progress --prune --tags origin)" || { + stale="$(awk '$1 == "!" { sub("^refs/tags/", "", $4); print $4 }' <<< "$fetched" | paste -sd ' ' -)" + [ -z "$stale" ] || die "the local tag(s) $stale differ from origin, check them with git ls-remote --tags origin, then run git fetch --tags --force origin" + die "could not fetch origin" + } counts="$(git rev-list --left-right --count "origin/$branch...HEAD" 2> /dev/null)" || die "origin has no branch $branch, push it first" read -r behind ahead <<< "$counts" [ "$behind$ahead" = 00 ] || die "$branch is $ahead commit(s) ahead of and $behind behind origin/$branch, expected it in sync" ok "$branch is clean and in sync with origin/$branch" - entries="$(awk '/^## \[Unreleased\]$/ { inside = 1; next } /^## / { inside = 0 } inside && /^- / { count++ } END { print count + 0 }' CHANGELOG.md)" - [ "$entries" != 0 ] || die "CHANGELOG.md has no entries under ## [Unreleased], describe the changes first" - ok "CHANGELOG.md lists $BOLD$entries$RESET change(s) under [Unreleased], the notes of this release" + entries="$(git ls-files "$CHANGE_DIRECTORY" | wc -l | tr -d ' ')" + [ "$entries" != 0 ] || die "$CHANGE_DIRECTORY has no change files, describe the changes first with scripts/change.sh" + ok "$CHANGE_DIRECTORY lists $BOLD$entries$RESET change(s), the notes of this release" } read_versions() { @@ -77,19 +83,17 @@ read_versions() { } read_changes() { - local tag range hash subject - tag="$(git describe --tags --abbrev=0 --match "v${UPSTREAM:+*-v}$stable" 2> /dev/null)" || true - range="${tag:+$tag..}HEAD" + local file required="patch" - if [ -n "$(git log -E --grep="$BREAKING_CHANGE" --format=%h "$range")" ]; then + if grep -qx 'breaking: true' "$CHANGE_DIRECTORY"/*.md; then required=major [ "${stable%%.*}" != 0 ] || required=minor - elif git log --format=%s "$range" | awk '/^feat(\([^)]*\))?!?: / && !/^feat\(release\): bump / { found = 1 } END { exit !found }'; then + elif grep -qxE 'type: (added|changed|deprecated|removed)' "$CHANGE_DIRECTORY"/*.md; then required=minor fi - info "the commits since ${tag:-the first commit} need a $BOLD$required$RESET release" - git log -E --grep="$BREAKING_CHANGE" --format='%h %s' "$range" | while read -r hash subject; do - warn "$DIM$hash$RESET $subject is a breaking change" + info "the change files need a $BOLD$required$RESET release" + for file in $(grep -lx 'breaking: true' "$CHANGE_DIRECTORY"/*.md || true); do + warn "$file is a breaking change" done } @@ -148,19 +152,59 @@ select_version() { } update_changelog() { - local today - if [ "$channel" = preview ]; then - info "CHANGELOG.md keeps the entries under [Unreleased] until the stable release dates them" - return - fi - today="$(date +%Y-%m-%d)" - awk -v heading="## [v$new_version] - $today" ' - /^#/ && NR > 1 && previous != "" { print "" } - { print; previous = $0 } - /^## \[Unreleased\]$/ { print ""; print heading; previous = heading } + local heading='' + [ "$channel" = preview ] || heading="## [v$new_version] - $(date +%Y-%m-%d)" + print_notes > "$WORK_DIR/notes.md" + awk -v heading="$heading" -v notes="$WORK_DIR/notes.md" ' + /^## / { inside = 0 } + /^## \[Unreleased\]$/ { + inside = 1 + print + print "" + if (heading != "") { print heading; print "" } + while ((getline line < notes) > 0) print line + next + } + !inside ' CHANGELOG.md > "$WORK_DIR/CHANGELOG.md" cp "$WORK_DIR/CHANGELOG.md" CHANGELOG.md - ok "CHANGELOG.md dates the unreleased entries as ${BOLD}v$new_version$RESET, $today" + ok "CHANGELOG.md lists the changes under $BOLD${heading:-## [Unreleased]}$RESET" +} + +print_notes() { + local section entries sections + IFS=, read -ra sections <<< "$SECTIONS" + for section in "${sections[@]}"; do + entries="$(awk -v section="$section" ' + FNR == 1 { fences = 0; type = ""; breaking = 0 } + fences < 2 { + if ($0 == "---") fences++ + else if ($1 == "type:") type = $2 + else if ($0 == "breaking: true") breaking = 1 + next + } + NF && tolower(section) == (breaking ? "breaking changes" : type) { + prefix = /^- / ? " " : "- " + print prefix $0 + } + ' "$CHANGE_DIRECTORY"/*.md)" + [ -z "$entries" ] || printf '### %s\n\n%s\n\n' "$section" "$entries" + done +} + +update_change_files() { + local file + if [ "$channel" = stable ]; then + rm "$CHANGE_DIRECTORY"/*.md + ok "$CHANGE_DIRECTORY is empty, CHANGELOG.md lists the changes now" + return + fi + for file in "$CHANGE_DIRECTORY"/*.md; do + ! grep -q '^version: ' "$file" || continue + awk -v version="$new_version" '/^---$/ && ++fences == 2 { print "version: " version } { print }' "$file" > "$WORK_DIR/change.md" + cp "$WORK_DIR/change.md" "$file" + done + ok "$CHANGE_DIRECTORY names ${BOLD}v$new_version$RESET in the change files that had no version" } update_version() { @@ -170,9 +214,14 @@ update_version() { } commit() { - git add "$VERSION_FILE" CHANGELOG.md - git commit --quiet -S --message "feat(release): bump $ARTIFACT version to v$new_version" - ok "signed commit $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" + git add --all -- "$VERSION_FILE" CHANGELOG.md "$CHANGE_DIRECTORY" + local sign=-S kind=signed + if ! git commit-tree -S -m 'release signing check' 'HEAD^{tree}' > /dev/null 2>&1 < /dev/null; then + sign=--no-gpg-sign kind=unsigned + warn "git cannot sign a commit (no user.signingkey, or no agent that holds the key), the release commit is unsigned" + fi + git commit --quiet "$sign" --message "feat(release): bump $ARTIFACT version to v$new_version" + ok "$kind commit $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" } push() { diff --git a/scripts/test.sh b/scripts/test.sh index f20fdc8..cce256f 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -53,7 +53,7 @@ check_default_database() { local settings docker run -d --name "$CONTAINER-default" --health-interval 2s --health-start-period 1s "$IMAGE" > /dev/null wait_healthy "$CONTAINER-default" - settings="$(docker exec "$CONTAINER-default" sqlite3 database.sqlite 'PRAGMA page_size; PRAGMA auto_vacuum; PRAGMA journal_mode;' | tr '\n' ' ')" + settings="$(docker exec "$CONTAINER-default" sqlite3 -cmd '.timeout 5000' database.sqlite 'PRAGMA page_size; PRAGMA auto_vacuum; PRAGMA journal_mode;' | tr '\n' ' ')" [ "$settings" = '4096 0 wal ' ] || die "database.sqlite was created with '$settings', expected page size 4096, no auto-vacuum, WAL" ok "database.sqlite is created by default with page size 4096, no auto-vacuum and WAL" } @@ -171,7 +171,7 @@ recover_database() { backup_database() { local result - docker exec "$CONTAINER" sh -c "sqlite3 app.sqlite \"VACUUM INTO 'backup.sqlite'\" && cat backup.sqlite && rm backup.sqlite" > "$WORK_DIR/backup.sqlite" + docker exec "$CONTAINER" sh -c "sqlite3 -cmd '.timeout 5000' app.sqlite \"VACUUM INTO 'backup.sqlite'\" && cat backup.sqlite && rm backup.sqlite" > "$WORK_DIR/backup.sqlite" result="$(docker run --rm -i "$IMAGE" sh -c 'cat > backup.sqlite && sqlite3 backup.sqlite "PRAGMA integrity_check; SELECT count(*) FROM t;"' < "$WORK_DIR/backup.sqlite" | tr '\n' ' ')" [ "$result" = 'ok 3 ' ] || die "the backup is not intact and complete, got '$result'" ok "a backup with VACUUM INTO streams out through docker exec, intact and complete" @@ -180,7 +180,7 @@ backup_database() { restore_database() { local result sql app.sqlite 'DELETE FROM t;' - docker exec -i "$CONTAINER" sh -c 'cat > restore.sqlite && sqlite3 app.sqlite ".restore restore.sqlite" && rm restore.sqlite' < "$WORK_DIR/backup.sqlite" + docker exec -i "$CONTAINER" sh -c 'cat > restore.sqlite && sqlite3 -cmd ".timeout 5000" app.sqlite ".restore restore.sqlite" && rm restore.sqlite' < "$WORK_DIR/backup.sqlite" result="$(sql app.sqlite 'PRAGMA integrity_check; PRAGMA journal_mode; SELECT count(*) FROM t;' | tr '\n' ' ')" [ "$result" = 'ok wal 3 ' ] || die "the restored database is not intact, got '$result'" [ "$(sql app.sqlite "SELECT body FROM docs WHERE docs MATCH 'sqlite';")" = 'sqlite rocks' ] || die "FTS5 after restore" @@ -206,7 +206,7 @@ wait_healthy() { } sql() { - docker exec "$CONTAINER" sqlite3 "$@" + docker exec "$CONTAINER" sqlite3 -cmd '.timeout 5000' "$@" } cleanup() { diff --git a/scripts/undo-release.sh b/scripts/undo-release.sh index 843e107..8444a4e 100755 --- a/scripts/undo-release.sh +++ b/scripts/undo-release.sh @@ -3,6 +3,7 @@ set -euo pipefail OPTION="$*" VERSION_FILE=release.json +CHANGE_DIRECTORY=.changes BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' @@ -26,7 +27,7 @@ main() { [ -z "$OPTION" ] || die "'$OPTION' is not an option, expected none" check_repository heading "Undoing $(git log -1 --format='%h %s')" - info "resets the commit and restores $VERSION_FILE and CHANGELOG.md" + info "resets the commit and restores $VERSION_FILE, CHANGELOG.md and $CHANGE_DIRECTORY" confirm "Continue?" || die "aborted, nothing changed" undo_commit } @@ -35,18 +36,18 @@ check_repository() { local branch git rev-parse --is-inside-work-tree > /dev/null 2>&1 || die "$PWD is not a git repository" branch="$(git rev-parse --abbrev-ref HEAD)" - git diff --quiet HEAD~1 HEAD -- . ":(exclude)$VERSION_FILE" ":(exclude)CHANGELOG.md" || die "the last commit changes more than $VERSION_FILE and CHANGELOG.md, expected a release commit" + git diff --quiet HEAD~1 HEAD -- . ":(exclude)$VERSION_FILE" ":(exclude)CHANGELOG.md" ":(exclude)$CHANGE_DIRECTORY" || die "the last commit changes more than $VERSION_FILE, CHANGELOG.md and $CHANGE_DIRECTORY, expected a release commit" [ "$(versions_at HEAD~1)" != "$(versions_at HEAD)" ] || die "the last commit keeps the versions in $VERSION_FILE at $(versions_at HEAD), expected a release commit" - [ -z "$(git status --porcelain -- "$VERSION_FILE" CHANGELOG.md)" ] || die "$VERSION_FILE or CHANGELOG.md has uncommitted changes, commit or stash them first" + [ -z "$(git status --porcelain -- "$VERSION_FILE" CHANGELOG.md "$CHANGE_DIRECTORY")" ] || die "$VERSION_FILE, CHANGELOG.md or $CHANGE_DIRECTORY has uncommitted changes, commit or stash them first" git fetch --quiet origin || die "could not fetch origin" [ "$(git rev-list --count "origin/$branch..HEAD")" -ge 1 ] || die "the release commit is on origin/$branch already, nothing to undo locally" } undo_commit() { git reset --quiet --soft HEAD~1 - git restore --staged --worktree "$VERSION_FILE" CHANGELOG.md + git restore --staged --worktree "$VERSION_FILE" CHANGELOG.md "$CHANGE_DIRECTORY" ok "reset to $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" - ok "$VERSION_FILE is back to $BOLD$(versions_at HEAD)$RESET, the entries are back under [Unreleased]" + ok "$VERSION_FILE is back to $BOLD$(versions_at HEAD)$RESET, the change files are back" } versions_at() { diff --git a/src/Dockerfile b/src/Dockerfile index 8878e64..d6e2066 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -10,6 +10,8 @@ ARG BASE_IMAGE=alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf5196 ARG TINI_VERSION=0.19.0-r3 # https://pkgs.alpinelinux.org/package/v3.24/main/x86_64/busybox-static ARG BUSYBOX_VERSION=1.37.0-r31 +# https://github.com/anchore/syft/tree/v1.57.1/schema/json +ARG SYFT_SCHEMA_VERSION=16.1.13 # Set by the build workflow ARG IMAGE_VERSION @@ -23,6 +25,7 @@ ARG SQLITE_VERSION ARG IMAGE_VERSION ARG TINI_VERSION ARG BUSYBOX_VERSION +ARG SYFT_SCHEMA_VERSION # curl, jq and openssl are only used during the build stage, so they are not pinned to a specific version # hadolint ignore=DL3018 diff --git a/src/rootfs.sh b/src/rootfs.sh index 5a61243..808e307 100644 --- a/src/rootfs.sh +++ b/src/rootfs.sh @@ -30,12 +30,13 @@ install_binaries() { create_user() { install -d /rootfs/etc - echo "nonroot:x:65532:65532::/var/lib/sqlite:/sbin/nologin" > /rootfs/etc/passwd + echo "nonroot:x:65532:65532:nonroot:/home/nonroot:/sbin/nologin" > /rootfs/etc/passwd echo "nonroot:x:65532:" > /rootfs/etc/group } create_directories() { install -d -m 0777 -o 65532 -g 65532 /rootfs/var/lib/sqlite + install -d -m 0755 -o 65532 -g 65532 /rootfs/home/nonroot install -d -m 1777 /rootfs/tmp } diff --git a/src/sbom.sh b/src/sbom.sh index 318262f..3377870 100644 --- a/src/sbom.sh +++ b/src/sbom.sh @@ -4,7 +4,9 @@ set -eu USAGE="usage: sbom.sh IMAGE COMPONENT..., a component is apk|NAME|VENDOR:PRODUCT|PATH..., deb|NAME|LICENSE|VENDOR:PRODUCT|PATH... or generic|NAME|VERSION|LICENSE|URL|VENDOR:PRODUCT" IMAGE="${1:-}" REPOSITORY="${IMAGE%:*}" -OUTPUT_FILE="/rootfs/var/lib/db/sbom/${REPOSITORY##*/}-${IMAGE##*:}.syft.json" +ROOT_DIR="${ROOT_DIR:-/rootfs}" +SCHEMA_VERSION="${SYFT_SCHEMA_VERSION:-}" +OUTPUT_FILE="$ROOT_DIR/var/lib/db/sbom/${REPOSITORY##*/}-${IMAGE##*:}.syft.json" die() { echo "sbom: $*" >&2 @@ -13,6 +15,7 @@ die() { main() { [ "$#" -gt 1 ] || die "$USAGE" + [ -n "$SCHEMA_VERSION" ] || die "SYFT_SCHEMA_VERSION is not set, src/Dockerfile pins the version of the syft schema" shift write_document "$@" verify_files @@ -21,7 +24,7 @@ main() { write_document() { mkdir -p "${OUTPUT_FILE%/*}" - cat /lib/apk/db/installed /var/lib/dpkg/status 2> /dev/null | jq -Rsc --rawfile os_release /etc/os-release --arg usage "$USAGE" ' + cat /lib/apk/db/installed /var/lib/dpkg/status 2> /dev/null | jq -Rsc --rawfile os_release /etc/os-release --arg usage "$USAGE" --arg image "$IMAGE" --arg repository "$REPOSITORY" --arg tag "${IMAGE##*:}" --arg schema "$SCHEMA_VERSION" ' ([$os_release | split("\n")[] | capture("^(?[A-Z_]+)=\"?(?[^\"]*)")] | from_entries) as $os | [ split("\n\n")[] | [split("\n")[] | capture("^(?[^ :]+): ?(?.*)$")] | from_entries @@ -33,10 +36,14 @@ write_document() { def package($type; $name; $version; $license; $cpe; $purl): ($cpe | select(test("^[a-z0-9._-]+:[a-z0-9._-]+$")) // error("sbom: \($cpe) is not a CPE, expected VENDOR:PRODUCT")) as $cpe | { + id: "\($type)-\($name)-\($version)", name: $name, version: $version, type: $type, - licenses: [{ value: $license, spdxExpression: $license, type: "declared" }], + foundBy: "sbom.sh", + locations: [], + licenses: [{ value: $license, spdxExpression: $license, type: "declared", urls: [], locations: [] }], + language: "", cpes: [{ cpe: "cpe:2.3:a:\($cpe):\($version | gsub("(?[^A-Za-z0-9._-])"; "\\\(.c)")):*:*:*:*:*:*:*", source: "declared" }], purl: $purl }; @@ -62,13 +69,17 @@ write_document() { elif $type == "generic" and length == 6 then generic($name; .[2]; .[3]; .[4]; .[5]) else error("sbom: \(join("|")) is not a component, \($usage)") end ], - schema: { version: "16.1.10", url: "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-16.1.10.json" } + artifactRelationships: [], + source: { id: "", name: $repository, version: $tag, type: "image", metadata: { userInput: $image } }, + distro: {}, + descriptor: { name: "sbom.sh", version: "" }, + schema: { version: $schema, url: "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-\($schema).json" } }' --args "$@" > "$OUTPUT_FILE" } verify_files() { jq -r '.artifacts[].metadata.files[]?.path' "$OUTPUT_FILE" | while read -r path; do - [ -f "/rootfs$path" ] || die "$path is not a file in /rootfs" + [ -e "$ROOT_DIR$path" ] || die "$path is not in $ROOT_DIR" done }