From 8ababa23f67cfc0a5dd9f24b76c8f3b44fc5b001 Mon Sep 17 00:00:00 2001 From: Asen Lekov Date: Thu, 5 Nov 2026 13:57:44 +0200 Subject: [PATCH 1/3] feat(pruner): keep images and containers of other workloads DOCKER_PRUNE_KEEP_LABELS and DOCKER_PRUNE_KEEP_IMAGES protect images and stopped containers from the docker prune, in the normal and the low-space pass. Each full pass logs what it kept, with sizes. With either set, the pruner selects what to remove itself instead of calling docker system prune: that call has no reference filter, and with several label!= filters it spares only what carries all of them (containers on every image store, images on overlay2). Both empty keeps today's exact call. tests/pruner-docker.sh runs this on throwaway docker:dind daemons, one per image store. Closes #39 Co-Authored-By: Claude Opus 5.5 --- .env.example | 9 ++ README.md | 10 +++ pruner.sh | 140 ++++++++++++++++++++++++++++++- tests/pruner-docker.sh | 183 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 340 insertions(+), 2 deletions(-) create mode 100755 tests/pruner-docker.sh diff --git a/.env.example b/.env.example index 03d57fc..d65dfce 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,15 @@ RUNNER_GROUP=Default # PREFETCH_SOURCES=my-org/my-workflows@main # repos scanned for `uses:` # PREFETCH_EXTRA=actions/checkout@v4 # actions fetched as-is +# Images and containers of other workloads on this host, which the pruner must +# leave alone. Space-separated; both empty by default. +# Labels (key or key=value): images and stopped containers carrying any of them, +# and the image a kept container uses. +# DOCKER_PRUNE_KEEP_LABELS="com.example.keep com.acme.tier=build" +# Name patterns for images you cannot label, matched against what +# `docker image ls` prints. * also crosses /, a bare repository covers all tags. +# DOCKER_PRUNE_KEEP_IMAGES="variscite:* ghcr.io/acme/*" + # Other tunables (defaults shown) # RUNNER_IMAGE=ghcr.io/dodi-smart/github-runner # CACHE_BUDGET_BYTES=64424509440 # host-wide cache budget, split across slots diff --git a/README.md b/README.md index 080744b..e760a89 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,14 @@ path: - **Adaptive.** It runs daily, and early when free disk drops below `PRUNE_FREE_PCT`. Then `docker system prune` / `docker builder prune` clean old images and build cache, more aggressively when disk is tight. +- **Other workloads stay.** On a host that also runs other Docker work, list + what to keep in `DOCKER_PRUNE_KEEP_LABELS` and `DOCKER_PRUNE_KEEP_IMAGES`. + Each full pass logs every kept image with its size, so you can see why disk + space did not come back. Docker's age limit counts from when an image was + built, not last used, so an idle image without protection goes soon after + its container stops. Networks and build cache are not protected: a kept + stopped container on a custom network needs `docker compose up`, not + `docker start`. - **Hourly:** it fills the action archive cache. After each job, [`cleanup.sh`](cleanup.sh) does a quick tidy-up only: `/tmp`, @@ -327,6 +335,8 @@ comments. | `RUNNER_NAME_PREFIX`, `RUNNER_GROUP` | `runner`, `Default` | runner name and group | | `CACHE_BUDGET_BYTES`, `CACHE_BUDGET_PCT` | 60 GiB, `40` | pruner cache budget per host | | `PRUNE_FREE_PCT`, `PRUNE_INTERVAL_SECONDS` | `15`, `86400` | when the pruner runs | +| `DOCKER_PRUNE_KEEP_LABELS` | empty | space-separated `key` or `key=value`: images and stopped containers the pruner keeps, plus the image of a kept container | +| `DOCKER_PRUNE_KEEP_IMAGES` | empty | space-separated name patterns, e.g. `variscite:* ghcr.io/acme/*`, for images you cannot label | | `PREFETCH_SOURCES`, `PREFETCH_EXTRA` | — | action archive cache sources | | `DISABLE_AUTO_UPDATE` | empty | any non-empty value (even `false`) disables runner self-update | diff --git a/pruner.sh b/pruner.sh index 28c0f18..e962063 100755 --- a/pruner.sh +++ b/pruner.sh @@ -45,6 +45,12 @@ # PREFETCH_INTERVAL_SECONDS prefetch period (default 3600; 0 = off) # DOCKER_PRUNE_UNTIL / DOCKER_PRUNE_UNTIL_LOW / DOCKER_BUILDER_PRUNE_UNTIL # (24h / 2h when below PRUNE_FREE_PCT / 72h) +# DOCKER_PRUNE_KEEP_LABELS space-separated key or key=value entries: images and +# stopped containers carrying any of them stay, and so +# does the image of such a container (empty) +# DOCKER_PRUNE_KEEP_IMAGES space-separated patterns on the names `docker image +# ls` prints, * crossing /: "variscite:* ghcr.io/acme/*". +# A bare repository covers all of its tags (empty) # # Intentionally not using set -e: one failing step must not stop the others. @@ -63,6 +69,8 @@ GITHUB_API="https://api.github.com" DOCKER_PRUNE_UNTIL="${DOCKER_PRUNE_UNTIL:-24h}" DOCKER_PRUNE_UNTIL_LOW="${DOCKER_PRUNE_UNTIL_LOW:-2h}" DOCKER_BUILDER_PRUNE_UNTIL="${DOCKER_BUILDER_PRUNE_UNTIL:-72h}" +DOCKER_PRUNE_KEEP_LABELS="${DOCKER_PRUNE_KEEP_LABELS:-}" +DOCKER_PRUNE_KEEP_IMAGES="${DOCKER_PRUNE_KEEP_IMAGES:-}" log() { echo "[pruner] $(date -u +%FT%TZ) $*"; } @@ -484,6 +492,130 @@ prefetch_actions() { GH_TOKEN="" } +# Epoch of docker's `until` value $1: a duration back from now (24h, 90m, 1h30m) +# or a point in time. Prints nothing when it is neither. +until_epoch() { + local rest="$1" secs=0 unit + while [[ $rest =~ ^([0-9]+)([hms])(.*)$ ]]; do + case "${BASH_REMATCH[2]}" in + h) unit=3600 ;; + m) unit=60 ;; + s) unit=1 ;; + esac + secs=$((secs + BASH_REMATCH[1] * unit)) + rest="${BASH_REMATCH[3]}" + done + if [ -z "$rest" ]; then + echo $(($(date +%s) - secs)) + elif [[ $1 == [0-9][0-9][0-9][0-9]-* ]]; then # date alone would take "2d" for a time + date -d "$1" +%s 2>/dev/null + fi +} + +# Whether creation time $1 is before epoch $2. An image can have no creation +# time: like docker's own prune, leave it (date would read "" as this morning). +created_before() { + local t + [ -n "$1" ] && t=$(date -d "$1" +%s 2>/dev/null) && [ "$t" -lt "$2" ] +} + +# What `docker system prune -af --filter until=$1` does, minus the images and +# stopped containers DOCKER_PRUNE_KEEP_* protect. The prune itself cannot do it: +# it has no reference filter, and given several label!= filters it spares only +# what carries all of them (containers on every image store, images on overlay2; +# only the containerd store spares what carries any). So docker is asked about +# one label at a time and this function removes what is left, without -f: an +# image a job has just started on stays. Any failed listing ends the pass, as a +# missing list of what to keep must never read as nothing to keep. +prune_docker_keeping() { + local until="$1" cutoff out containers images label id name created image repo tag size pat ref + local removed_containers=0 removed_refs=0 + local -a labels patterns + local -A kept_containers=() kept_images=() in_use=() sizes=() refs=() created_at=() + read -ra labels <<<"$DOCKER_PRUNE_KEEP_LABELS" + read -ra patterns <<<"$DOCKER_PRUNE_KEEP_IMAGES" # read: a pattern never globs the cwd + cutoff=$(until_epoch "$until") + if [ -z "$cutoff" ]; then + log "cannot read until=${until}; skipping docker prune" + return 0 + fi + log "docker prune until=${until}, keeping labels [${labels[*]}] images [${patterns[*]}]" + + for label in "${labels[@]}"; do + if ! containers=$(docker ps -aq --no-trunc --filter "label=$label") || + ! images=$(docker image ls -q --no-trunc --filter "label=$label"); then + log "cannot list what label $label keeps; skipping docker prune" + return 0 + fi + for id in $containers; do kept_containers[$id]="$label"; done + for id in $images; do kept_images[$id]="label $label"; done + done + out=$(docker image ls --no-trunc --format '{{.ID}} {{.Repository}} {{.Tag}} {{.Size}}') || + { log "cannot list images; skipping docker prune"; return 0; } + while read -r id repo tag size; do + [ -n "$id" ] || continue + sizes[$id]="$size" + [ "$repo" != '' ] || continue + [ "$tag" = '' ] || refs[$id]+=" $repo:$tag" + for pat in "${patterns[@]}"; do + # shellcheck disable=SC2053 # the right side is a pattern on purpose + if [[ $repo:$tag == $pat || $repo == $pat ]]; then kept_images[$id]="pattern $pat"; fi + done + done <<<"$out" + + out=$(docker ps -aq --no-trunc --filter status=created --filter status=exited --filter status=dead) || + { log "cannot list stopped containers; skipping docker prune"; return 0; } + if [ -n "$out" ]; then + # A container gone by now fails the call but not the lines of the others. + # shellcheck disable=SC2086 # one word per container id + out=$(docker inspect --format '{{.Id}} {{.Name}} {{.Image}} {{.Created}}' $out 2>/dev/null) || true + fi + while read -r id name image created; do + [ -n "$id" ] || continue + if [ -n "${kept_containers[$id]:-}" ]; then + log "kept container ${name#/} (label ${kept_containers[$id]})" + kept_images[$image]="container ${name#/}" + elif created_before "$created" "$cutoff" && docker rm "$id" >/dev/null 2>&1; then + removed_containers=$((removed_containers + 1)) + fi + done <<<"$out" + + docker network prune -f --filter "until=${until}" 2>&1 | sed 's/^/[pruner] /' + + # After the containers: an image is free once its last container has gone. + out=$(docker ps -aq --no-trunc) || { log "cannot list containers; skipping image prune"; return 0; } + if [ -n "$out" ]; then + # shellcheck disable=SC2086 # one word per container id + out=$(docker inspect --format '{{.Image}}' $out 2>/dev/null) || true + [ -n "$out" ] || { log "cannot inspect containers; skipping image prune"; return 0; } + for id in $out; do in_use[$id]=1; done + fi + out=$(docker image inspect --format '{{.Id}} {{.Created}}' "${!sizes[@]}" 2>/dev/null) || true + while read -r id created; do + [ -n "$id" ] && created_at[$id]="$created" + done <<<"$out" + for id in "${!sizes[@]}"; do + ref="${refs[$id]:-$id}" + if [ -n "${kept_images[$id]:-}" ]; then + log "kept image ${ref# } ${sizes[$id]} (${kept_images[$id]})" + elif [ -z "${in_use[$id]:-}" ] && created_before "${created_at[$id]:-}" "$cutoff"; then + # Every tag by name: an id with several tags is refused without -f. + for ref in $ref; do + if out=$(docker image rm "$ref" 2>&1); then + removed_refs=$((removed_refs + 1)) + log "removed image $ref" + else + log "left image $ref: $out" + fi + done + fi + done + log "removed ${removed_containers} containers and ${removed_refs} image references" + + log "docker builder prune -af --filter until=${until}" + docker builder prune -af --filter "until=${until}" 2>&1 | sed 's/^/[pruner] /' +} + # $1 = low: free space is below PRUNE_FREE_PCT, so prune docker harder. prune_docker() { local until="$DOCKER_PRUNE_UNTIL" @@ -492,8 +624,12 @@ prune_docker() { log "docker daemon unreachable; skipping docker prune" return 0 fi - log "docker system prune -af --filter until=${until}" - docker system prune -af --filter "until=${until}" 2>&1 | sed 's/^/[pruner] /' + if [ -n "${DOCKER_PRUNE_KEEP_LABELS// /}${DOCKER_PRUNE_KEEP_IMAGES// /}" ]; then + prune_docker_keeping "$until" + else + log "docker system prune -af --filter until=${until}" + docker system prune -af --filter "until=${until}" 2>&1 | sed 's/^/[pruner] /' + fi log "docker builder prune -af --filter until=${DOCKER_BUILDER_PRUNE_UNTIL}" docker builder prune -af --filter "until=${DOCKER_BUILDER_PRUNE_UNTIL}" 2>&1 | sed 's/^/[pruner] /' } diff --git a/tests/pruner-docker.sh b/tests/pruner-docker.sh new file mode 100755 index 0000000..8331779 --- /dev/null +++ b/tests/pruner-docker.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# prune_docker against real daemons: keep labels, keep patterns and stopped +# containers carrying a keep label, in the normal and the low-space pass. +# +# Real daemons on purpose. The two image stores disagree on what several +# label!= filters mean, which a fake docker would never show, so every case runs +# on both. Each daemon is a throwaway docker:dind: this test prunes, so it talks +# to nothing else and stops if it finds itself connected to any other daemon. +# Needs GNU date and a docker that may run privileged containers: +# bash tests/pruner-docker.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +dind_image=docker:29-dind +outer_host="${DOCKER_HOST:-}" +outer() { DOCKER_HOST="$outer_host" docker "$@"; } +export BUILDX_CONFIG="$work/buildx" # no builder state left behind for a daemon that is gone + +# Age limits, which need no daemon. +now=$(date +%s) +age=$((now - $(until_epoch 1h30m))) +[ "$age" -ge 5400 ] && [ "$age" -le 5405 ] || fail "1h30m read as ${age}s" +[ "$(until_epoch 2026-11-03T21:00:00Z)" = "$(date -d 2026-11-03T21:00:00Z +%s)" ] || fail "a timestamp misread" +[ -z "$(until_epoch 2d)" ] || fail "2d is no docker duration and must not read as a time" +if created_before "" "$now"; then fail "an image without a creation time counted as old"; fi + +if ! outer info >/dev/null 2>&1; then + echo "SKIP: no docker daemon to start docker:dind on" + exit 0 +fi + +# start_daemon ...: a throwaway daemon, which every docker +# call of this test then talks to. +start_daemon() { + local name="$1" port i + shift + at_exit "outer rm -f $name >/dev/null 2>&1" + outer run -d --rm --privileged --name "$name" --hostname "$name" -p 127.0.0.1::2375 \ + -e DOCKER_TLS_CERTDIR= "$dind_image" --host=tcp://0.0.0.0:2375 --tls=false "$@" >/dev/null + port=$(outer port "$name" 2375/tcp) + export DOCKER_HOST="tcp://${port##*[ ]}" + for i in $(seq 1 60); do + docker info >/dev/null 2>&1 && break + [ "$i" -lt 60 ] || { echo "daemon $name did not start"; exit 1; } + sleep 1 + done + if [ "$(docker info --format '{{.Name}}')" != "$name" ]; then + echo "connected to a daemon other than $name, refusing to prune it" + exit 1 + fi +} + +# mkimage [