From 73f370429d6263e10cfb36091b30265da07aed5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= Date: Fri, 31 Jul 2026 16:32:37 +0200 Subject: [PATCH 1/3] ci/cleanup_caches: group all cleanup runs together A cleanup is not tied to the build that triggered it, it always operates on the whole cache state. Just cancel them and let the newest one finish. --- .github/workflows/cleanup_caches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cleanup_caches.yml b/.github/workflows/cleanup_caches.yml index 1b324ddf5a4dc..3d661e360131e 100644 --- a/.github/workflows/cleanup_caches.yml +++ b/.github/workflows/cleanup_caches.yml @@ -6,7 +6,7 @@ on: types: [completed] concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }} cancel-in-progress: true permissions: From 784f101692442110f0ee09627d2be56d03102301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= Date: Fri, 31 Jul 2026 16:50:07 +0200 Subject: [PATCH 2/3] ci/cleanup_caches: rewrite the selection as a single jq program The nested jq invocations were hard to extend and left no trace of what they decided. Delete by id rather than by key, since a key is only unique per ref, and log each deletion. No change in which caches are selected. --- .github/workflows/cleanup_caches.yml | 57 ++++++++++++++++------------ 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/.github/workflows/cleanup_caches.yml b/.github/workflows/cleanup_caches.yml index 3d661e360131e..a74abfb05384d 100644 --- a/.github/workflows/cleanup_caches.yml +++ b/.github/workflows/cleanup_caches.yml @@ -17,31 +17,38 @@ jobs: runs-on: ubuntu-latest steps: - run: | - gh cache list -L 100 --json id,key,ref -S last_accessed_at -O desc --jq ' - map(select(.key | startswith("x86_64-w64-mingw32-") or - startswith("i686-w64-mingw32-") or - startswith("aarch64-pc-windows-msvc-") or - startswith("x86_64-pc-windows-msvc-"))) | - group_by(.ref) | - map({ - ref: .[0].ref, - caches: map({ - key: .key, - prefix: (.key | capture("^(?[\\w_-]+-)\\d+$").prefix) - }) | group_by(.prefix) | map({keys: map(.key)}) - }) | - .[] - ' | - while read -r group; do - pr=$(echo "$group" | jq -r '.ref | capture("refs/pull/(?[0-9]+)/merge").num') - if [[ -n "$pr" ]] && [ "$(gh pr view $pr --json state --jq '.state')" != "OPEN" ]; then - keys=$(echo "$group" | jq -c '.caches | map(.keys) | .[]') - else - keys=$(echo "$group" | jq -c '.caches | map(.keys[1:]) | .[]') - fi - for key in $(echo "$keys" | jq -r '.[]'); do - gh cache delete "$key" - done + set -o pipefail + gh cache list -L 1000 --json id,key,ref -S last_accessed_at -O desc > caches.json + + # Resolve the state of every pull request that owns a cache. + for pr in $(jq -r '.[].ref | select(test("^refs/pull/[0-9]+/merge$")) | + split("/")[2]' caches.json | sort -u); do + jq -n --arg pr "$pr" --arg state "$(gh pr view $pr --json state --jq .state)" \ + '{($pr): $state}' + done | jq -s 'add // {}' > prs.json + + jq -r --slurpfile prs prs.json ' + $prs[0] as $state | + + [ .[] | select(.key | startswith("x86_64-w64-mingw32-") or + startswith("i686-w64-mingw32-") or + startswith("aarch64-pc-windows-msvc-") or + startswith("x86_64-pc-windows-msvc-")) + | . + { group: (.key | sub("-[0-9]+$"; "")), + pr: (.ref | if test("^refs/pull/[0-9]+/merge$") + then split("/")[2] else null end) } ] as $all + + # Caches of merged or closed pull requests. + | ( $all | map(select(.pr != null and $state[.pr] != "OPEN")) ) as $dead + + # Of the rest only the newest cache of each kind per ref is used. + | ( ($all - $dead) | group_by([.ref, .group]) | map(.[1:]) | add // [] ) as $superseded + + | ($dead + $superseded)[] | "\(.id)\t\(.ref)\t\(.key)" + ' caches.json | + while IFS=$'\t' read -r id ref key; do + echo "deleting $ref $key" + gh cache delete "$id" done env: GH_REPO: ${{ github.repository }} From 24a1c004f0074e543907b7f09d6f4df736aa9e5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= Date: Fri, 31 Jul 2026 16:51:35 +0200 Subject: [PATCH 3/3] ci/cleanup_caches: prune setup-msys2 package caches setup-msys2 saves the pacman package cache under a key ending in a hash of its contents. So with each update it saves more caches, what's more important it saves it per each PR ref. We don't want that. Keep only the msys2 cache on main branch and only the latest one per each configuration. If PR saves new one, it will be dropped, we prefer master branch case that can be used by all PR, rather then per PR cache of this. The few packages that will be refetched are ok. This change is an atempt to keep our cache under 10GB limit, to avoid evicting more important ccache caches. --- .github/workflows/cleanup_caches.yml | 31 +++++++++++++++++++--------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cleanup_caches.yml b/.github/workflows/cleanup_caches.yml index a74abfb05384d..48b0985f13abb 100644 --- a/.github/workflows/cleanup_caches.yml +++ b/.github/workflows/cleanup_caches.yml @@ -27,19 +27,29 @@ jobs: '{($pr): $state}' done | jq -s 'add // {}' > prs.json - jq -r --slurpfile prs prs.json ' + jq -r --slurpfile prs prs.json --arg default "$DEFAULT_REF" ' $prs[0] as $state | - [ .[] | select(.key | startswith("x86_64-w64-mingw32-") or - startswith("i686-w64-mingw32-") or - startswith("aarch64-pc-windows-msvc-") or - startswith("x86_64-pc-windows-msvc-")) - | . + { group: (.key | sub("-[0-9]+$"; "")), - pr: (.ref | if test("^refs/pull/[0-9]+/merge$") - then split("/")[2] else null end) } ] as $all + # Group caches by kind, dropping the timestamp or content hash + # that every save appends to the key. + def family: if startswith("msys2-") then "msys2" + elif test("-[0-9]+$") then "ccache" + else "other" end; + def group: if family == "msys2" then sub("-(files|state):.*$"; "") + else sub("-[0-9]+$"; "") end; - # Caches of merged or closed pull requests. - | ( $all | map(select(.pr != null and $state[.pr] != "OPEN")) ) as $dead + [ .[] | . + { family: (.key | family), + group: (.key | group), + pr: (.ref | if test("^refs/pull/[0-9]+/merge$") + then split("/")[2] else null end) } ] + | map(select(.family != "other")) as $all + + # Caches of merged or closed pull requests, and msys2 caches + # outside the default branch, which every ref restores from. + | ( $all | map(select( + (.pr != null and $state[.pr] != "OPEN") or + (.family == "msys2" and .ref != $default) + )) ) as $dead # Of the rest only the newest cache of each kind per ref is used. | ( ($all - $dead) | group_by([.ref, .group]) | map(.[1:]) | add // [] ) as $superseded @@ -53,3 +63,4 @@ jobs: env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEFAULT_REF: refs/heads/${{ github.event.repository.default_branch }}