diff --git a/.github/workflows/container-image-build.yml b/.github/workflows/container-image-build.yml
index 38595cc..bd9a5e4 100644
--- a/.github/workflows/container-image-build.yml
+++ b/.github/workflows/container-image-build.yml
@@ -87,6 +87,38 @@ jobs:
           tags: |
             type=raw,value=${{ inputs.base-tag }}
 
+      - name: Compute build cache references
+        id: cache
+        env:
+          IMAGE: ${{ inputs.image-name-prefix }}-${{ inputs.image-name-suffix }}
+          CACHE_SUFFIX: ${{ inputs.image-cache-suffix }}
+          CACHE_KEY: ${{ inputs.cache-key }}
+          BASE_TAG: ${{ inputs.base-tag }}
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          # Image tags only allow [A-Za-z0-9_.-], and branch names such as
+          # "feature/x" don't, so normalize the keys.
+          sanitize() { printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '-' | cut -c1-100; }
+          key="$(sanitize "$CACHE_KEY")"
+          base="$(sanitize "$BASE_TAG")"
+          default="$(sanitize "$DEFAULT_BRANCH")"
+          # Write the cache for the branch (or the given cache-key), so the next
+          # build of the same branch, and the push job of the same run, reuse it.
+          echo "cache-to=type=registry,ref=${IMAGE}:${CACHE_SUFFIX}-${key},mode=max" >> "$GITHUB_OUTPUT"
+          # Read the cache of this commit, then of the branch, then of the
+          # default branch as a fallback for new branches.
+          {
+            echo "cache-from<<EOF"
+            echo "type=registry,ref=${IMAGE}:${CACHE_SUFFIX}-${base}"
+            if [ "$key" != "$base" ]; then
+              echo "type=registry,ref=${IMAGE}:${CACHE_SUFFIX}-${key}"
+            fi
+            if [ -n "$default" ] && [ "$default" != "$key" ]; then
+              echo "type=registry,ref=${IMAGE}:${CACHE_SUFFIX}-${default}"
+            fi
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
       - name: Build Image
         uses: docker/build-push-action@v7
         with:
@@ -96,6 +128,6 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           platforms: ${{ inputs.platforms }}
           push: true
-          cache-from: type=registry,ref=${{ inputs.image-name-prefix }}-${{ inputs.image-name-suffix }}:${{ inputs.image-cache-suffix }}-${{ inputs.cache-key }}
-          cache-to: type=registry,ref=${{ inputs.image-name-prefix }}-${{ inputs.image-name-suffix }}:${{ inputs.image-cache-suffix }}-${{ inputs.base-tag }},mode=max
+          cache-from: ${{ steps.cache.outputs.cache-from }}
+          cache-to: ${{ steps.cache.outputs.cache-to }}
           build-args: ${{ inputs.build-args }}
diff --git a/.github/workflows/container-image-push.yml b/.github/workflows/container-image-push.yml
index 0b2022f..5fb2a5c 100644
--- a/.github/workflows/container-image-push.yml
+++ b/.github/workflows/container-image-push.yml
@@ -93,6 +93,38 @@ jobs:
             type=pep440,pattern={{major}}
             type=raw,value=latest,enable={{is_default_branch}}
 
+      - name: Compute build cache references
+        id: cache
+        env:
+          IMAGE: ${{ inputs.image-name-prefix }}-${{ inputs.image-name-suffix }}
+          CACHE_SUFFIX: ${{ inputs.image-cache-suffix }}
+          CACHE_KEY: ${{ inputs.cache-key }}
+          BASE_TAG: ${{ inputs.base-tag }}
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          # Image tags only allow [A-Za-z0-9_.-], and branch names such as
+          # "feature/x" don't, so normalize the keys.
+          sanitize() { printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '-' | cut -c1-100; }
+          key="$(sanitize "$CACHE_KEY")"
+          base="$(sanitize "$BASE_TAG")"
+          default="$(sanitize "$DEFAULT_BRANCH")"
+          # Write the cache for the branch (or the given cache-key), so the next
+          # build of the same branch, and the push job of the same run, reuse it.
+          echo "cache-to=type=registry,ref=${IMAGE}:${CACHE_SUFFIX}-${key},mode=max" >> "$GITHUB_OUTPUT"
+          # Read the cache of this commit, then of the branch, then of the
+          # default branch as a fallback for new branches.
+          {
+            echo "cache-from<<EOF"
+            echo "type=registry,ref=${IMAGE}:${CACHE_SUFFIX}-${base}"
+            if [ "$key" != "$base" ]; then
+              echo "type=registry,ref=${IMAGE}:${CACHE_SUFFIX}-${key}"
+            fi
+            if [ -n "$default" ] && [ "$default" != "$key" ]; then
+              echo "type=registry,ref=${IMAGE}:${CACHE_SUFFIX}-${default}"
+            fi
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
       - name: Build Image
         uses: docker/build-push-action@v7
         with:
@@ -102,6 +134,6 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           platforms: ${{ inputs.platforms }}
           push: true
-          cache-from: type=registry,ref=${{ inputs.image-name-prefix }}-${{ inputs.image-name-suffix }}:${{ inputs.image-cache-suffix }}-${{ inputs.cache-key }}
-          cache-to: type=registry,ref=${{ inputs.image-name-prefix }}-${{ inputs.image-name-suffix }}:${{ inputs.image-cache-suffix }}-${{ inputs.base-tag }},mode=max
+          cache-from: ${{ steps.cache.outputs.cache-from }}
+          cache-to: ${{ steps.cache.outputs.cache-to }}
           build-args: ${{ inputs.build-args }}
diff --git a/docs/sources/reference/shared-workflows.md b/docs/sources/reference/shared-workflows.md
index 8fd4a33..f1a9df1 100644
--- a/docs/sources/reference/shared-workflows.md
+++ b/docs/sources/reference/shared-workflows.md
@@ -417,7 +417,7 @@ Useful for validation in pull requests.
 | `dockerfile` | Dockerfile to build | No | `"Dockerfile"` |
 | `registry` | Container registry to log in to | No | `"ghcr.io"` |
 | `build-args` | Build arguments passed to `docker/build-push-action` | No | `""` |
-| `cache-key` | Cache key used for the build cache | No | `${{ github.ref_name }}` |
+| `cache-key` | Key of the build cache to write, usually the branch name. Characters not allowed in image tags are replaced with `-` | No | `${{ github.ref_name }}` |
 
 **Secrets:**
 
@@ -446,6 +446,9 @@ jobs:
 
 Builds a container image reusing the cache written by `container-image-build`, and pushes it to the registry.
 
+Both `container-image-build` and `container-image-push` read the build cache from the image tagged `<image-cache-suffix>-<base-tag>`, then `<image-cache-suffix>-<cache-key>`, then `<image-cache-suffix>-<default branch>`, and write it to `<image-cache-suffix>-<cache-key>`.
+A build of a new branch starts from the cache of the default branch, and `container-image-push` reuses the cache that `container-image-build` wrote earlier in the same run.
+
 **Inputs:**
 
 | Input | Description | Required | Default |
@@ -459,7 +462,7 @@ Builds a container image reusing the cache written by `container-image-build`, a
 | `dockerfile` | Dockerfile to build | No | `"Dockerfile"` |
 | `registry` | Container registry to log in to | No | `"ghcr.io"` |
 | `build-args` | Build arguments passed to `docker/build-push-action` | No | `""` |
-| `cache-key` | Cache key used for the build cache | No | `${{ github.ref_name }}` |
+| `cache-key` | Key of the build cache to write, usually the branch name. Characters not allowed in image tags are replaced with `-` | No | `${{ github.ref_name }}` |
 
 **Secrets:**
 
diff --git a/news/+container-image-cache.bugfix b/news/+container-image-cache.bugfix
new file mode 100644
index 0000000..8b785dd
--- /dev/null
+++ b/news/+container-image-cache.bugfix
@@ -0,0 +1 @@
+Fix the build cache of the `container-image-build` and `container-image-push` workflows, which was written under the base tag but read under the cache key, so it was never reused. The cache is now written per branch and falls back to the default branch. @sneridagh