From 7865317a800b251e236c1fe425b18cd2cc591d0f Mon Sep 17 00:00:00 2001 From: Ken VanDine Date: Tue, 9 Jun 2026 16:53:21 -0400 Subject: [PATCH 1/7] ci: fail release if any expected artifacts are missing The previous check only verified that at least one artifact existed. This meant a release could be published with missing CUDA builds (as happened with b9549, which was missing all Windows CUDA artifacts and ubuntu-cuda-sm_90-x64). The updated check explicitly enumerates every expected artifact: - All 7 CUDA SM variants (sm_75/80/86/89/90/100/120) for each of: ubuntu x64, ubuntu arm64, windows x64 - ubuntu ROCm, ubuntu OpenVINO, Windows ROCm, Windows CPU If any are absent the step fails before the release is created, so a partially-populated release can never be tagged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/release.yml | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6492e41db58d..16ff47a1010a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -993,13 +993,39 @@ jobs: fi - name: Check release artifacts + env: + TAG: ${{ steps.tag.outputs.name }} run: | - files=$(find ./release -maxdepth 1 \( -name '*.zip' -o -name '*.tar.gz' -o -name '*.tar.xz' -o -name '*.7z' \) 2>/dev/null | wc -l) - if [ "$files" -eq 0 ]; then - echo "No release artifacts found in ./release — aborting before creating a release." + missing=0 + + check_glob() { + local pattern="$1" + if ! ls $pattern >/dev/null 2>&1; then + echo "::error::Missing artifact: $pattern" + missing=$((missing + 1)) + fi + } + + # Verify all CUDA SM variants are present for each platform + for sm in sm_75 sm_80 sm_86 sm_89 sm_90 sm_100 sm_120; do + check_glob "release/llama-${TAG}-ubuntu-cuda-${sm}-x64.tar.xz" + check_glob "release/llama-${TAG}-ubuntu-cuda-${sm}-arm64.tar.xz" + check_glob "release/llama-${TAG}-windows-cuda-${sm}-x64.7z" + done + + # Verify non-CUDA platform artifacts are present + check_glob "release/llama-${TAG}-bin-ubuntu-rocm-*-x64.tar.gz" + check_glob "release/llama-${TAG}-bin-ubuntu-openvino-*-x64.tar.gz" + check_glob "release/llama-${TAG}-bin-win-rocm-*-x64.zip" + check_glob "release/llama-${TAG}-bin-win-cpu-x64.zip" + + if [ "$missing" -gt 0 ]; then + echo "::error::$missing artifact(s) missing — aborting before creating a release." exit 1 fi - echo "Found $files artifact(s) ready to upload." + + files=$(find ./release -maxdepth 1 \( -name '*.zip' -o -name '*.tar.gz' -o -name '*.tar.xz' -o -name '*.7z' \) 2>/dev/null | wc -l) + echo "All expected artifacts present. Found $files artifact(s) ready to upload." # Get the release for this tag, creating it if it does not exist yet. # This is idempotent: if a previous run created the release but failed From 73fe29e1601fc0eeb4c9ed3c43105e4ca2e528b6 Mon Sep 17 00:00:00 2001 From: Ken VanDine Date: Wed, 1 Jul 2026 20:21:21 -0400 Subject: [PATCH 2/7] ci: decouple per-family artifact checks so one gap doesn't block others A missing artifact in one backend family (e.g. a single failed CUDA sm_* build) no longer blocks publishing the artifacts from families that did build successfully. Each family (cuda, rocm, openvino, cpu) is checked independently; the release is still created and available artifacts are still uploaded, but the job fails afterward so the gap is still surfaced. Addresses review feedback on lemonade-sdk/llama.cpp#16. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/release.yml | 50 ++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 16ff47a1010a..76c3fb0ede74 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -993,39 +993,47 @@ jobs: fi - name: Check release artifacts + id: check_artifacts env: TAG: ${{ steps.tag.outputs.name }} run: | - missing=0 + declare -A missing_by_family=() check_glob() { - local pattern="$1" + local family="$1" pattern="$2" if ! ls $pattern >/dev/null 2>&1; then - echo "::error::Missing artifact: $pattern" - missing=$((missing + 1)) + echo "::error::[$family] Missing artifact: $pattern" + missing_by_family[$family]=$(( ${missing_by_family[$family]:-0} + 1 )) fi } - # Verify all CUDA SM variants are present for each platform + # Each backend family is checked independently: a gap in one family + # (e.g. a single failed CUDA sm_* build) must not block the release + # of artifacts from families that built successfully. for sm in sm_75 sm_80 sm_86 sm_89 sm_90 sm_100 sm_120; do - check_glob "release/llama-${TAG}-ubuntu-cuda-${sm}-x64.tar.xz" - check_glob "release/llama-${TAG}-ubuntu-cuda-${sm}-arm64.tar.xz" - check_glob "release/llama-${TAG}-windows-cuda-${sm}-x64.7z" + check_glob cuda "release/llama-${TAG}-ubuntu-cuda-${sm}-x64.tar.xz" + check_glob cuda "release/llama-${TAG}-ubuntu-cuda-${sm}-arm64.tar.xz" + check_glob cuda "release/llama-${TAG}-windows-cuda-${sm}-x64.7z" done - # Verify non-CUDA platform artifacts are present - check_glob "release/llama-${TAG}-bin-ubuntu-rocm-*-x64.tar.gz" - check_glob "release/llama-${TAG}-bin-ubuntu-openvino-*-x64.tar.gz" - check_glob "release/llama-${TAG}-bin-win-rocm-*-x64.zip" - check_glob "release/llama-${TAG}-bin-win-cpu-x64.zip" + check_glob rocm "release/llama-${TAG}-bin-ubuntu-rocm-*-x64.tar.gz" + check_glob rocm "release/llama-${TAG}-bin-win-rocm-*-x64.zip" + check_glob openvino "release/llama-${TAG}-bin-ubuntu-openvino-*-x64.tar.gz" + check_glob cpu "release/llama-${TAG}-bin-win-cpu-x64.zip" - if [ "$missing" -gt 0 ]; then - echo "::error::$missing artifact(s) missing — aborting before creating a release." - exit 1 - fi + missing=0 + for family in "${!missing_by_family[@]}"; do + echo "::warning::$family family is missing ${missing_by_family[$family]} artifact(s) — publishing the other families anyway." + missing=$((missing + missing_by_family[$family])) + done + echo "missing=$missing" >> "$GITHUB_OUTPUT" files=$(find ./release -maxdepth 1 \( -name '*.zip' -o -name '*.tar.gz' -o -name '*.tar.xz' -o -name '*.7z' \) 2>/dev/null | wc -l) - echo "All expected artifacts present. Found $files artifact(s) ready to upload." + if [ "$missing" -gt 0 ]; then + echo "$missing artifact(s) missing across ${#missing_by_family[@]} family/families — continuing to publish the $files artifact(s) that did build. The job will still fail below so the gap gets noticed." + else + echo "All expected artifacts present. Found $files artifact(s) ready to upload." + fi # Get the release for this tag, creating it if it does not exist yet. # This is idempotent: if a previous run created the release but failed @@ -1154,3 +1162,9 @@ jobs: } core.info(`Done: ${uploaded} uploaded, ${skipped} already present, ${files.length} total.`); + + - name: Fail if any artifacts were missing + if: steps.check_artifacts.outputs.missing != '0' + run: | + echo "::error::${{ steps.check_artifacts.outputs.missing }} artifact(s) were missing from this release — see the 'Check release artifacts' step above for details. The artifacts that did build were still published." + exit 1 From 34e6acd3315cbe9dec2539f7b36aa94b6a9ee899 Mon Sep 17 00:00:00 2001 From: Ken VanDine Date: Thu, 2 Jul 2026 08:53:41 -0400 Subject: [PATCH 3/7] ci: guard against creating an empty release and clarify the failure message Fail before touching the GitHub release if ./release ends up with zero artifacts, instead of relying on per-family checks alone. Also carry the per-family missing summary into the final failure message so it's self-contained instead of pointing back at an earlier step's log. Addresses review feedback on lemonade-sdk/llama.cpp#16. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/release.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 76c3fb0ede74..851c11ff5c87 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1022,15 +1022,25 @@ jobs: check_glob cpu "release/llama-${TAG}-bin-win-cpu-x64.zip" missing=0 + summary="" for family in "${!missing_by_family[@]}"; do echo "::warning::$family family is missing ${missing_by_family[$family]} artifact(s) — publishing the other families anyway." missing=$((missing + missing_by_family[$family])) + summary="${summary}${summary:+, }${family}: ${missing_by_family[$family]} missing" done echo "missing=$missing" >> "$GITHUB_OUTPUT" + echo "summary=$summary" >> "$GITHUB_OUTPUT" files=$(find ./release -maxdepth 1 \( -name '*.zip' -o -name '*.tar.gz' -o -name '*.tar.xz' -o -name '*.7z' \) 2>/dev/null | wc -l) + + # Never create or update a release with nothing to publish. + if [ "$files" -eq 0 ]; then + echo "::error::No release artifacts found in ./release — aborting before creating a release." + exit 1 + fi + if [ "$missing" -gt 0 ]; then - echo "$missing artifact(s) missing across ${#missing_by_family[@]} family/families — continuing to publish the $files artifact(s) that did build. The job will still fail below so the gap gets noticed." + echo "$missing artifact(s) missing ($summary) — continuing to publish the $files artifact(s) that did build. The job will still fail below so the gap gets noticed." else echo "All expected artifacts present. Found $files artifact(s) ready to upload." fi @@ -1166,5 +1176,5 @@ jobs: - name: Fail if any artifacts were missing if: steps.check_artifacts.outputs.missing != '0' run: | - echo "::error::${{ steps.check_artifacts.outputs.missing }} artifact(s) were missing from this release — see the 'Check release artifacts' step above for details. The artifacts that did build were still published." + echo "::error::${{ steps.check_artifacts.outputs.missing }} artifact(s) were missing (${{ steps.check_artifacts.outputs.summary }}) — the artifacts that did build were still published, see the 'Check release artifacts' step for the full per-file list." exit 1 From 33312dd54b84e86d77d8bf2769c357918abe5118 Mon Sep 17 00:00:00 2001 From: Ken VanDine Date: Thu, 2 Jul 2026 09:05:06 -0400 Subject: [PATCH 4/7] ci: drop the whole backend family when any of its artifacts are missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than publishing whichever individual files happened to build, a family (cuda, rocm, openvino, cpu) is now all-or-nothing: if any required artifact in a family is missing, every artifact for that family is dropped from the release. This guarantees that if a family is present in a release at all, it's a complete, internally-consistent set across every platform/variant — e.g. CUDA is never split across sm_* or platforms between two different backend versions. Other families are unaffected by a gap in one. Addresses review feedback on lemonade-sdk/llama.cpp#16. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/release.yml | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 851c11ff5c87..6343453e1204 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -999,6 +999,16 @@ jobs: run: | declare -A missing_by_family=() + # Glob that matches every artifact belonging to a family, used to + # drop the whole family if any piece of it is missing. Must not + # overlap with any other family's files. + declare -A family_glob=( + [cuda]="release/llama-${TAG}-*cuda-sm_*" + [rocm]="release/llama-${TAG}-bin-*rocm-*" + [openvino]="release/llama-${TAG}-bin-*openvino-*" + [cpu]="release/llama-${TAG}-bin-win-cpu-*" + ) + check_glob() { local family="$1" pattern="$2" if ! ls $pattern >/dev/null 2>&1; then @@ -1007,9 +1017,12 @@ jobs: fi } - # Each backend family is checked independently: a gap in one family - # (e.g. a single failed CUDA sm_* build) must not block the release - # of artifacts from families that built successfully. + # Each backend family is checked independently, but must be + # published as a complete, internally-consistent set across every + # platform/variant it covers (e.g. all 7 CUDA sm_* builds for + # ubuntu x64/arm64 and windows) — never a partial mix. A gap in one + # family drops that whole family from the release; other families + # are unaffected. for sm in sm_75 sm_80 sm_86 sm_89 sm_90 sm_100 sm_120; do check_glob cuda "release/llama-${TAG}-ubuntu-cuda-${sm}-x64.tar.xz" check_glob cuda "release/llama-${TAG}-ubuntu-cuda-${sm}-arm64.tar.xz" @@ -1024,9 +1037,11 @@ jobs: missing=0 summary="" for family in "${!missing_by_family[@]}"; do - echo "::warning::$family family is missing ${missing_by_family[$family]} artifact(s) — publishing the other families anyway." - missing=$((missing + missing_by_family[$family])) - summary="${summary}${summary:+, }${family}: ${missing_by_family[$family]} missing" + count=${missing_by_family[$family]} + echo "::warning::$family family is incomplete ($count artifact(s) missing) — dropping all $family artifacts from this release." + rm -fv ${family_glob[$family]} 2>/dev/null || true + missing=$((missing + count)) + summary="${summary}${summary:+, }${family}: $count missing, family dropped" done echo "missing=$missing" >> "$GITHUB_OUTPUT" echo "summary=$summary" >> "$GITHUB_OUTPUT" @@ -1040,7 +1055,7 @@ jobs: fi if [ "$missing" -gt 0 ]; then - echo "$missing artifact(s) missing ($summary) — continuing to publish the $files artifact(s) that did build. The job will still fail below so the gap gets noticed." + echo "$missing artifact(s) missing ($summary) — continuing to publish the $files artifact(s) from complete families. The job will still fail below so the gap gets noticed." else echo "All expected artifacts present. Found $files artifact(s) ready to upload." fi @@ -1176,5 +1191,5 @@ jobs: - name: Fail if any artifacts were missing if: steps.check_artifacts.outputs.missing != '0' run: | - echo "::error::${{ steps.check_artifacts.outputs.missing }} artifact(s) were missing (${{ steps.check_artifacts.outputs.summary }}) — the artifacts that did build were still published, see the 'Check release artifacts' step for the full per-file list." + echo "::error::${{ steps.check_artifacts.outputs.missing }} artifact(s) were missing (${{ steps.check_artifacts.outputs.summary }}) — the incomplete families were dropped from this release; other families were still published. See the 'Check release artifacts' step for the full per-file list." exit 1 From e8991e55c9c05a317700167684542c5254853a6c Mon Sep 17 00:00:00 2001 From: Ken VanDine Date: Thu, 2 Jul 2026 09:36:34 -0400 Subject: [PATCH 5/7] ci: run release job even when a build job fails, and don't let a missing Windows CPU artifact block unrelated families MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps that let the per-family completeness logic get bypassed entirely: - The release job had no always(), so GitHub Actions' default skip-if-a-needed-job-failed behavior meant a single failed build job (e.g. one CUDA sm_* matrix leg) could skip the release job outright, before the family checks ever ran. - The Windows CPU-backend merge step still hard-exited the whole job if the CPU zip was missing, which — since it runs before the Ubuntu/CUDA renaming loops — blocked Ubuntu ROCm/OpenVINO/CUDA from publishing too, for a reason that had nothing to do with them. Now the release job always runs when the workflow is a schedule/dispatch run, and a missing Windows CPU artifact just skips moving the Windows zip-based artifacts (which structurally require it merged in) into release/, leaving Ubuntu/CUDA .7z artifacts unaffected. The existing completeness check then correctly sees the missing cpu/rocm files and drops those families. Addresses review feedback on lemonade-sdk/llama.cpp#16. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/release.yml | 38 ++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6343453e1204..35e77b2861f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -883,7 +883,12 @@ jobs: name: llama-windows-cuda-${{ matrix.sm }}-x64.7z release: - if: ${{ github.event_name == 'schedule' || github.event.inputs.create_release == 'true' }} + # always() overrides the default behavior of skipping this job when a + # needed build job fails or is skipped (e.g. one CUDA sm_* matrix leg + # fails, or windows-cuda gets skipped because windows-cpu failed). The + # job must still run so the per-family checks below can decide what's + # actually complete and safe to publish. + if: ${{ always() && (github.event_name == 'schedule' || github.event.inputs.create_release == 'true') }} # Fine-grant permission # https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token @@ -925,12 +930,21 @@ jobs: run: | mkdir -p release + # Windows CPU is a hard prerequisite for every Windows zip-based + # artifact (its binaries get merged into e.g. the ROCm zip below), + # not just its own "cpu" family. If it's missing, none of those + # zips can be completed, so skip moving any of them into release/ + # rather than hard-failing the whole job — Ubuntu/CUDA artifacts + # below are unaffected, and the completeness check further down + # will see the missing files and drop the cpu/rocm families. echo "Adding CPU backend files to Windows ZIP archives..." + cpu_zip_available=true for arch in x64; do cpu_zip="artifact/llama-bin-win-cpu-${arch}.zip" if [ ! -f "$cpu_zip" ]; then - echo "::error::Missing required CPU artifact $cpu_zip" - exit 1 + echo "::warning::Missing CPU artifact $cpu_zip — Windows zip-based artifacts (cpu, rocm) cannot be completed and will be dropped by the completeness check below." + cpu_zip_available=false + continue fi temp_dir=$(mktemp -d) echo "Extracting CPU backend for $arch..." @@ -949,13 +963,17 @@ jobs: rm -rf "$temp_dir" done - echo "Renaming and moving zips to release..." - for zip_file in artifact/llama-bin-win-*.zip; do - base_name=$(basename "$zip_file" .zip) - zip_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.zip" - echo "Moving $zip_file to release/$zip_name" - mv "$zip_file" "release/$zip_name" - done + if [ "$cpu_zip_available" = true ]; then + echo "Renaming and moving zips to release..." + for zip_file in artifact/llama-bin-win-*.zip; do + base_name=$(basename "$zip_file" .zip) + zip_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.zip" + echo "Moving $zip_file to release/$zip_name" + mv "$zip_file" "release/$zip_name" + done + else + echo "Skipping Windows zip artifacts (cpu, rocm) — CPU backend was unavailable to merge into them." + fi echo "Renaming and moving tar.gz files to release..." for tar_file in artifact/*.tar.gz; do From 87f8d0fe0d9ffe26edf5391e458a9c1739e6055c Mon Sep 17 00:00:00 2001 From: Ken VanDine Date: Thu, 2 Jul 2026 09:56:34 -0400 Subject: [PATCH 6/7] ci: guard the tar.gz move loop against an unmatched glob The CUDA .tar.xz and .7z move loops already skip when their glob doesn't match any files; the tar.gz loop (ROCm/OpenVINO) was missing the same guard. If both tar.gz producers are absent, the loop would try to mv the literal artifact/*.tar.gz and fail before the family completeness check could run and drop ROCm/OpenVINO while still publishing CUDA/Windows CPU. Addresses review feedback on lemonade-sdk/llama.cpp#16. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 35e77b2861f2..5557f26cdcf8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -977,6 +977,7 @@ jobs: echo "Renaming and moving tar.gz files to release..." for tar_file in artifact/*.tar.gz; do + [ -f "$tar_file" ] || continue base_name=$(basename "$tar_file" .tar.gz) tar_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.tar.gz" echo "Moving $tar_file to release/$tar_name" From f0883c99a756f29fdcf534c387e443b0fbf97481 Mon Sep 17 00:00:00 2001 From: Ken VanDine Date: Thu, 2 Jul 2026 10:01:54 -0400 Subject: [PATCH 7/7] ci: prune stale release assets from families dropped in the current run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same fix as lemonade-sdk/stable-diffusion.cpp#14: the release is reused across runs against the same tag (idempotent create-or-get). If a family (e.g. rocm) was complete and published in an earlier run but the current run finds it incomplete and drops it locally, its assets from that earlier run were never removed — the release would keep looking complete for a family this run couldn't actually reproduce. The upload step now reconciles existing release assets against the current ./release file set: anything not fully uploaded (a broken partial upload, as before) or not present in this run's file set (stale leftovers from a dropped family) gets deleted before uploading. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/release.yml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5557f26cdcf8..3199f72f9a63 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1159,22 +1159,29 @@ jobs: } } - // Assets already attached (from an earlier partial run). GitHub only - // creates an asset once its upload completes, so anything listed here - // is intact and can be skipped — re-runs only fill the gaps. + const files = fs.readdirSync('./release').filter((f) => + f.endsWith('.zip') || f.endsWith('.tar.gz') || f.endsWith('.tar.xz') || f.endsWith('.7z')); + const wanted = new Set(files); + + // Reconcile existing release assets against what this run decided to + // publish: drop anything not fully uploaded (a broken partial upload) + // and anything not in the current file set. The latter case matters + // because the release is reused across runs against the same tag — + // if a family was complete and published in an earlier run but this + // run finds it incomplete and drops it locally, its assets from that + // earlier run would otherwise linger and make the family look + // complete even though this run couldn't reproduce it. const existing = new Set(); for (const a of await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id })) { - if (a.state === 'uploaded') { + if (a.state === 'uploaded' && wanted.has(a.name)) { existing.add(a.name); } else { - core.warning(`deleting incomplete asset ${a.name} (${a.state})`); + const reason = a.state !== 'uploaded' ? `incomplete (${a.state})` : 'stale (not part of this run)'; + core.warning(`deleting ${reason} asset ${a.name}`); await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: a.id }); } } - const files = fs.readdirSync('./release').filter((f) => - f.endsWith('.zip') || f.endsWith('.tar.gz') || f.endsWith('.tar.xz') || f.endsWith('.7z')); - let uploaded = 0; let skipped = 0; for (const file of files) {