diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6492e41db58d..3199f72f9a63 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -883,7 +883,12 @@ jobs: name: llama-windows-cuda-${{ matrix.sm }}-x64.7z release: - if: ${{ github.event_name == 'schedule' || github.event.inputs.create_release == 'true' }} + # always() overrides the default behavior of skipping this job when a + # needed build job fails or is skipped (e.g. one CUDA sm_* matrix leg + # fails, or windows-cuda gets skipped because windows-cpu failed). The + # job must still run so the per-family checks below can decide what's + # actually complete and safe to publish. + if: ${{ always() && (github.event_name == 'schedule' || github.event.inputs.create_release == 'true') }} # Fine-grant permission # https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token @@ -925,12 +930,21 @@ jobs: run: | mkdir -p release + # Windows CPU is a hard prerequisite for every Windows zip-based + # artifact (its binaries get merged into e.g. the ROCm zip below), + # not just its own "cpu" family. If it's missing, none of those + # zips can be completed, so skip moving any of them into release/ + # rather than hard-failing the whole job — Ubuntu/CUDA artifacts + # below are unaffected, and the completeness check further down + # will see the missing files and drop the cpu/rocm families. echo "Adding CPU backend files to Windows ZIP archives..." + cpu_zip_available=true for arch in x64; do cpu_zip="artifact/llama-bin-win-cpu-${arch}.zip" if [ ! -f "$cpu_zip" ]; then - echo "::error::Missing required CPU artifact $cpu_zip" - exit 1 + echo "::warning::Missing CPU artifact $cpu_zip — Windows zip-based artifacts (cpu, rocm) cannot be completed and will be dropped by the completeness check below." + cpu_zip_available=false + continue fi temp_dir=$(mktemp -d) echo "Extracting CPU backend for $arch..." @@ -949,16 +963,21 @@ jobs: rm -rf "$temp_dir" done - echo "Renaming and moving zips to release..." - for zip_file in artifact/llama-bin-win-*.zip; do - base_name=$(basename "$zip_file" .zip) - zip_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.zip" - echo "Moving $zip_file to release/$zip_name" - mv "$zip_file" "release/$zip_name" - done + if [ "$cpu_zip_available" = true ]; then + echo "Renaming and moving zips to release..." + for zip_file in artifact/llama-bin-win-*.zip; do + base_name=$(basename "$zip_file" .zip) + zip_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.zip" + echo "Moving $zip_file to release/$zip_name" + mv "$zip_file" "release/$zip_name" + done + else + echo "Skipping Windows zip artifacts (cpu, rocm) — CPU backend was unavailable to merge into them." + fi echo "Renaming and moving tar.gz files to release..." for tar_file in artifact/*.tar.gz; do + [ -f "$tar_file" ] || continue base_name=$(basename "$tar_file" .tar.gz) tar_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.tar.gz" echo "Moving $tar_file to release/$tar_name" @@ -993,13 +1012,72 @@ jobs: fi - name: Check release artifacts + id: check_artifacts + env: + TAG: ${{ steps.tag.outputs.name }} run: | + declare -A missing_by_family=() + + # Glob that matches every artifact belonging to a family, used to + # drop the whole family if any piece of it is missing. Must not + # overlap with any other family's files. + declare -A family_glob=( + [cuda]="release/llama-${TAG}-*cuda-sm_*" + [rocm]="release/llama-${TAG}-bin-*rocm-*" + [openvino]="release/llama-${TAG}-bin-*openvino-*" + [cpu]="release/llama-${TAG}-bin-win-cpu-*" + ) + + check_glob() { + local family="$1" pattern="$2" + if ! ls $pattern >/dev/null 2>&1; then + echo "::error::[$family] Missing artifact: $pattern" + missing_by_family[$family]=$(( ${missing_by_family[$family]:-0} + 1 )) + fi + } + + # Each backend family is checked independently, but must be + # published as a complete, internally-consistent set across every + # platform/variant it covers (e.g. all 7 CUDA sm_* builds for + # ubuntu x64/arm64 and windows) — never a partial mix. A gap in one + # family drops that whole family from the release; other families + # are unaffected. + for sm in sm_75 sm_80 sm_86 sm_89 sm_90 sm_100 sm_120; do + check_glob cuda "release/llama-${TAG}-ubuntu-cuda-${sm}-x64.tar.xz" + check_glob cuda "release/llama-${TAG}-ubuntu-cuda-${sm}-arm64.tar.xz" + check_glob cuda "release/llama-${TAG}-windows-cuda-${sm}-x64.7z" + done + + check_glob rocm "release/llama-${TAG}-bin-ubuntu-rocm-*-x64.tar.gz" + check_glob rocm "release/llama-${TAG}-bin-win-rocm-*-x64.zip" + check_glob openvino "release/llama-${TAG}-bin-ubuntu-openvino-*-x64.tar.gz" + check_glob cpu "release/llama-${TAG}-bin-win-cpu-x64.zip" + + missing=0 + summary="" + for family in "${!missing_by_family[@]}"; do + count=${missing_by_family[$family]} + echo "::warning::$family family is incomplete ($count artifact(s) missing) — dropping all $family artifacts from this release." + rm -fv ${family_glob[$family]} 2>/dev/null || true + missing=$((missing + count)) + summary="${summary}${summary:+, }${family}: $count missing, family dropped" + done + echo "missing=$missing" >> "$GITHUB_OUTPUT" + echo "summary=$summary" >> "$GITHUB_OUTPUT" + files=$(find ./release -maxdepth 1 \( -name '*.zip' -o -name '*.tar.gz' -o -name '*.tar.xz' -o -name '*.7z' \) 2>/dev/null | wc -l) + + # Never create or update a release with nothing to publish. if [ "$files" -eq 0 ]; then - echo "No release artifacts found in ./release — aborting before creating a release." + echo "::error::No release artifacts found in ./release — aborting before creating a release." exit 1 fi - echo "Found $files artifact(s) ready to upload." + + if [ "$missing" -gt 0 ]; then + echo "$missing artifact(s) missing ($summary) — continuing to publish the $files artifact(s) from complete families. The job will still fail below so the gap gets noticed." + else + echo "All expected artifacts present. Found $files artifact(s) ready to upload." + fi # Get the release for this tag, creating it if it does not exist yet. # This is idempotent: if a previous run created the release but failed @@ -1081,22 +1159,29 @@ jobs: } } - // Assets already attached (from an earlier partial run). GitHub only - // creates an asset once its upload completes, so anything listed here - // is intact and can be skipped — re-runs only fill the gaps. + const files = fs.readdirSync('./release').filter((f) => + f.endsWith('.zip') || f.endsWith('.tar.gz') || f.endsWith('.tar.xz') || f.endsWith('.7z')); + const wanted = new Set(files); + + // Reconcile existing release assets against what this run decided to + // publish: drop anything not fully uploaded (a broken partial upload) + // and anything not in the current file set. The latter case matters + // because the release is reused across runs against the same tag — + // if a family was complete and published in an earlier run but this + // run finds it incomplete and drops it locally, its assets from that + // earlier run would otherwise linger and make the family look + // complete even though this run couldn't reproduce it. const existing = new Set(); for (const a of await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id })) { - if (a.state === 'uploaded') { + if (a.state === 'uploaded' && wanted.has(a.name)) { existing.add(a.name); } else { - core.warning(`deleting incomplete asset ${a.name} (${a.state})`); + const reason = a.state !== 'uploaded' ? `incomplete (${a.state})` : 'stale (not part of this run)'; + core.warning(`deleting ${reason} asset ${a.name}`); await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: a.id }); } } - const files = fs.readdirSync('./release').filter((f) => - f.endsWith('.zip') || f.endsWith('.tar.gz') || f.endsWith('.tar.xz') || f.endsWith('.7z')); - let uploaded = 0; let skipped = 0; for (const file of files) { @@ -1128,3 +1213,9 @@ jobs: } core.info(`Done: ${uploaded} uploaded, ${skipped} already present, ${files.length} total.`); + + - name: Fail if any artifacts were missing + if: steps.check_artifacts.outputs.missing != '0' + run: | + echo "::error::${{ steps.check_artifacts.outputs.missing }} artifact(s) were missing (${{ steps.check_artifacts.outputs.summary }}) — the incomplete families were dropped from this release; other families were still published. See the 'Check release artifacts' step for the full per-file list." + exit 1